beugley · February 1, 2022 16:07
diff --git a/run_parallel.sh b/run_parallel.sh
 #!/bin/bash
 # Contains common functions that can be sourced by other bash scripts

 function RunParallel
 # Usage: Set the PROCESSES and CONCURRENCY variables and invoke RunParallel without parameters
 # PROCESSES: An array of commands to execute
 # CONCURRENCY: The number of commands to execute concurrently
 # - Takes a list of processes in the PROCESSES variable, runs them in parallel, waits for
 #   each to complete, and then serializes their output (i.e. not interleaved).
 # - If any process fails, then RunParallel will return a non-0 value.
 # - Assumes that an error trap is NOT in effect.  An error trap will cause termination
 #   of this function upon the first command to fail.
 {
    # Check inputs.
    if ((CONCURRENCY < 1))
    then
        echo "ERROR: CONCURRENCY must be >0"
        return 1
    fi
    num_processes=${#PROCESSES[@]}
    if ((num_processes == 0))
    then
        echo "PROCESSES array can not be empty"
        return 2
    fi

    # Determine concurrency level.
    ((concurrency = num_processes<CONCURRENCY ? num_processes : CONCURRENCY))
    echo "Running ${num_processes} processes total, ${concurrency} concurrently"

    # Loop through "PROCESSES" and execute each in the background, capturing stdout/stderr in
    # a separate file for each.  Once concurrency is reached, wait for an existing process to
    # complete before starting a new one.
    ((total_rc = 0))
    for ((nr=0,np=0; np<${num_processes-1}; np++))
    do
        echo "Loop: nr=${nr}, np=${np}, total_rc=${total_rc}"
        if ((nr >= concurrency))
        then
            # At full concurrency.  Wait for a process to complete before starting another.
            WaitForProcess
        fi

        # Spawn a child process and capture stdout/stderr in a temp file.
        output[np]=$(mktemp -t parallel.XXXXXX)
        echo "Spawning ${np}: \"${PROCESSES[np]}\"..."
        ${PROCESSES[np]} >${output[np]} 2>&1 &
        ((pids[np] = $!))
        ((nr = nr + 1))
        echo "Spawned ${np}: pid ${pids[np]}"
    done

    # Wait for remaining processes to complete, if any.
    while ((nr > 0))
    do
        WaitForProcess
    done
    return $total_rc
 }

 function WaitForProcess
 # - Called by RunParallel
 # - Waits for a specific process to complete
 # - Captures the return code and prints the output to stdout
 {
    ((wait_for = np - nr))   # The index of the process to wait for.
    echo "Waiting for ${wait_for}: ${pids[wait_for]}"
    eval "wait ${pids[wait_for]}"
    rc=$?
    ((total_rc = total_rc|rc))
    echo "=============================================================================="
    echo "PID ${pids[wait_for]}: \"${PROCESSES[wait_for]}\" exited with $rc"
    eval "cat ${output[wait_for]}"
    echo "=============================================================================="
    eval "rm ${output[wait_for]}"
    ((nr = nr - 1))
 }
	#!/bin/bash
	# Contains common functions that can be sourced by other bash scripts

	function RunParallel
	# Usage: Set the PROCESSES and CONCURRENCY variables and invoke RunParallel without parameters
	# PROCESSES: An array of commands to execute
	# CONCURRENCY: The number of commands to execute concurrently
	# - Takes a list of processes in the PROCESSES variable, runs them in parallel, waits for
	# each to complete, and then serializes their output (i.e. not interleaved).
	# - If any process fails, then RunParallel will return a non-0 value.
	# - Assumes that an error trap is NOT in effect. An error trap will cause termination
	# of this function upon the first command to fail.
	{
	# Check inputs.
	if ((CONCURRENCY < 1))
	then
	echo "ERROR: CONCURRENCY must be >0"
	return 1
	fi
	num_processes=${#PROCESSES[@]}
	if ((num_processes == 0))
	then
	echo "PROCESSES array can not be empty"
	return 2
	fi

	# Determine concurrency level.
	((concurrency = num_processes<CONCURRENCY ? num_processes : CONCURRENCY))
	echo "Running ${num_processes} processes total, ${concurrency} concurrently"

	# Loop through "PROCESSES" and execute each in the background, capturing stdout/stderr in
	# a separate file for each. Once concurrency is reached, wait for an existing process to
	# complete before starting a new one.
	((total_rc = 0))
	for ((nr=0,np=0; np<${num_processes-1}; np++))
	do
	echo "Loop: nr=${nr}, np=${np}, total_rc=${total_rc}"
	if ((nr >= concurrency))
	then
	# At full concurrency. Wait for a process to complete before starting another.
	WaitForProcess
	fi

	# Spawn a child process and capture stdout/stderr in a temp file.
	output[np]=$(mktemp -t parallel.XXXXXX)
	echo "Spawning ${np}: \"${PROCESSES[np]}\"..."
	${PROCESSES[np]} >${output[np]} 2>&1 &
	((pids[np] = $!))
	((nr = nr + 1))
	echo "Spawned ${np}: pid ${pids[np]}"
	done

	# Wait for remaining processes to complete, if any.
	while ((nr > 0))
	do
	WaitForProcess
	done
	return $total_rc
	}

	function WaitForProcess
	# - Called by RunParallel
	# - Waits for a specific process to complete
	# - Captures the return code and prints the output to stdout
	{
	((wait_for = np - nr)) # The index of the process to wait for.
	echo "Waiting for ${wait_for}: ${pids[wait_for]}"
	eval "wait ${pids[wait_for]}"
	rc=$?
	((total_rc = total_rc\|rc))
	echo "=============================================================================="
	echo "PID ${pids[wait_for]}: \"${PROCESSES[wait_for]}\" exited with $rc"
	eval "cat ${output[wait_for]}"
	echo "=============================================================================="
	eval "rm ${output[wait_for]}"
	((nr = nr - 1))
	}