diff --git a/bin/common.py b/bin/common.py index 0416b964..3d2f83f2 100755 --- a/bin/common.py +++ b/bin/common.py @@ -1,4 +1,5 @@ import os +import subprocess import sys import time import yaml @@ -34,3 +35,42 @@ class ComposeFile(object): with open(filename, "w") as f: yaml.safe_dump(self.data, f, default_flow_style=False) +# Executes a bunch of commands in parallel, but no more than N at a time. +# This allows to execute concurrently a large number of tasks, without +# turning into a fork bomb. +# `parallelism` is the number of tasks to execute simultaneously. +# `commands` is a list of tasks to execute. +# Each task is itself a list, where the first element is a descriptive +# string, and the folloowing elements are the arguments to pass to Popen. +def parallel_run(commands, parallelism): + running = [] + # While stuff is running, or we have stuff to run... + while commands or running: + # While there is stuff to run, and room in the pipe... + while commands and len(running)