From b47dccdb8d0db82276875c31f82b82a4ff5b8e1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Mon, 29 Feb 2016 18:17:06 +0000 Subject: [PATCH] Move scripts to `bin` directory; abstract Compose V1/V2 formats --- .gitignore | 1 + bin/add-load-balancer.py | 42 +++++++++++ bin/build-tag-push.py | 61 ++++++++++++++++ bin/common.py | 36 ++++++++++ .../configure-ambassadors.py | 0 .../create-ambassadors.py | 0 .../link-to-ambassadors.py | 0 build-tag-push.py | 69 ------------------- 8 files changed, 140 insertions(+), 69 deletions(-) create mode 100755 bin/add-load-balancer.py create mode 100755 bin/build-tag-push.py create mode 100755 bin/common.py rename configure-ambassadors.py => bin/configure-ambassadors.py (100%) rename create-ambassadors.py => bin/create-ambassadors.py (100%) rename link-to-ambassadors.py => bin/link-to-ambassadors.py (100%) delete mode 100755 build-tag-push.py diff --git a/.gitignore b/.gitignore index 334ab705..f60fb0d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.pyc *.swp *~ ips.txt diff --git a/bin/add-load-balancer.py b/bin/add-load-balancer.py new file mode 100755 index 00000000..d2f666f8 --- /dev/null +++ b/bin/add-load-balancer.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +import os +import sys +import yaml + +# arg 1 = service name +# arg 2 = number of instances + +service_name = sys.argv[1] +desired_instances = int(sys.argv[2]) + +compose_file = os.environ["COMPOSE_FILE"] +input_file, output_file = compose_file, compose_file + +config = yaml.load(open(input_file)) + +# The ambassadors need to know the service port to use. +# Those ports must be declared here. +ports = yaml.load(open("ports.yml")) + +port = str(ports[service_name]) + +command_line = port + +depends_on = [] + +for n in range(1, 1+desired_instances): + config["services"]["{}{}".format(service_name, n)] = config["services"][service_name] + command_line += " {}{}:{}".format(service_name, n, port) + depends_on.append("{}{}".format(service_name, n)) + +config["services"][service_name] = { + "image": "jpetazzo/hamba", + "command": command_line, + "depends_on": depends_on, +} +if "networks" in config["services"]["{}1".format(service_name)]: + config["services"][service_name]["networks"] = config["services"]["{}1".format(service_name)]["networks"] + +yaml.safe_dump(config, open(output_file, "w"), default_flow_style=False) + diff --git a/bin/build-tag-push.py b/bin/build-tag-push.py new file mode 100755 index 00000000..77401b0e --- /dev/null +++ b/bin/build-tag-push.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +from common import ComposeFile +import os +import subprocess +import time + +registry = os.environ.get("DOCKER_REGISTRY") + +if not registry: + print("Please set the DOCKER_REGISTRY variable, e.g.:") + print("export DOCKER_REGISTRY=jpetazzo # use the Docker Hub") + print("export DOCKER_REGISTRY=localhost:5000 # use a local registry") + exit(1) + +# Get the name of the current directory. +project_name = os.path.basename(os.path.realpath(".")) + +# Generate a Docker image tag, using the UNIX timestamp. +# (i.e. number of seconds since January 1st, 1970) +version = str(int(time.time())) + +# Execute "docker-compose build" and abort if it fails. +subprocess.check_call(["docker-compose", "-f", "docker-compose.yml", "build"]) + +# Load the services from the input docker-compose.yml file. +# TODO: run parallel builds. +compose_file = ComposeFile("docker-compose.yml") + +# Iterate over all services that have a "build" definition. +# Tag them, and initiate a push in the background. +push_operations = dict() +for service_name, service in compose_file.services.items(): + if "build" in service: + compose_image = "{}_{}".format(project_name, service_name) + registry_image = "{}/{}_{}:{}".format(registry, project_name, service_name, version) + # Re-tag the image so that it can be uploaded to the registry. + subprocess.check_call(["docker", "tag", compose_image, registry_image]) + # Spawn "docker push" to upload the image. + push_operations[service_name] = subprocess.Popen(["docker", "push", registry_image]) + # Replace the "build" definition by an "image" definition, + # using the name of the image on the registry. + del service["build"] + service["image"] = registry_image + +# Wait for push operations to complete. +for service_name, popen_object in push_operations.items(): + print("Waiting for {} push to complete...".format(service_name)) + popen_object.wait() + print("Done.") + +# Write the new docker-compose.yml file. +if "COMPOSE_FILE" not in os.environ: + os.environ["COMPOSE_FILE"] = "docker-compose.yml-{}".format(version) + print("Writing to new Compose file:") +else: + print("Writing to provided Compose file:") + +print("COMPOSE_FILE={}".format(os.environ["COMPOSE_FILE"])) +compose_file.save() + diff --git a/bin/common.py b/bin/common.py new file mode 100755 index 00000000..0416b964 --- /dev/null +++ b/bin/common.py @@ -0,0 +1,36 @@ +import os +import sys +import time +import yaml + + +def COMPOSE_FILE(): + if "COMPOSE_FILE" not in os.environ: + print("The $COMPOSE_FILE environment variable is not set. Aborting.") + exit(1) + return os.environ["COMPOSE_FILE"] + + +class ComposeFile(object): + + def __init__(self, filename=None): + if filename is None: + filename = COMPOSE_FILE() + if not os.path.isfile(filename): + print("File {!r} does not exist. Aborting.".format(filename)) + exit(1) + self.data = yaml.load(open(filename)) + + @property + def services(self): + if self.data.get("version") == "2": + return self.data["services"] + else: + return self.data + + def save(self, filename=None): + if filename is None: + filename = COMPOSE_FILE() + with open(filename, "w") as f: + yaml.safe_dump(self.data, f, default_flow_style=False) + diff --git a/configure-ambassadors.py b/bin/configure-ambassadors.py similarity index 100% rename from configure-ambassadors.py rename to bin/configure-ambassadors.py diff --git a/create-ambassadors.py b/bin/create-ambassadors.py similarity index 100% rename from create-ambassadors.py rename to bin/create-ambassadors.py diff --git a/link-to-ambassadors.py b/bin/link-to-ambassadors.py similarity index 100% rename from link-to-ambassadors.py rename to bin/link-to-ambassadors.py diff --git a/build-tag-push.py b/build-tag-push.py deleted file mode 100755 index 55fbdb9d..00000000 --- a/build-tag-push.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python - -import os -import subprocess -import time -import yaml - -user_name = os.environ.get("DOCKERHUB_USER") - -if not user_name: - print("Please set the DOCKERHUB_USER to your user name, e.g.:") - print("export DOCKERHUB_USER=zoe") - exit(1) - -# Get the name of the current directory. -project_name = os.path.basename(os.path.realpath(".")) - -# Generate a Docker image tag, using the UNIX timestamp. -# (i.e. number of seconds since January 1st, 1970) -version = str(int(time.time())) - -input_file = os.environ.get( - "DOCKER_COMPOSE_YML", "docker-compose.yml") -output_file = os.environ.get( - "DOCKER_COMPOSE_YML", "docker-compose.yml-{}".format(version)) - -if input_file == output_file == "docker-compose.yml": - print("I will not clobber your docker-compose.yml file.") - print("Unset DOCKER_COMPOSE_YML or set it to something else.") - exit(1) - -print("Input file: {}".format(input_file)) -print("Output file: {}".format(output_file)) - -# Execute "docker-compose build" and abort if it fails. -subprocess.check_call(["docker-compose", "-f", input_file, "build"]) - -# Load the services from the input docker-compose.yml file. -# TODO: run parallel builds. -stack = yaml.load(open(input_file)) - -# Iterate over all services that have a "build" definition. -# Tag them, and initiate a push in the background. -push_operations = dict() -for service_name, service in stack.items(): - if "build" in service: - compose_image = "{}_{}".format(project_name, service_name) - hub_image = "{}/{}_{}:{}".format(user_name, project_name, service_name, version) - # Re-tag the image so that it can be uploaded to the Docker Hub. - subprocess.check_call(["docker", "tag", compose_image, hub_image]) - # Spawn "docker push" to upload the image. - push_operations[service_name] = subprocess.Popen(["docker", "push", hub_image]) - # Replace the "build" definition by an "image" definition, - # using the name of the image on the Docker Hub. - del service["build"] - service["image"] = hub_image - -# Wait for push operations to complete. -for service_name, popen_object in push_operations.items(): - print("Waiting for {} push to complete...".format(service_name)) - popen_object.wait() - print("Done.") - -# Write the new docker-compose.yml file. -with open(output_file, "w") as f: - yaml.safe_dump(stack, f, default_flow_style=False) - -print("Wrote new compose file.") -print("COMPOSE_FILE={}".format(output_file))