Refactor release and build scripts

- Make use of the same Dockerfile when producing
an image for testing and for deploying to
DockerHub

Signed-off-by: Ulysses Souza <[email protected]>

Jenkinsfile

@@ -10,7 +10,13 @@ def buildImage = { String baseImage ->
       try {
         image.pull()
       } catch (Exception exc) {
-        sh "docker build -t ${imageName} --target build --build-arg BUILD_PLATFORM=${baseImage} ."
+        sh """GIT_COMMIT=\$(script/build/write-git-sha) && \\
+            docker build -t ${imageName} \\
+            --target build \\
+            --build-arg BUILD_PLATFORM="${baseImage}" \\
+            --build-arg GIT_COMMIT="${GIT_COMMIT}" \\
+            .\\
+        """
         sh "docker push ${imageName}"
         echo "${imageName}"
         return imageName

script/build/image

@@ -7,11 +7,14 @@ if [ -z "$1" ]; then
     exit 1
 fi
 
-TAG=$1
+TAG="$1"
 
 VERSION="$(python setup.py --version)"
 
-./script/build/write-git-sha
+DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)"
+echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA
 python setup.py sdist bdist_wheel
-./script/build/linux
-docker build -t docker/compose:$TAG -f Dockerfile.run .
+
+docker build \
+    --build-arg GIT_COMMIT="${DOCKER_COMPOSE_GITSHA}" \
+    -t "${TAG}" .

script/build/linux

@@ -4,15 +4,14 @@ set -ex
 
 ./script/clean
 
-TMP_CONTAINER="tmpcontainer"
-TAG="docker/compose:tmp-glibc-linux-binary"
-DOCKER_COMPOSE_GITSHA=$(script/build/write-git-sha)
+DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)"
+TAG="docker/compose:tmp-glibc-linux-binary-${DOCKER_COMPOSE_GITSHA}"
 
 docker build -t "${TAG}" . \
        --build-arg BUILD_PLATFORM=debian \
-       --build-arg GIT_COMMIT=${DOCKER_COMPOSE_GITSHA}
-docker create --name ${TMP_CONTAINER} ${TAG}
+       --build-arg GIT_COMMIT="${DOCKER_COMPOSE_GITSHA}"
+TMP_CONTAINER=$(docker create "${TAG}")
 mkdir -p dist
-docker cp ${TMP_CONTAINER}:/usr/local/bin/docker-compose dist/docker-compose-Linux-x86_64
-docker container rm -f ${TMP_CONTAINER}
-docker image rm -f ${TAG}
+docker cp "${TMP_CONTAINER}":/usr/local/bin/docker-compose dist/docker-compose-Linux-x86_64
+docker container rm -f "${TMP_CONTAINER}"
+docker image rm -f "${TAG}"

script/build/linux-entrypoint

@@ -3,16 +3,19 @@
 set -ex
 
 CODE_PATH=/code
-VENV=${CODE_PATH}/.tox/py37
+VENV="${CODE_PATH}"/.tox/py37
 
-cd ${CODE_PATH}
+cd "${CODE_PATH}"
 mkdir -p dist
 chmod 777 dist
 
-${VENV}/bin/pip3 install -q -r requirements-build.txt
+"${VENV}"/bin/pip3 install -q -r requirements-build.txt
 
 # TODO(ulyssessouza) To check if really needed
-./script/build/write-git-sha
+if [ -z "${DOCKER_COMPOSE_GITSHA}" ]; then
+    DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)"
+fi
+echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA
 
 export PATH="${CODE_PATH}/pyinstaller:${PATH}"
 
@@ -21,15 +24,15 @@ if [ ! -z "${BUILD_BOOTLOADER}" ]; then
     git clone --single-branch --branch master https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller
     cd /tmp/pyinstaller/bootloader
     git checkout v3.4
-    ${VENV}/bin/python3 ./waf configure --no-lsb all
-    ${VENV}/bin/pip3 install ..
-    cd ${CODE_PATH}
+    "${VENV}"/bin/python3 ./waf configure --no-lsb all
+    "${VENV}"/bin/pip3 install ..
+    cd "${CODE_PATH}"
     rm -Rf /tmp/pyinstaller
 else
     echo "NOT compiling bootloader!!!"
 fi
 
-${VENV}/bin/pyinstaller --exclude-module pycrypto --exclude-module PyInstaller docker-compose.spec
+"${VENV}"/bin/pyinstaller --exclude-module pycrypto --exclude-module PyInstaller docker-compose.spec
 ls -la dist/
 ldd dist/docker-compose
 mv dist/docker-compose /usr/local/bin

script/build/osx

@@ -5,11 +5,12 @@ TOOLCHAIN_PATH="$(realpath $(dirname $0)/../../build/toolchain)"
 
 rm -rf venv
 
-virtualenv -p ${TOOLCHAIN_PATH}/bin/python3 venv
+virtualenv -p "${TOOLCHAIN_PATH}"/bin/python3 venv
 venv/bin/pip install -r requirements.txt
 venv/bin/pip install -r requirements-build.txt
 venv/bin/pip install --no-deps .
-./script/build/write-git-sha
+DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)"
+echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA
 venv/bin/pyinstaller docker-compose.spec
 mv dist/docker-compose dist/docker-compose-Darwin-x86_64
 dist/docker-compose-Darwin-x86_64 version

script/build/test-image

@@ -10,9 +10,9 @@ fi
 TAG="$1"
 IMAGE="docker/compose-tests"
 
-DOCKER_COMPOSE_GITSHA=$(script/build/write-git-sha)
+DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)"
 docker build -t "${IMAGE}:${TAG}" . \
        --target build \
-       --build-arg BUILD_PLATFORM=debian \
-       --build-arg GIT_COMMIT=${DOCKER_COMPOSE_GITSHA}
-docker tag ${IMAGE}:${TAG} ${IMAGE}:latest
+       --build-arg BUILD_PLATFORM="debian" \
+       --build-arg GIT_COMMIT="${DOCKER_COMPOSE_GITSHA}"
+docker tag "${IMAGE}":"${TAG}" "${IMAGE}":latest

script/build/write-git-sha

@@ -9,4 +9,4 @@ if [[ "${?}" != "0" ]]; then
     echo "Couldn't get revision of the git repository. Setting to 'unknown' instead"
     DOCKER_COMPOSE_GITSHA="unknown"
 fi
-echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA
+echo "${DOCKER_COMPOSE_GITSHA}"

script/release/release.py

@@ -204,7 +204,7 @@ def resume(args):
         delete_assets(gh_release)
         upload_assets(gh_release, files)
         img_manager = ImageManager(args.release)
-        img_manager.build_images(repository, files)
+        img_manager.build_images(repository)
     except ScriptError as e:
         print(e)
         return 1
@@ -244,7 +244,7 @@ def start(args):
         gh_release = create_release_draft(repository, args.release, pr_data, files)
         upload_assets(gh_release, files)
         img_manager = ImageManager(args.release)
-        img_manager.build_images(repository, files)
+        img_manager.build_images(repository)
     except ScriptError as e:
         print(e)
         return 1
@@ -258,7 +258,8 @@ def finalize(args):
     try:
         check_pypirc()
         repository = Repository(REPO_ROOT, args.repo)
-        img_manager = ImageManager(args.release)
+        tag_as_latest = _check_if_tag_latest(args.release)
+        img_manager = ImageManager(args.release, tag_as_latest)
         pr_data = repository.find_release_pr(args.release)
         if not pr_data:
             raise ScriptError('No PR found for {}'.format(args.release))
@@ -314,6 +315,12 @@ EPILOG = '''Example uses:
 '''
 
 
+# Checks if this version respects the GA version format ('x.y.z') and not an RC
+def _check_if_tag_latest(version):
+    ga_version = all(n.isdigit() for n in version.split('.')) and version.count('.') == 2
+    return ga_version and yesno('Should this release be tagged as \"latest\"? Y/n', default=True)
+
+
 def main():
     if 'GITHUB_TOKEN' not in os.environ:
         print('GITHUB_TOKEN environment variable must be set')

script/release/release/images.py

@@ -5,18 +5,29 @@ from __future__ import unicode_literals
 import base64
 import json
 import os
-import shutil
 
 import docker
+from enum import Enum
 
+from .const import NAME
 from .const import REPO_ROOT
 from .utils import ScriptError
 
 
+class Platform(Enum):
+    ALPINE = 'alpine'
+    DEBIAN = 'debian'
+
+    def __str__(self):
+        return self.value
+
+
 class ImageManager(object):
-    def __init__(self, version):
+    def __init__(self, version, latest=False):
+        self.built_tags = []
         self.docker_client = docker.APIClient(**docker.utils.kwargs_from_env())
         self.version = version
+        self.latest = latest
         if 'HUB_CREDENTIALS' in os.environ:
             print('HUB_CREDENTIALS found in environment, issuing login')
             credentials = json.loads(base64.urlsafe_b64decode(os.environ['HUB_CREDENTIALS']))
@@ -24,16 +35,31 @@ class ImageManager(object):
                 username=credentials['Username'], password=credentials['Password']
             )
 
-    def build_images(self, repository, files):
-        print("Building release images...")
-        repository.write_git_sha()
-        distdir = os.path.join(REPO_ROOT, 'dist')
-        os.makedirs(distdir, exist_ok=True)
-        shutil.copy(files['docker-compose-Linux-x86_64'][0], distdir)
-        os.chmod(os.path.join(distdir, 'docker-compose-Linux-x86_64'), 0o755)
-        print('Building docker/compose image')
+    def _tag(self, image, existing_tag, new_tag):
+        existing_repo_tag = '{image}:{tag}'.format(image=image, tag=existing_tag)
+        new_repo_tag = '{image}:{tag}'.format(image=image, tag=new_tag)
+        self.docker_client.tag(existing_repo_tag, new_repo_tag)
+        self.built_tags.append(new_repo_tag)
+
+    def build_runtime_image(self, repository, platform):
+        git_sha = repository.write_git_sha()
+        compose_image_base_name = NAME
+        print('Building {image} image ({platform} based)'.format(
+            image=compose_image_base_name,
+            platform=platform
+        ))
+        full_version = '{version}-{platform}'.format(version=self.version, platform=platform)
+        build_tag = '{image_base_image}:{full_version}'.format(
+            image_base_image=compose_image_base_name,
+            full_version=full_version
+        )
         logstream = self.docker_client.build(
-            REPO_ROOT, tag='docker/compose:{}'.format(self.version), dockerfile='Dockerfile.run',
+            REPO_ROOT,
+            tag=build_tag,
+            buildargs={
+                'BUILD_PLATFORM': platform.value,
+                'GIT_COMMIT': git_sha,
+            },
             decode=True
         )
         for chunk in logstream:
@@ -42,9 +68,32 @@ class ImageManager(object):
             if 'stream' in chunk:
                 print(chunk['stream'], end='')
 
-        print('Building test image (for UCP e2e)')
+        self.built_tags.append(build_tag)
+        if platform == Platform.ALPINE:
+            self._tag(compose_image_base_name, full_version, self.version)
+        if self.latest:
+            self._tag(compose_image_base_name, full_version, platform)
+            if platform == Platform.ALPINE:
+                self._tag(compose_image_base_name, full_version, 'latest')
+
+    # Used for producing a test image for UCP
+    def build_ucp_test_image(self, repository):
+        print('Building test image (debian based for UCP e2e)')
+        git_sha = repository.write_git_sha()
+        compose_tests_image_base_name = NAME + '-tests'
+        ucp_test_image_tag = '{image}:{tag}'.format(
+            image=compose_tests_image_base_name,
+            tag=self.version
+        )
         logstream = self.docker_client.build(
-            REPO_ROOT, tag='docker-compose-tests:tmp', decode=True
+            REPO_ROOT,
+            tag=ucp_test_image_tag,
+            target='build',
+            buildargs={
+                'BUILD_PLATFORM': Platform.DEBIAN.value,
+                'GIT_COMMIT': git_sha,
+            },
+            decode=True
         )
         for chunk in logstream:
             if 'error' in chunk:
@@ -52,26 +101,16 @@ class ImageManager(object):
             if 'stream' in chunk:
                 print(chunk['stream'], end='')
 
-        container = self.docker_client.create_container(
-            'docker-compose-tests:tmp', entrypoint='tox'
-        )
-        self.docker_client.commit(container, 'docker/compose-tests', 'latest')
-        self.docker_client.tag(
-            'docker/compose-tests:latest', 'docker/compose-tests:{}'.format(self.version)
-        )
-        self.docker_client.remove_container(container, force=True)
-        self.docker_client.remove_image('docker-compose-tests:tmp', force=True)
+        self.built_tags.append(ucp_test_image_tag)
+        self._tag(compose_tests_image_base_name, self.version, 'latest')
 
-    @property
-    def image_names(self):
-        return [
-            'docker/compose-tests:latest',
-            'docker/compose-tests:{}'.format(self.version),
-            'docker/compose:{}'.format(self.version)
-        ]
+    def build_images(self, repository):
+        self.build_runtime_image(repository, Platform.ALPINE)
+        self.build_runtime_image(repository, Platform.DEBIAN)
+        self.build_ucp_test_image(repository)
 
     def check_images(self):
-        for name in self.image_names:
+        for name in self.built_tags:
             try:
                 self.docker_client.inspect_image(name)
             except docker.errors.ImageNotFound:
@@ -80,7 +119,7 @@ class ImageManager(object):
         return True
 
     def push_images(self):
-        for name in self.image_names:
+        for name in self.built_tags:
             print('Pushing {} to Docker Hub'.format(name))
             logstream = self.docker_client.push(name, stream=True, decode=True)
             for chunk in logstream:

script/release/release/repository.py

@@ -175,6 +175,7 @@ class Repository(object):
     def write_git_sha(self):
         with open(os.path.join(REPO_ROOT, 'compose', 'GITSHA'), 'w') as f:
             f.write(self.git_repo.head.commit.hexsha[:7])
+        return self.git_repo.head.commit.hexsha[:7]
 
     def cherry_pick_prs(self, release_branch, ids):
         if not ids:

script/run/run.sh

@@ -48,7 +48,7 @@ fi
 
 # Only allocate tty if we detect one
 if [ -t 0 -a -t 1 ]; then
-        DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t"
+    DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t"
 fi
 
 # Always set -i to support piped and terminal input in run/exec

script/test/all

@@ -8,8 +8,7 @@ set -e
 docker run --rm \
   --tty \
   ${GIT_VOLUME} \
-  --entrypoint="tox" \
-  "$TAG" -e pre-commit
+  "$TAG" tox -e pre-commit
 
 get_versions="docker run --rm
     --entrypoint=/code/.tox/py27/bin/python

script/test/default

@@ -5,16 +5,16 @@ set -ex
 
 TAG="docker-compose:alpine-$(git rev-parse --short HEAD)"
 
-# By default use the Dockerfile.alpine, but can be overridden to use an alternative file
+# By default use the Dockerfile, but can be overridden to use an alternative file
 # e.g DOCKERFILE=Dockerfile.armhf script/test/default
-DOCKERFILE="${DOCKERFILE:-Dockerfile.alpine}"
+DOCKERFILE="${DOCKERFILE:-Dockerfile}"
 DOCKER_BUILD_TARGET="${DOCKER_BUILD_TARGET:-build}"
 
 rm -rf coverage-html
 # Create the host directory so it's owned by $USER
 mkdir -p coverage-html
 
-docker build -f ${DOCKERFILE} -t "${TAG}" --target "${DOCKER_BUILD_TARGET}" .
+docker build -f "${DOCKERFILE}" -t "${TAG}" --target "${DOCKER_BUILD_TARGET}" .
 
 GIT_VOLUME="--volume=$(pwd)/.git:/code/.git"
 . script/test/all