Refactor Dockerfiles for generating musl binaries

- Refactor Dockerfile to be used for tests and distribution on docker hub on debian and alpine
to use for final usage and also tests
- Adapt test scripts to the new Dockerfiles' structure
- Adapt Jenkinsfile to add alpine to the test matrix

Signed-off-by: Ulysses Souza <[email protected]>

Dockerfile

@@ -1,36 +1,71 @@
-FROM docker:18.06.1 as docker
-FROM python:3.7.2-stretch
+ARG DOCKER_VERSION=18.09.5
+ARG PYTHON_VERSION=3.7.3
+ARG BUILD_ALPINE_VERSION=3.9
+ARG BUILD_DEBIAN_VERSION=slim-stretch
+ARG RUNTIME_ALPINE_VERSION=3.9.3
+ARG RUNTIME_DEBIAN_VERSION=stretch-20190326-slim
 
-RUN set -ex; \
-    apt-get update -qq; \
-    apt-get install -y \
-        locales \
-        python-dev \
-        git
+ARG BUILD_PLATFORM=alpine
 
-COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker
+FROM docker:${DOCKER_VERSION} AS docker-cli
 
-# Python3 requires a valid locale
-RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen
-ENV LANG en_US.UTF-8
+FROM python:${PYTHON_VERSION}-alpine${BUILD_ALPINE_VERSION} AS build-alpine
+RUN apk add --no-cache \
+    bash \
+    build-base \
+    ca-certificates \
+    curl \
+    gcc \
+    git \
+    libc-dev \
+    libffi-dev \
+    libgcc \
+    make \
+    musl-dev \
+    openssl \
+    openssl-dev \
+    python2 \
+    python2-dev \
+    zlib-dev
+ENV BUILD_BOOTLOADER=1
 
-RUN useradd -d /home/user -m -s /bin/bash user
-WORKDIR /code/
+FROM python:${PYTHON_VERSION}-${BUILD_DEBIAN_VERSION} AS build-debian
+RUN apt-get update && apt-get install -y \
+    curl \
+    gcc \
+    git \
+    libc-dev \
+    libgcc-6-dev \
+    make \
+    openssl \
+    python2.7-dev
 
+FROM build-${BUILD_PLATFORM} AS build
+COPY docker-compose-entrypoint.sh /usr/local/bin/
+ENTRYPOINT ["sh", "/usr/local/bin/docker-compose-entrypoint.sh"]
+COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker
+WORKDIR /code/
 # FIXME(chris-crone): virtualenv 16.3.0 breaks build, force 16.2.0 until fixed
 RUN pip install virtualenv==16.2.0
 RUN pip install tox==2.9.1
 
-ADD requirements.txt /code/
-ADD requirements-dev.txt /code/
-ADD .pre-commit-config.yaml /code/
-ADD setup.py /code/
-ADD tox.ini /code/
-ADD compose /code/compose/
-ADD README.md /code/
+COPY requirements.txt .
+COPY requirements-dev.txt .
+COPY .pre-commit-config.yaml .
+COPY tox.ini .
+COPY setup.py .
+COPY README.md .
+COPY compose compose/
 RUN tox --notest
+COPY . .
+ARG GIT_COMMIT=unknown
+ENV DOCKER_COMPOSE_GITSHA=$GIT_COMMIT
+RUN script/build/linux-entrypoint
 
-ADD . /code/
-RUN chown -R user /code/
-
-ENTRYPOINT ["/code/.tox/py37/bin/docker-compose"]
+FROM alpine:${RUNTIME_ALPINE_VERSION} AS runtime-alpine
+FROM debian:${RUNTIME_DEBIAN_VERSION} AS runtime-debian
+FROM runtime-${BUILD_PLATFORM} AS runtime
+COPY docker-compose-entrypoint.sh /usr/local/bin/
+ENTRYPOINT ["sh", "/usr/local/bin/docker-compose-entrypoint.sh"]
+COPY --from=docker-cli  /usr/local/bin/docker           /usr/local/bin/docker
+COPY --from=build       /usr/local/bin/docker-compose   /usr/local/bin/docker-compose

Dockerfile.run

@@ -1,19 +0,0 @@
-FROM docker:18.06.1 as docker
-FROM alpine:3.8
-
-ENV GLIBC 2.28-r0
-
-RUN apk update && apk add --no-cache openssl ca-certificates curl libgcc && \
-    curl -fsSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \
-    curl -fsSL -o glibc-$GLIBC.apk https://github.com/sgerrand/alpine-pkg-glibc/releases/download/$GLIBC/glibc-$GLIBC.apk && \
-    apk add --no-cache glibc-$GLIBC.apk && \
-    ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \
-    ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib && \
-    ln -s /usr/lib/libgcc_s.so.1 /usr/glibc-compat/lib && \
-    rm /etc/apk/keys/sgerrand.rsa.pub glibc-$GLIBC.apk && \
-    apk del curl
-
-COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker
-COPY dist/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
-
-ENTRYPOINT ["docker-compose"]

Jenkinsfile

@@ -1,29 +1,32 @@
 #!groovy
 
-def image
-
-def buildImage = { ->
+def buildImage = { String baseImage ->
+  def image
   wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) {
-    stage("build image") {
+    stage("build image for \"${baseImage}\"") {
       checkout(scm)
-      def imageName = "dockerbuildbot/compose:${gitCommit()}"
+      def imageName = "dockerbuildbot/compose:${baseImage}-${gitCommit()}"
       image = docker.image(imageName)
       try {
         image.pull()
       } catch (Exception exc) {
-        image = docker.build(imageName, ".")
-        image.push()
+        sh "docker build -t ${imageName} --target build --build-arg BUILD_PLATFORM=${baseImage} ."
+        sh "docker push ${imageName}"
+        echo "${imageName}"
+        return imageName
       }
     }
   }
+  echo "image.id: ${image.id}"
+  return image.id
 }
 
-def get_versions = { int number ->
+def get_versions = { String imageId, int number ->
   def docker_versions
   wrappedNode(label: "ubuntu && !zfs") {
     def result = sh(script: """docker run --rm \\
         --entrypoint=/code/.tox/py27/bin/python \\
-        ${image.id} \\
+        ${imageId} \\
         /code/script/test/versions.py -n ${number} docker/docker-ce recent
       """, returnStdout: true
     )
@@ -35,6 +38,8 @@ def get_versions = { int number ->
 def runTests = { Map settings ->
   def dockerVersions = settings.get("dockerVersions", null)
   def pythonVersions = settings.get("pythonVersions", null)
+  def baseImage = settings.get("baseImage", null)
+  def imageName = settings.get("image", null)
 
   if (!pythonVersions) {
     throw new Exception("Need Python versions to test. e.g.: `runTests(pythonVersions: 'py27,py37')`")
@@ -45,7 +50,7 @@ def runTests = { Map settings ->
 
   { ->
     wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) {
-      stage("test python=${pythonVersions} / docker=${dockerVersions}") {
+      stage("test python=${pythonVersions} / docker=${dockerVersions} / baseImage=${baseImage}") {
         checkout(scm)
         def storageDriver = sh(script: 'docker info | awk -F \': \' \'$1 == "Storage Driver" { print $2; exit }\'', returnStdout: true).trim()
         echo "Using local system's storage driver: ${storageDriver}"
@@ -55,13 +60,13 @@ def runTests = { Map settings ->
           --privileged \\
           --volume="\$(pwd)/.git:/code/.git" \\
           --volume="/var/run/docker.sock:/var/run/docker.sock" \\
-          -e "TAG=${image.id}" \\
+          -e "TAG=${imageName}" \\
           -e "STORAGE_DRIVER=${storageDriver}" \\
           -e "DOCKER_VERSIONS=${dockerVersions}" \\
           -e "BUILD_NUMBER=\$BUILD_TAG" \\
           -e "PY_TEST_VERSIONS=${pythonVersions}" \\
           --entrypoint="script/test/ci" \\
-          ${image.id} \\
+          ${imageName} \\
           --verbose
         """
       }
@@ -69,15 +74,16 @@ def runTests = { Map settings ->
   }
 }
 
-buildImage()
-
 def testMatrix = [failFast: true]
-def docker_versions = get_versions(2)
-
-for (int i = 0; i < docker_versions.length; i++) {
-  def dockerVersion = docker_versions[i]
-  testMatrix["${dockerVersion}_py27"] = runTests([dockerVersions: dockerVersion, pythonVersions: "py27"])
-  testMatrix["${dockerVersion}_py37"] = runTests([dockerVersions: dockerVersion, pythonVersions: "py37"])
+def baseImages = ['alpine', 'debian']
+def pythonVersions = ['py27', 'py37']
+baseImages.each { baseImage ->
+  def imageName = buildImage(baseImage)
+  get_versions(imageName, 2).each { dockerVersion ->
+    pythonVersions.each { pyVersion ->
+      testMatrix["${baseImage}_${dockerVersion}_${pyVersion}"] = runTests([baseImage: baseImage, image: imageName, dockerVersions: dockerVersion, pythonVersions: pyVersion])
+    }
+  }
 }
 
 parallel(testMatrix)

docker-compose-entrypoint.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -e
+
+# first arg is `-f` or `--some-option`
+if [ "${1#-}" != "$1" ]; then
+	set -- docker-compose "$@"
+fi
+
+# if our command is a valid Docker subcommand, let's invoke it through Docker instead
+# (this allows for "docker run docker ps", etc)
+if docker-compose help "$1" > /dev/null 2>&1; then
+	set -- docker-compose "$@"
+fi
+
+# if we have "--link some-docker:docker" and not DOCKER_HOST, let's set DOCKER_HOST automatically
+if [ -z "$DOCKER_HOST" -a "$DOCKER_PORT_2375_TCP" ]; then
+	export DOCKER_HOST='tcp://docker:2375'
+fi
+
+exec "$@"

pyinstaller/ldd

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# From http://wiki.musl-libc.org/wiki/FAQ#Q:_where_is_ldd_.3F
+#
+#     Musl's dynlinker comes with ldd functionality built in. just create a
+#     symlink from ld-musl-$ARCH.so to /bin/ldd. If the dynlinker was started
+#     as "ldd", it will detect that and print the appropriate DSO information.
+#
+# Instead, this string replaced "ldd" with the package so that pyinstaller
+# can find the actual lib.
+exec /usr/bin/ldd "$@" | \
+    sed -r 's/([^[:space:]]+) => ldd/\1 => \/lib\/\1/g' | \
+    sed -r 's/ldd \(.*\)//g'

script/build/linux

@@ -4,10 +4,15 @@ set -ex
 
 ./script/clean
 
-TAG="docker-compose"
-docker build -t "$TAG" .
-docker run \
-    --rm --entrypoint="script/build/linux-entrypoint" \
-    -v $(pwd)/dist:/code/dist \
-    -v $(pwd)/.git:/code/.git \
-    "$TAG"
+TMP_CONTAINER="tmpcontainer"
+TAG="docker/compose:tmp-glibc-linux-binary"
+DOCKER_COMPOSE_GITSHA=$(script/build/write-git-sha)
+
+docker build -t "${TAG}" . \
+       --build-arg BUILD_PLATFORM=debian \
+       --build-arg GIT_COMMIT=${DOCKER_COMPOSE_GITSHA}
+docker create --name ${TMP_CONTAINER} ${TAG}
+mkdir -p dist
+docker cp ${TMP_CONTAINER}:/usr/local/bin/docker-compose dist/docker-compose-Linux-x86_64
+docker container rm -f ${TMP_CONTAINER}
+docker image rm -f ${TAG}

script/build/linux-entrypoint

@@ -2,14 +2,35 @@
 
 set -ex
 
-TARGET=dist/docker-compose-$(uname -s)-$(uname -m)
-VENV=/code/.tox/py37
+CODE_PATH=/code
+VENV=${CODE_PATH}/.tox/py37
 
-mkdir -p `pwd`/dist
-chmod 777 `pwd`/dist
+cd ${CODE_PATH}
+mkdir -p dist
+chmod 777 dist
 
-$VENV/bin/pip install -q -r requirements-build.txt
+${VENV}/bin/pip3 install -q -r requirements-build.txt
+
+# TODO(ulyssessouza) To check if really needed
 ./script/build/write-git-sha
-su -c "$VENV/bin/pyinstaller docker-compose.spec" user
-mv dist/docker-compose $TARGET
-$TARGET version
+
+export PATH="${CODE_PATH}/pyinstaller:${PATH}"
+
+if [ ! -z "${BUILD_BOOTLOADER}" ]; then
+    # Build bootloader for alpine
+    git clone --single-branch --branch master https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller
+    cd /tmp/pyinstaller/bootloader
+    git checkout v3.4
+    ${VENV}/bin/python3 ./waf configure --no-lsb all
+    ${VENV}/bin/pip3 install ..
+    cd ${CODE_PATH}
+    rm -Rf /tmp/pyinstaller
+else
+    echo "NOT compiling bootloader!!!"
+fi
+
+${VENV}/bin/pyinstaller --exclude-module pycrypto --exclude-module PyInstaller docker-compose.spec
+ls -la dist/
+ldd dist/docker-compose
+mv dist/docker-compose /usr/local/bin
+docker-compose version

script/build/test-image

@@ -7,11 +7,12 @@ if [ -z "$1" ]; then
     exit 1
 fi
 
-TAG=$1
+TAG="$1"
+IMAGE="docker/compose-tests"
 
-docker build -t docker-compose-tests:tmp .
-ctnr_id=$(docker create --entrypoint=tox docker-compose-tests:tmp)
-docker commit $ctnr_id docker/compose-tests:latest
-docker tag docker/compose-tests:latest docker/compose-tests:$TAG
-docker rm -f $ctnr_id
-docker rmi -f docker-compose-tests:tmp
+DOCKER_COMPOSE_GITSHA=$(script/build/write-git-sha)
+docker build -t "${IMAGE}:${TAG}" . \
+       --target build \
+       --build-arg BUILD_PLATFORM=debian \
+       --build-arg GIT_COMMIT=${DOCKER_COMPOSE_GITSHA}
+docker tag ${IMAGE}:${TAG} ${IMAGE}:latest

script/test/ci

@@ -20,6 +20,3 @@ export DOCKER_DAEMON_ARGS="--storage-driver=$STORAGE_DRIVER"
 
 GIT_VOLUME="--volumes-from=$(hostname)"
 . script/test/all
-
->&2 echo "Building Linux binary"
-. script/build/linux-entrypoint

script/test/default

@@ -3,17 +3,18 @@
 
 set -ex
 
-TAG="docker-compose:$(git rev-parse --short HEAD)"
+TAG="docker-compose:alpine-$(git rev-parse --short HEAD)"
 
-# By default use the Dockerfile, but can be overridden to use an alternative file
+# By default use the Dockerfile.alpine, but can be overridden to use an alternative file
 # e.g DOCKERFILE=Dockerfile.armhf script/test/default
-DOCKERFILE="${DOCKERFILE:-Dockerfile}"
+DOCKERFILE="${DOCKERFILE:-Dockerfile.alpine}"
+DOCKER_BUILD_TARGET="${DOCKER_BUILD_TARGET:-build}"
 
 rm -rf coverage-html
 # Create the host directory so it's owned by $USER
 mkdir -p coverage-html
 
-docker build -f ${DOCKERFILE} -t "$TAG" .
+docker build -f ${DOCKERFILE} -t "${TAG}" --target "${DOCKER_BUILD_TARGET}" .
 
 GIT_VOLUME="--volume=$(pwd)/.git:/code/.git"
 . script/test/all