docker-compose/script/release/release.py

from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

import argparse
import os
import shutil
import sys
import time
from distutils.core import run_setup

import pypandoc
from jinja2 import Template
from release.bintray import BintrayAPI
from release.const import BINTRAY_ORG
from release.const import NAME
from release.const import REPO_ROOT
from release.downloader import BinaryDownloader
from release.images import ImageManager
from release.repository import delete_assets
from release.repository import get_contributors
from release.repository import Repository
from release.repository import upload_assets
from release.utils import branch_name
from release.utils import compatibility_matrix
from release.utils import read_release_notes_from_changelog
from release.utils import ScriptError
from release.utils import update_init_py_version
from release.utils import update_run_sh_version
from release.utils import yesno
from requests.exceptions import HTTPError
from twine.commands.upload import main as twine_upload


def create_initial_branch(repository, args):
    release_branch = repository.create_release_branch(args.release, args.base)
    if args.base and args.cherries:
        print('Detected patch version.')
        cherries = input('Indicate (space-separated) PR numbers to cherry-pick then press Enter:\n')
        repository.cherry_pick_prs(release_branch, cherries.split())

    return create_bump_commit(repository, release_branch, args.bintray_user, args.bintray_org)


def create_bump_commit(repository, release_branch, bintray_user, bintray_org):
    with release_branch.config_reader() as cfg:
        release = cfg.get('release')
    print('Updating version info in __init__.py and run.sh')
    update_run_sh_version(release)
    update_init_py_version(release)

    input('Please add the release notes to the CHANGELOG.md file, then press Enter to continue.')
    proceed = None
    while not proceed:
        print(repository.diff())
        proceed = yesno('Are these changes ok? y/N ', default=False)

    if repository.diff():
        repository.create_bump_commit(release_branch, release)
    repository.push_branch_to_remote(release_branch)

    bintray_api = BintrayAPI(os.environ['BINTRAY_TOKEN'], bintray_user)
    if not bintray_api.repository_exists(bintray_org, release_branch.name):
        print('Creating data repository {} on bintray'.format(release_branch.name))
        bintray_api.create_repository(bintray_org, release_branch.name, 'generic')
    else:
        print('Bintray repository {} already exists. Skipping'.format(release_branch.name))


def monitor_pr_status(pr_data):
    print('Waiting for CI to complete...')
    last_commit = pr_data.get_commits().reversed[0]
    while True:
        status = last_commit.get_combined_status()
        if status.state == 'pending' or status.state == 'failure':
            summary = {
                'pending': 0,
                'success': 0,
                'failure': 0,
                'error': 0,
            }
            for detail in status.statuses:
                if detail.context == 'dco-signed':
                    # dco-signed check breaks on merge remote-tracking ; ignore it
                    continue
                if detail.state in summary:
                    summary[detail.state] += 1
            print(
                '{pending} pending, {success} successes, {failure} failures, '
                '{error} errors'.format(**summary)
            )
            if summary['failure'] > 0 or summary['error'] > 0:
                raise ScriptError('CI failures detected!')
            elif summary['pending'] == 0 and summary['success'] > 0:
                # This check assumes at least 1 non-DCO CI check to avoid race conditions.
                # If testing on a repo without CI, use --skip-ci-check to avoid looping eternally
                return True
            time.sleep(30)
        elif status.state == 'success':
            print('{} successes: all clear!'.format(status.total_count))
            return True


def check_pr_mergeable(pr_data):
    if pr_data.mergeable is False:
        # mergeable can also be null, in which case the warning would be a false positive.
        print(
            'WARNING!! PR #{} can not currently be merged. You will need to '
            'resolve the conflicts manually before finalizing the release.'.format(pr_data.number)
        )

    return pr_data.mergeable is True


def create_release_draft(repository, version, pr_data, files):
    print('Creating Github release draft')
    with open(os.path.join(os.path.dirname(__file__), 'release.md.tmpl'), 'r') as f:
        template = Template(f.read())
    print('Rendering release notes based on template')
    release_notes = template.render(
        version=version,
        compat_matrix=compatibility_matrix(),
        integrity=files,
        contributors=get_contributors(pr_data),
        changelog=read_release_notes_from_changelog(),
    )
    gh_release = repository.create_release(
        version, release_notes, draft=True, prerelease='-rc' in version,
        target_commitish='release'
    )
    print('Release draft initialized')
    return gh_release


def print_final_instructions(args):
    print(
        "You're almost done! Please verify that everything is in order and "
        "you are ready to make the release public, then run the following "
        "command:\n{exe} -b {user} finalize {version}".format(
            exe='./script/release/release.sh', user=args.bintray_user, version=args.release
        )
    )


def distclean():
    print('Running distclean...')
    dirs = [
        os.path.join(REPO_ROOT, 'build'), os.path.join(REPO_ROOT, 'dist'),
        os.path.join(REPO_ROOT, 'docker-compose.egg-info')
    ]
    files = []
    for base, dirnames, fnames in os.walk(REPO_ROOT):
        for fname in fnames:
            path = os.path.normpath(os.path.join(base, fname))
            if fname.endswith('.pyc'):
                files.append(path)
            elif fname.startswith('.coverage.'):
                files.append(path)
        for dirname in dirnames:
            path = os.path.normpath(os.path.join(base, dirname))
            if dirname == '__pycache__':
                dirs.append(path)
            elif dirname == '.coverage-binfiles':
                dirs.append(path)

    for file in files:
        os.unlink(file)

    for folder in dirs:
        shutil.rmtree(folder, ignore_errors=True)


def pypi_upload(args):
    print('Uploading to PyPi')
    try:
        rel = args.release.replace('-rc', 'rc')
        twine_upload([
            'dist/docker_compose-{}*.whl'.format(rel),
            'dist/docker-compose-{}*.tar.gz'.format(rel)
        ])
    except HTTPError as e:
        if e.response.status_code == 400 and 'File already exists' in e.message:
            if not args.finalize_resume:
                raise ScriptError(
                    'Package already uploaded on PyPi.'
                )
            print('Skipping PyPi upload - package already uploaded')
        else:
            raise ScriptError('Unexpected HTTP error uploading package to PyPi: {}'.format(e))


def resume(args):
    try:
        distclean()
        repository = Repository(REPO_ROOT, args.repo)
        br_name = branch_name(args.release)
        if not repository.branch_exists(br_name):
            raise ScriptError('No local branch exists for this release.')
        gh_release = repository.find_release(args.release)
        if gh_release and not gh_release.draft:
            print('WARNING!! Found non-draft (public) release for this version!')
            proceed = yesno(
                'Are you sure you wish to proceed? Modifying an already '
                'released version is dangerous! y/N ', default=False
            )
            if proceed.lower() is not True:
                raise ScriptError('Aborting release')

        release_branch = repository.checkout_branch(br_name)
        if args.cherries:
            cherries = input('Indicate (space-separated) PR numbers to cherry-pick then press Enter:\n')
            repository.cherry_pick_prs(release_branch, cherries.split())

        create_bump_commit(repository, release_branch, args.bintray_user, args.bintray_org)
        pr_data = repository.find_release_pr(args.release)
        if not pr_data:
            pr_data = repository.create_release_pull_request(args.release)
        check_pr_mergeable(pr_data)
        if not args.skip_ci:
            monitor_pr_status(pr_data)
        downloader = BinaryDownloader(args.destination)
        files = downloader.download_all(args.release)
        if not gh_release:
            gh_release = create_release_draft(repository, args.release, pr_data, files)
        delete_assets(gh_release)
        upload_assets(gh_release, files)
        img_manager = ImageManager(args.release)
        img_manager.build_images(repository, files)
    except ScriptError as e:
        print(e)
        return 1

    print_final_instructions(args)
    return 0


def cancel(args):
    try:
        repository = Repository(REPO_ROOT, args.repo)
        repository.close_release_pr(args.release)
        repository.remove_release(args.release)
        repository.remove_bump_branch(args.release)
        bintray_api = BintrayAPI(os.environ['BINTRAY_TOKEN'], args.bintray_user)
        print('Removing Bintray data repository for {}'.format(args.release))
        bintray_api.delete_repository(args.bintray_org, branch_name(args.release))
        distclean()
    except ScriptError as e:
        print(e)
        return 1
    print('Release cancellation complete.')
    return 0


def start(args):
    distclean()
    try:
        repository = Repository(REPO_ROOT, args.repo)
        create_initial_branch(repository, args)
        pr_data = repository.create_release_pull_request(args.release)
        check_pr_mergeable(pr_data)
        if not args.skip_ci:
            monitor_pr_status(pr_data)
        downloader = BinaryDownloader(args.destination)
        files = downloader.download_all(args.release)
        gh_release = create_release_draft(repository, args.release, pr_data, files)
        upload_assets(gh_release, files)
        img_manager = ImageManager(args.release)
        img_manager.build_images(repository, files)
    except ScriptError as e:
        print(e)
        return 1

    print_final_instructions(args)
    return 0


def finalize(args):
    distclean()
    try:
        repository = Repository(REPO_ROOT, args.repo)
        img_manager = ImageManager(args.release)
        pr_data = repository.find_release_pr(args.release)
        if not pr_data:
            raise ScriptError('No PR found for {}'.format(args.release))
        if not check_pr_mergeable(pr_data):
            raise ScriptError('Can not finalize release with an unmergeable PR')
        if not img_manager.check_images(args.release):
            raise ScriptError('Missing release image')
        br_name = branch_name(args.release)
        if not repository.branch_exists(br_name):
            raise ScriptError('No local branch exists for this release.')
        gh_release = repository.find_release(args.release)
        if not gh_release:
            raise ScriptError('No Github release draft for this version')

        repository.checkout_branch(br_name)

        pypandoc.convert_file(
            os.path.join(REPO_ROOT, 'README.md'), 'rst', outputfile=os.path.join(REPO_ROOT, 'README.rst')
        )
        run_setup(os.path.join(REPO_ROOT, 'setup.py'), script_args=['sdist', 'bdist_wheel'])

        merge_status = pr_data.merge()
        if not merge_status.merged and not args.finalize_resume:
            raise ScriptError(
                'Unable to merge PR #{}: {}'.format(pr_data.number, merge_status.message)
            )

        pypi_upload(args)

        img_manager.push_images()
        repository.publish_release(gh_release)
    except ScriptError as e:
        print(e)
        return 1

    return 0


ACTIONS = [
    'start',
    'cancel',
    'resume',
    'finalize',
]

EPILOG = '''Example uses:
    * Start a new feature release (includes all changes currently in master)
        release.sh -b user start 1.23.0
    * Start a new patch release
        release.sh -b user --patch 1.21.0 start 1.21.1
    * Cancel / rollback an existing release draft
        release.sh -b user cancel 1.23.0
    * Restart a previously aborted patch release
        release.sh -b user -p 1.21.0 resume 1.21.1
'''


def main():
    if 'GITHUB_TOKEN' not in os.environ:
        print('GITHUB_TOKEN environment variable must be set')
        return 1

    if 'BINTRAY_TOKEN' not in os.environ:
        print('BINTRAY_TOKEN environment variable must be set')
        return 1

    parser = argparse.ArgumentParser(
        description='Orchestrate a new release of docker/compose. This tool assumes that you have '
                    'obtained a Github API token and Bintray API key and set the GITHUB_TOKEN and '
                    'BINTRAY_TOKEN environment variables accordingly.',
        epilog=EPILOG, formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument(
        'action', choices=ACTIONS, help='The action to be performed for this release'
    )
    parser.add_argument('release', help='Release number, e.g. 1.9.0-rc1, 2.1.1')
    parser.add_argument(
        '--patch', '-p', dest='base',
        help='Which version is being patched by this release'
    )
    parser.add_argument(
        '--repo', '-r', dest='repo', default=NAME,
        help='Start a release for the given repo (default: {})'.format(NAME)
    )
    parser.add_argument(
        '-b', dest='bintray_user', required=True, metavar='USER',
        help='Username associated with the Bintray API key'
    )
    parser.add_argument(
        '--bintray-org', dest='bintray_org', metavar='ORG', default=BINTRAY_ORG,
        help='Organization name on bintray where the data repository will be created.'
    )
    parser.add_argument(
        '--destination', '-o', metavar='DIR', default='binaries',
        help='Directory where release binaries will be downloaded relative to the project root'
    )
    parser.add_argument(
        '--no-cherries', '-C', dest='cherries', action='store_false',
        help='If set, the program will not prompt the user for PR numbers to cherry-pick'
    )
    parser.add_argument(
        '--skip-ci-checks', dest='skip_ci', action='store_true',
        help='If set, the program will not wait for CI jobs to complete'
    )
    parser.add_argument(
        '--finalize-resume', dest='finalize_resume', action='store_true',
        help='If set, finalize will continue through steps that have already been completed.'
    )
    args = parser.parse_args()

    if args.action == 'start':
        return start(args)
    elif args.action == 'resume':
        return resume(args)
    elif args.action == 'cancel':
        return cancel(args)
    elif args.action == 'finalize':
        return finalize(args)

    print('Unexpected action "{}"'.format(args.action), file=sys.stderr)
    return 1


if __name__ == '__main__':
    sys.exit(main())