import logging import os import random from shutil import rmtree import string import docker import git from summary import Summary DEFAULT_REPOSITORY = 'git://github.com/shin-/brew' DEFAULT_BRANCH = 'master' client = docker.Client(timeout=10000) processed = {} processed_folders = [] def build_library(repository=None, branch=None, namespace=None, push=False, debug=False, prefill=True, registry=None, targetlist=None, repos_folder=None, logger=None): ''' Entrypoint method build_library. repository: Repository containing a library/ folder. Can be a local path or git repository branch: If repository is a git repository, checkout this branch (default: DEFAULT_BRANCH) namespace: Created repositories will use the following namespace. (default: no namespace) push: If set to true, push images to the repository debug: Enables debug logging if set to True prefill: Retrieve images from public repository before building. Serves to prefill the builder cache. registry: URL to the private registry where results should be pushed. (only if push=True) targetlist: String indicating which library files are targeted by this build. Entries should be comma-separated. Default is all files. repos_folder: Fixed location where cloned repositories should be stored. Default is None, meaning folders are temporary and cleaned up after the build finishes. logger: Logger instance to use. Default is None, in which case build_library will create its own logger. ''' dst_folder = None summary = Summary() if logger is None: logger = logging.getLogger(__name__) logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level='INFO') if repository is None: repository = DEFAULT_REPOSITORY if branch is None: branch = DEFAULT_BRANCH if debug: logger.setLevel('DEBUG') if targetlist is not None: targetlist = targetlist.split(',') if not repository.startswith(('https://', 'git://')): logger.info('Repository provided assumed to be a local path') dst_folder = repository try: client.version() except Exception as e: logger.error('Could not reach the docker daemon. Please make sure it ' 'is running.') logger.warning('Also make sure you have access to the docker UNIX ' 'socket (use sudo)') return if not dst_folder: logger.info('Cloning docker repo from {0}, branch: {1}'.format( repository, branch)) try: rep, dst_folder = git.clone_branch(repository, branch) except Exception as e: logger.exception(e) logger.error('Source repository could not be fetched. Check ' 'that the address is correct and the branch exists.') return try: dirlist = os.listdir(os.path.join(dst_folder, 'library')) except OSError as e: logger.error('The path provided ({0}) could not be found or didn\'t' 'contain a library/ folder.'.format(dst_folder)) return for buildfile in dirlist: if buildfile == 'MAINTAINERS': continue if (targetlist and buildfile not in targetlist): continue f = open(os.path.join(dst_folder, 'library', buildfile)) linecnt = 0 for line in f: linecnt += 1 if not line or line.strip() == '': continue elif line.lstrip().startswith('#'): # # It's a comment! continue logger.debug('{0} ---> {1}'.format(buildfile, line)) try: tag, url, ref, dfile = parse_line(line, logger) if prefill: logger.debug('Pulling {0} from official repository (cache ' 'fill)'.format(buildfile)) try: client.pull(buildfile) except: # Image is not on official repository, ignore prefill pass img, commit = build_repo(url, ref, buildfile, dfile, tag, namespace, push, registry, repos_folder, logger) summary.add_success(buildfile, (linecnt, line), img, commit) processed['{0}@{1}'.format(url, ref)] = img except Exception as e: logger.exception(e) summary.add_exception(buildfile, (linecnt, line), e) f.close() cleanup(dst_folder, dst_folder != repository, repos_folder is None) summary.print_summary(logger) return summary def parse_line(line, logger): df_folder = '.' args = line.split(':', 1) if len(args) != 2: logger.debug("Invalid line: {0}".format(line)) raise RuntimeError('Incorrect line format, please refer to the docs') try: repo = args[1].strip().split() if len(repo) == 2: df_folder = repo[1].strip() url, ref = repo[0].strip().rsplit('@', 1) return (args[0].strip(), url, ref, df_folder) except ValueError: logger.debug("Invalid line: {0}".format(line)) raise RuntimeError('Incorrect line format, please refer to the docs') def cleanup(libfolder, clean_libfolder=False, clean_repos=True): ''' Cleanup method called at the end of build_library. libfolder: Folder containing the library definition. clean_libfolder: If set to True, libfolder will be removed. Only if libfolder was temporary clean_repos: Remove library repos. Also resets module variables "processed" and "processed_folders" if set to true. ''' global processed_folders global processed if clean_libfolder: rmtree(libfolder, True) if clean_repos: for d in processed_folders: rmtree(d, True) processed_folders = [] processed = {} def _random_suffix(): return ''.join([ random.choice(string.ascii_letters + string.digits) for i in xrange(6) ]) def build_repo(repository, ref, docker_repo, dockerfile_location, docker_tag, namespace, push, registry, repos_folder, logger): ''' Builds one line of a library file. repository: URL of the git repository that needs to be built ref: Git reference (or commit ID) that needs to be built docker_repo: Name of the docker repository where the image will end up. dockerfile_location: Folder containing the Dockerfile docker_tag: Tag for the image in the docker repository. namespace: Namespace for the docker repository. push: If the image should be pushed at the end of the build registry: URL to private registry where image should be pushed repos_folder: Directory where repositories should be cloned logger: Logger instance ''' dst_folder = None img_id = None commit_id = None if repos_folder: # Repositories are stored in a fixed location and can be reused dst_folder = os.path.join(repos_folder, docker_repo + _random_suffix()) docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo) if '{0}@{1}'.format(repository, ref) in processed.keys() or\ '{0}@{1}'.format(repository, 'refs/tags' + ref) in processed.keys(): if '{0}@{1}'.format(repository, ref) not in processed.keys(): ref = 'refs/tags/' + ref logger.info('This ref has already been built, reusing image ID') img_id = processed['{0}@{1}'.format(repository, ref)] if ref.startswith('refs/'): commit_id = processed[repository].ref(ref) else: commit_id = ref else: # Not already built rep = None logger.info('Cloning {0} (ref: {1})'.format(repository, ref)) if repository not in processed: # Repository not cloned yet try: rep, dst_folder = git.clone(repository, ref, dst_folder) except Exception: if dst_folder: rmtree(dst_folder) ref = 'refs/tags/' + ref rep, dst_folder = git.clone(repository, ref, dst_folder) processed[repository] = rep processed_folders.append(dst_folder) else: rep = processed[repository] if ref in rep.refs: # The ref already exists, we just need to checkout dst_folder = git.checkout(rep, ref) elif 'refs/tags/' + ref in rep.refs: ref = 'refs/tags/' + ref dst_folder = git.checkout(rep, ref) else: # ref is not present, try pulling it from the remote origin try: rep, dst_folder = git.pull(repository, rep, ref) except Exception: ref = 'refs/tags/' + ref rep, dst_folder = git.pull(repository, rep, ref) dockerfile_location = os.path.join(dst_folder, dockerfile_location) if not 'Dockerfile' in os.listdir(dockerfile_location): raise RuntimeError('Dockerfile not found in cloned repository') commit_id = rep.head() logger.info('Building using dockerfile...') img_id, logs = client.build(path=dockerfile_location, quiet=True) if img_id is None: logger.error('Image ID not found. Printing build logs...') logger.debug(logs) raise RuntimeError('Build failed') logger.info('Committing to {0}:{1}'.format(docker_repo, docker_tag or 'latest')) client.tag(img_id, docker_repo, docker_tag) if push: logger.info('Pushing result to registry {0}'.format( registry or "default")) push_repo(img_id, docker_repo, registry=registry, logger=logger) return img_id, commit_id def push_repo(img_id, repo, registry=None, docker_tag=None, logger=None): ''' Pushes a repository to a registry img_id: Image ID to push repo: Repository name where img_id should be tagged registry: Private registry where image needs to be pushed docker_tag: Tag to be applied to the image in docker repo logger: Logger instance ''' exc = None if registry is not None: repo = '{0}/{1}'.format(registry, repo) logger.info('Also tagging {0}'.format(repo)) client.tag(img_id, repo, docker_tag) for i in xrange(4): try: pushlog = client.push(repo) if '"error":"' in pushlog: raise RuntimeError('Error while pushing: {0}'.format(pushlog)) except Exception as e: exc = e continue return raise exc