project.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947
  1. from __future__ import absolute_import
  2. from __future__ import unicode_literals
  3. import datetime
  4. import logging
  5. import operator
  6. import re
  7. from functools import reduce
  8. from os import path
  9. import enum
  10. import six
  11. from docker.errors import APIError
  12. from docker.errors import ImageNotFound
  13. from docker.errors import NotFound
  14. from docker.utils import version_lt
  15. from . import parallel
  16. from .cli.errors import UserError
  17. from .config import ConfigurationError
  18. from .config.config import V1
  19. from .config.sort_services import get_container_name_from_network_mode
  20. from .config.sort_services import get_service_name_from_network_mode
  21. from .const import LABEL_ONE_OFF
  22. from .const import LABEL_PROJECT
  23. from .const import LABEL_SERVICE
  24. from .container import Container
  25. from .network import build_networks
  26. from .network import get_networks
  27. from .network import ProjectNetworks
  28. from .progress_stream import read_status
  29. from .service import BuildAction
  30. from .service import ContainerNetworkMode
  31. from .service import ContainerPidMode
  32. from .service import ConvergenceStrategy
  33. from .service import NetworkMode
  34. from .service import NoSuchImageError
  35. from .service import parse_repository_tag
  36. from .service import PidMode
  37. from .service import Service
  38. from .service import ServiceNetworkMode
  39. from .service import ServicePidMode
  40. from .utils import microseconds_from_time_nano
  41. from .utils import truncate_string
  42. from .volume import ProjectVolumes
  43. log = logging.getLogger(__name__)
  44. @enum.unique
  45. class OneOffFilter(enum.Enum):
  46. include = 0
  47. exclude = 1
  48. only = 2
  49. @classmethod
  50. def update_labels(cls, value, labels):
  51. if value == cls.only:
  52. labels.append('{0}={1}'.format(LABEL_ONE_OFF, "True"))
  53. elif value == cls.exclude:
  54. labels.append('{0}={1}'.format(LABEL_ONE_OFF, "False"))
  55. elif value == cls.include:
  56. pass
  57. else:
  58. raise ValueError("Invalid value for one_off: {}".format(repr(value)))
  59. class Project(object):
  60. """
  61. A collection of services.
  62. """
  63. def __init__(self, name, services, client, networks=None, volumes=None, config_version=None):
  64. self.name = name
  65. self.services = services
  66. self.client = client
  67. self.volumes = volumes or ProjectVolumes({})
  68. self.networks = networks or ProjectNetworks({}, False)
  69. self.config_version = config_version
  70. def labels(self, one_off=OneOffFilter.exclude, legacy=False):
  71. name = self.name
  72. if legacy:
  73. name = re.sub(r'[_-]', '', name)
  74. labels = ['{0}={1}'.format(LABEL_PROJECT, name)]
  75. OneOffFilter.update_labels(one_off, labels)
  76. return labels
  77. @classmethod
  78. def from_config(cls, name, config_data, client, default_platform=None, extra_labels=[]):
  79. """
  80. Construct a Project from a config.Config object.
  81. """
  82. use_networking = (config_data.version and config_data.version != V1)
  83. networks = build_networks(name, config_data, client)
  84. project_networks = ProjectNetworks.from_services(
  85. config_data.services,
  86. networks,
  87. use_networking)
  88. volumes = ProjectVolumes.from_config(name, config_data, client)
  89. project = cls(name, [], client, project_networks, volumes, config_data.version)
  90. for service_dict in config_data.services:
  91. service_dict = dict(service_dict)
  92. if use_networking:
  93. service_networks = get_networks(service_dict, networks)
  94. else:
  95. service_networks = {}
  96. service_dict.pop('networks', None)
  97. links = project.get_links(service_dict)
  98. network_mode = project.get_network_mode(
  99. service_dict, list(service_networks.keys())
  100. )
  101. pid_mode = project.get_pid_mode(service_dict)
  102. volumes_from = get_volumes_from(project, service_dict)
  103. if config_data.version != V1:
  104. service_dict['volumes'] = [
  105. volumes.namespace_spec(volume_spec)
  106. for volume_spec in service_dict.get('volumes', [])
  107. ]
  108. secrets = get_secrets(
  109. service_dict['name'],
  110. service_dict.pop('secrets', None) or [],
  111. config_data.secrets)
  112. project.services.append(
  113. Service(
  114. service_dict.pop('name'),
  115. client=client,
  116. project=name,
  117. use_networking=use_networking,
  118. networks=service_networks,
  119. links=links,
  120. network_mode=network_mode,
  121. volumes_from=volumes_from,
  122. secrets=secrets,
  123. pid_mode=pid_mode,
  124. platform=service_dict.pop('platform', None),
  125. default_platform=default_platform,
  126. extra_labels=extra_labels,
  127. **service_dict)
  128. )
  129. return project
  130. @property
  131. def service_names(self):
  132. return [service.name for service in self.services]
  133. def get_service(self, name):
  134. """
  135. Retrieve a service by name. Raises NoSuchService
  136. if the named service does not exist.
  137. """
  138. for service in self.services:
  139. if service.name == name:
  140. return service
  141. raise NoSuchService(name)
  142. def validate_service_names(self, service_names):
  143. """
  144. Validate that the given list of service names only contains valid
  145. services. Raises NoSuchService if one of the names is invalid.
  146. """
  147. valid_names = self.service_names
  148. for name in service_names:
  149. if name not in valid_names:
  150. raise NoSuchService(name)
  151. def get_services(self, service_names=None, include_deps=False):
  152. """
  153. Returns a list of this project's services filtered
  154. by the provided list of names, or all services if service_names is None
  155. or [].
  156. If include_deps is specified, returns a list including the dependencies for
  157. service_names, in order of dependency.
  158. Preserves the original order of self.services where possible,
  159. reordering as needed to resolve dependencies.
  160. Raises NoSuchService if any of the named services do not exist.
  161. """
  162. if service_names is None or len(service_names) == 0:
  163. service_names = self.service_names
  164. unsorted = [self.get_service(name) for name in service_names]
  165. services = [s for s in self.services if s in unsorted]
  166. if include_deps:
  167. services = reduce(self._inject_deps, services, [])
  168. uniques = []
  169. [uniques.append(s) for s in services if s not in uniques]
  170. return uniques
  171. def get_services_without_duplicate(self, service_names=None, include_deps=False):
  172. services = self.get_services(service_names, include_deps)
  173. for service in services:
  174. service.remove_duplicate_containers()
  175. return services
  176. def get_links(self, service_dict):
  177. links = []
  178. if 'links' in service_dict:
  179. for link in service_dict.get('links', []):
  180. if ':' in link:
  181. service_name, link_name = link.split(':', 1)
  182. else:
  183. service_name, link_name = link, None
  184. try:
  185. links.append((self.get_service(service_name), link_name))
  186. except NoSuchService:
  187. raise ConfigurationError(
  188. 'Service "%s" has a link to service "%s" which does not '
  189. 'exist.' % (service_dict['name'], service_name))
  190. del service_dict['links']
  191. return links
  192. def get_network_mode(self, service_dict, networks):
  193. network_mode = service_dict.pop('network_mode', None)
  194. if not network_mode:
  195. if self.networks.use_networking:
  196. return NetworkMode(networks[0]) if networks else NetworkMode('none')
  197. return NetworkMode(None)
  198. service_name = get_service_name_from_network_mode(network_mode)
  199. if service_name:
  200. return ServiceNetworkMode(self.get_service(service_name))
  201. container_name = get_container_name_from_network_mode(network_mode)
  202. if container_name:
  203. try:
  204. return ContainerNetworkMode(Container.from_id(self.client, container_name))
  205. except APIError:
  206. raise ConfigurationError(
  207. "Service '{name}' uses the network stack of container '{dep}' which "
  208. "does not exist.".format(name=service_dict['name'], dep=container_name))
  209. return NetworkMode(network_mode)
  210. def get_pid_mode(self, service_dict):
  211. pid_mode = service_dict.pop('pid', None)
  212. if not pid_mode:
  213. return PidMode(None)
  214. service_name = get_service_name_from_network_mode(pid_mode)
  215. if service_name:
  216. return ServicePidMode(self.get_service(service_name))
  217. container_name = get_container_name_from_network_mode(pid_mode)
  218. if container_name:
  219. try:
  220. return ContainerPidMode(Container.from_id(self.client, container_name))
  221. except APIError:
  222. raise ConfigurationError(
  223. "Service '{name}' uses the PID namespace of container '{dep}' which "
  224. "does not exist.".format(name=service_dict['name'], dep=container_name)
  225. )
  226. return PidMode(pid_mode)
  227. def start(self, service_names=None, **options):
  228. containers = []
  229. def start_service(service):
  230. service_containers = service.start(quiet=True, **options)
  231. containers.extend(service_containers)
  232. services = self.get_services(service_names)
  233. def get_deps(service):
  234. return {
  235. (self.get_service(dep), config)
  236. for dep, config in service.get_dependency_configs().items()
  237. }
  238. parallel.parallel_execute(
  239. services,
  240. start_service,
  241. operator.attrgetter('name'),
  242. 'Starting',
  243. get_deps,
  244. fail_check=lambda obj: not obj.containers(),
  245. )
  246. return containers
  247. def stop(self, service_names=None, one_off=OneOffFilter.exclude, **options):
  248. containers = self.containers(service_names, one_off=one_off)
  249. def get_deps(container):
  250. # actually returning inversed dependencies
  251. return {(other, None) for other in containers
  252. if container.service in
  253. self.get_service(other.service).get_dependency_names()}
  254. parallel.parallel_execute(
  255. containers,
  256. self.build_container_operation_with_timeout_func('stop', options),
  257. operator.attrgetter('name'),
  258. 'Stopping',
  259. get_deps,
  260. )
  261. def pause(self, service_names=None, **options):
  262. containers = self.containers(service_names)
  263. parallel.parallel_pause(reversed(containers), options)
  264. return containers
  265. def unpause(self, service_names=None, **options):
  266. containers = self.containers(service_names)
  267. parallel.parallel_unpause(containers, options)
  268. return containers
  269. def kill(self, service_names=None, **options):
  270. parallel.parallel_kill(self.containers(service_names), options)
  271. def remove_stopped(self, service_names=None, one_off=OneOffFilter.exclude, **options):
  272. parallel.parallel_remove(self.containers(
  273. service_names, stopped=True, one_off=one_off
  274. ), options)
  275. def down(
  276. self,
  277. remove_image_type,
  278. include_volumes,
  279. remove_orphans=False,
  280. timeout=None,
  281. ignore_orphans=False):
  282. self.stop(one_off=OneOffFilter.include, timeout=timeout)
  283. if not ignore_orphans:
  284. self.find_orphan_containers(remove_orphans)
  285. self.remove_stopped(v=include_volumes, one_off=OneOffFilter.include)
  286. self.networks.remove()
  287. if include_volumes:
  288. self.volumes.remove()
  289. self.remove_images(remove_image_type)
  290. def remove_images(self, remove_image_type):
  291. for service in self.get_services():
  292. service.remove_image(remove_image_type)
  293. def restart(self, service_names=None, **options):
  294. containers = self.containers(service_names, stopped=True)
  295. parallel.parallel_execute(
  296. containers,
  297. self.build_container_operation_with_timeout_func('restart', options),
  298. operator.attrgetter('name'),
  299. 'Restarting',
  300. )
  301. return containers
  302. def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None,
  303. build_args=None, gzip=False, parallel_build=False, rm=True, silent=False, cli=False,
  304. progress=None):
  305. services = []
  306. for service in self.get_services(service_names):
  307. if service.can_be_built():
  308. services.append(service)
  309. elif not silent:
  310. log.info('%s uses an image, skipping' % service.name)
  311. if cli:
  312. log.warning("Native build is an experimental feature and could change at any time")
  313. if parallel_build:
  314. log.warning("Flag '--parallel' is ignored when building with "
  315. "COMPOSE_DOCKER_CLI_BUILD=1")
  316. if gzip:
  317. log.warning("Flag '--compress' is ignored when building with "
  318. "COMPOSE_DOCKER_CLI_BUILD=1")
  319. def build_service(service):
  320. service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli, progress)
  321. if parallel_build:
  322. _, errors = parallel.parallel_execute(
  323. services,
  324. build_service,
  325. operator.attrgetter('name'),
  326. 'Building',
  327. limit=5,
  328. )
  329. if len(errors):
  330. combined_errors = '\n'.join([
  331. e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values()
  332. ])
  333. raise ProjectError(combined_errors)
  334. else:
  335. for service in services:
  336. build_service(service)
  337. def create(
  338. self,
  339. service_names=None,
  340. strategy=ConvergenceStrategy.changed,
  341. do_build=BuildAction.none,
  342. ):
  343. services = self.get_services_without_duplicate(service_names, include_deps=True)
  344. for svc in services:
  345. svc.ensure_image_exists(do_build=do_build)
  346. plans = self._get_convergence_plans(services, strategy)
  347. for service in services:
  348. service.execute_convergence_plan(
  349. plans[service.name],
  350. detached=True,
  351. start=False)
  352. def _legacy_event_processor(self, service_names):
  353. # Only for v1 files or when Compose is forced to use an older API version
  354. def build_container_event(event, container):
  355. time = datetime.datetime.fromtimestamp(event['time'])
  356. time = time.replace(
  357. microsecond=microseconds_from_time_nano(event['timeNano'])
  358. )
  359. return {
  360. 'time': time,
  361. 'type': 'container',
  362. 'action': event['status'],
  363. 'id': container.id,
  364. 'service': container.service,
  365. 'attributes': {
  366. 'name': container.name,
  367. 'image': event['from'],
  368. },
  369. 'container': container,
  370. }
  371. service_names = set(service_names or self.service_names)
  372. for event in self.client.events(
  373. filters={'label': self.labels()},
  374. decode=True
  375. ):
  376. # This is a guard against some events broadcasted by swarm that
  377. # don't have a status field.
  378. # See https://github.com/docker/compose/issues/3316
  379. if 'status' not in event:
  380. continue
  381. try:
  382. # this can fail if the container has been removed or if the event
  383. # refers to an image
  384. container = Container.from_id(self.client, event['id'])
  385. except APIError:
  386. continue
  387. if container.service not in service_names:
  388. continue
  389. yield build_container_event(event, container)
  390. def events(self, service_names=None):
  391. if version_lt(self.client.api_version, '1.22'):
  392. # New, better event API was introduced in 1.22.
  393. return self._legacy_event_processor(service_names)
  394. def build_container_event(event):
  395. container_attrs = event['Actor']['Attributes']
  396. time = datetime.datetime.fromtimestamp(event['time'])
  397. time = time.replace(
  398. microsecond=microseconds_from_time_nano(event['timeNano'])
  399. )
  400. container = None
  401. try:
  402. container = Container.from_id(self.client, event['id'])
  403. except APIError:
  404. # Container may have been removed (e.g. if this is a destroy event)
  405. pass
  406. return {
  407. 'time': time,
  408. 'type': 'container',
  409. 'action': event['status'],
  410. 'id': event['Actor']['ID'],
  411. 'service': container_attrs.get(LABEL_SERVICE),
  412. 'attributes': dict([
  413. (k, v) for k, v in container_attrs.items()
  414. if not k.startswith('com.docker.compose.')
  415. ]),
  416. 'container': container,
  417. }
  418. def yield_loop(service_names):
  419. for event in self.client.events(
  420. filters={'label': self.labels()},
  421. decode=True
  422. ):
  423. # TODO: support other event types
  424. if event.get('Type') != 'container':
  425. continue
  426. try:
  427. if event['Actor']['Attributes'][LABEL_SERVICE] not in service_names:
  428. continue
  429. except KeyError:
  430. continue
  431. yield build_container_event(event)
  432. return yield_loop(set(service_names) if service_names else self.service_names)
  433. def up(self,
  434. service_names=None,
  435. start_deps=True,
  436. strategy=ConvergenceStrategy.changed,
  437. do_build=BuildAction.none,
  438. timeout=None,
  439. detached=False,
  440. remove_orphans=False,
  441. ignore_orphans=False,
  442. scale_override=None,
  443. rescale=True,
  444. start=True,
  445. always_recreate_deps=False,
  446. reset_container_image=False,
  447. renew_anonymous_volumes=False,
  448. silent=False,
  449. cli=False,
  450. ):
  451. if cli:
  452. log.warning("Native build is an experimental feature and could change at any time")
  453. self.initialize()
  454. if not ignore_orphans:
  455. self.find_orphan_containers(remove_orphans)
  456. if scale_override is None:
  457. scale_override = {}
  458. services = self.get_services_without_duplicate(
  459. service_names,
  460. include_deps=start_deps)
  461. for svc in services:
  462. svc.ensure_image_exists(do_build=do_build, silent=silent, cli=cli)
  463. plans = self._get_convergence_plans(
  464. services, strategy, always_recreate_deps=always_recreate_deps)
  465. def do(service):
  466. return service.execute_convergence_plan(
  467. plans[service.name],
  468. timeout=timeout,
  469. detached=detached,
  470. scale_override=scale_override.get(service.name),
  471. rescale=rescale,
  472. start=start,
  473. reset_container_image=reset_container_image,
  474. renew_anonymous_volumes=renew_anonymous_volumes,
  475. )
  476. def get_deps(service):
  477. return {
  478. (self.get_service(dep), config)
  479. for dep, config in service.get_dependency_configs().items()
  480. }
  481. results, errors = parallel.parallel_execute(
  482. services,
  483. do,
  484. operator.attrgetter('name'),
  485. None,
  486. get_deps,
  487. )
  488. if errors:
  489. raise ProjectError(
  490. 'Encountered errors while bringing up the project.'
  491. )
  492. return [
  493. container
  494. for svc_containers in results
  495. if svc_containers is not None
  496. for container in svc_containers
  497. ]
  498. def initialize(self):
  499. self.networks.initialize()
  500. self.volumes.initialize()
  501. def _get_convergence_plans(self, services, strategy, always_recreate_deps=False):
  502. plans = {}
  503. for service in services:
  504. updated_dependencies = [
  505. name
  506. for name in service.get_dependency_names()
  507. if name in plans and
  508. plans[name].action in ('recreate', 'create')
  509. ]
  510. if updated_dependencies and strategy.allows_recreate:
  511. log.debug('%s has upstream changes (%s)',
  512. service.name,
  513. ", ".join(updated_dependencies))
  514. containers_stopped = any(
  515. service.containers(stopped=True, filters={'status': ['created', 'exited']}))
  516. service_has_links = any(service.get_link_names())
  517. container_has_links = any(c.get('HostConfig.Links') for c in service.containers())
  518. should_recreate_for_links = service_has_links ^ container_has_links
  519. if always_recreate_deps or containers_stopped or should_recreate_for_links:
  520. plan = service.convergence_plan(ConvergenceStrategy.always)
  521. else:
  522. plan = service.convergence_plan(strategy)
  523. else:
  524. plan = service.convergence_plan(strategy)
  525. plans[service.name] = plan
  526. return plans
  527. def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False,
  528. include_deps=False):
  529. services = self.get_services(service_names, include_deps)
  530. if parallel_pull:
  531. self.parallel_pull(services, silent=silent)
  532. else:
  533. must_build = []
  534. for service in services:
  535. try:
  536. service.pull(ignore_pull_failures, silent=silent)
  537. except (ImageNotFound, NotFound):
  538. if service.can_be_built():
  539. must_build.append(service.name)
  540. else:
  541. raise
  542. if len(must_build):
  543. log.warning('Some service image(s) must be built from source by running:\n'
  544. ' docker-compose build {}'
  545. .format(' '.join(must_build)))
  546. def parallel_pull(self, services, ignore_pull_failures=False, silent=False):
  547. msg = 'Pulling' if not silent else None
  548. must_build = []
  549. def pull_service(service):
  550. strm = service.pull(ignore_pull_failures, True, stream=True)
  551. if strm is None: # Attempting to pull service with no `image` key is a no-op
  552. return
  553. try:
  554. writer = parallel.get_stream_writer()
  555. for event in strm:
  556. if 'status' not in event:
  557. continue
  558. status = read_status(event)
  559. writer.write(
  560. msg, service.name, truncate_string(status), lambda s: s
  561. )
  562. except (ImageNotFound, NotFound):
  563. if service.can_be_built():
  564. must_build.append(service.name)
  565. else:
  566. raise
  567. _, errors = parallel.parallel_execute(
  568. services,
  569. pull_service,
  570. operator.attrgetter('name'),
  571. msg,
  572. limit=5,
  573. )
  574. if len(must_build):
  575. log.warning('Some service image(s) must be built from source by running:\n'
  576. ' docker-compose build {}'
  577. .format(' '.join(must_build)))
  578. if len(errors):
  579. combined_errors = '\n'.join([
  580. e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values()
  581. ])
  582. raise ProjectError(combined_errors)
  583. def push(self, service_names=None, ignore_push_failures=False):
  584. unique_images = set()
  585. for service in self.get_services(service_names, include_deps=False):
  586. # Considering <image> and <image:latest> as the same
  587. repo, tag, sep = parse_repository_tag(service.image_name)
  588. service_image_name = sep.join((repo, tag)) if tag else sep.join((repo, 'latest'))
  589. if service_image_name not in unique_images:
  590. service.push(ignore_push_failures)
  591. unique_images.add(service_image_name)
  592. def _labeled_containers(self, stopped=False, one_off=OneOffFilter.exclude):
  593. ctnrs = list(filter(None, [
  594. Container.from_ps(self.client, container)
  595. for container in self.client.containers(
  596. all=stopped,
  597. filters={'label': self.labels(one_off=one_off)})])
  598. )
  599. if ctnrs:
  600. return ctnrs
  601. return list(filter(lambda c: c.has_legacy_proj_name(self.name), filter(None, [
  602. Container.from_ps(self.client, container)
  603. for container in self.client.containers(
  604. all=stopped,
  605. filters={'label': self.labels(one_off=one_off, legacy=True)})])
  606. ))
  607. def containers(self, service_names=None, stopped=False, one_off=OneOffFilter.exclude):
  608. if service_names:
  609. self.validate_service_names(service_names)
  610. else:
  611. service_names = self.service_names
  612. containers = self._labeled_containers(stopped, one_off)
  613. def matches_service_names(container):
  614. return container.labels.get(LABEL_SERVICE) in service_names
  615. return [c for c in containers if matches_service_names(c)]
  616. def find_orphan_containers(self, remove_orphans):
  617. def _find():
  618. containers = set(self._labeled_containers() + self._labeled_containers(stopped=True))
  619. for ctnr in containers:
  620. service_name = ctnr.labels.get(LABEL_SERVICE)
  621. if service_name not in self.service_names:
  622. yield ctnr
  623. orphans = list(_find())
  624. if not orphans:
  625. return
  626. if remove_orphans:
  627. for ctnr in orphans:
  628. log.info('Removing orphan container "{0}"'.format(ctnr.name))
  629. try:
  630. ctnr.kill()
  631. except APIError:
  632. pass
  633. ctnr.remove(force=True)
  634. else:
  635. log.warning(
  636. 'Found orphan containers ({0}) for this project. If '
  637. 'you removed or renamed this service in your compose '
  638. 'file, you can run this command with the '
  639. '--remove-orphans flag to clean it up.'.format(
  640. ', '.join(["{}".format(ctnr.name) for ctnr in orphans])
  641. )
  642. )
  643. def _inject_deps(self, acc, service):
  644. dep_names = service.get_dependency_names()
  645. if len(dep_names) > 0:
  646. dep_services = self.get_services(
  647. service_names=list(set(dep_names)),
  648. include_deps=True
  649. )
  650. else:
  651. dep_services = []
  652. dep_services.append(service)
  653. return acc + dep_services
  654. def build_container_operation_with_timeout_func(self, operation, options):
  655. def container_operation_with_timeout(container):
  656. _options = options.copy()
  657. if _options.get('timeout') is None:
  658. service = self.get_service(container.service)
  659. _options['timeout'] = service.stop_timeout(None)
  660. return getattr(container, operation)(**_options)
  661. return container_operation_with_timeout
  662. def get_volumes_from(project, service_dict):
  663. volumes_from = service_dict.pop('volumes_from', None)
  664. if not volumes_from:
  665. return []
  666. def build_volume_from(spec):
  667. if spec.type == 'service':
  668. try:
  669. return spec._replace(source=project.get_service(spec.source))
  670. except NoSuchService:
  671. pass
  672. if spec.type == 'container':
  673. try:
  674. container = Container.from_id(project.client, spec.source)
  675. return spec._replace(source=container)
  676. except APIError:
  677. pass
  678. raise ConfigurationError(
  679. "Service \"{}\" mounts volumes from \"{}\", which is not the name "
  680. "of a service or container.".format(
  681. service_dict['name'],
  682. spec.source))
  683. return [build_volume_from(vf) for vf in volumes_from]
  684. def get_secrets(service, service_secrets, secret_defs):
  685. secrets = []
  686. for secret in service_secrets:
  687. secret_def = secret_defs.get(secret.source)
  688. if not secret_def:
  689. raise ConfigurationError(
  690. "Service \"{service}\" uses an undefined secret \"{secret}\" "
  691. .format(service=service, secret=secret.source))
  692. if secret_def.get('external'):
  693. log.warning("Service \"{service}\" uses secret \"{secret}\" which is external. "
  694. "External secrets are not available to containers created by "
  695. "docker-compose.".format(service=service, secret=secret.source))
  696. continue
  697. if secret.uid or secret.gid or secret.mode:
  698. log.warning(
  699. "Service \"{service}\" uses secret \"{secret}\" with uid, "
  700. "gid, or mode. These fields are not supported by this "
  701. "implementation of the Compose file".format(
  702. service=service, secret=secret.source
  703. )
  704. )
  705. secret_file = secret_def.get('file')
  706. if not path.isfile(str(secret_file)):
  707. log.warning(
  708. "Service \"{service}\" uses an undefined secret file \"{secret_file}\", "
  709. "the following file should be created \"{secret_file}\"".format(
  710. service=service, secret_file=secret_file
  711. )
  712. )
  713. secrets.append({'secret': secret, 'file': secret_file})
  714. return secrets
  715. def get_image_digests(project):
  716. digests = {}
  717. needs_push = set()
  718. needs_pull = set()
  719. for service in project.services:
  720. try:
  721. digests[service.name] = get_image_digest(service)
  722. except NeedsPush as e:
  723. needs_push.add(e.image_name)
  724. except NeedsPull as e:
  725. needs_pull.add(e.service_name)
  726. if needs_push or needs_pull:
  727. raise MissingDigests(needs_push, needs_pull)
  728. return digests
  729. def get_image_digest(service):
  730. if 'image' not in service.options:
  731. raise UserError(
  732. "Service '{s.name}' doesn't define an image tag. An image name is "
  733. "required to generate a proper image digest. Specify an image repo "
  734. "and tag with the 'image' option.".format(s=service))
  735. _, _, separator = parse_repository_tag(service.options['image'])
  736. # Compose file already uses a digest, no lookup required
  737. if separator == '@':
  738. return service.options['image']
  739. digest = get_digest(service)
  740. if digest:
  741. return digest
  742. if 'build' not in service.options:
  743. raise NeedsPull(service.image_name, service.name)
  744. raise NeedsPush(service.image_name)
  745. def get_digest(service):
  746. digest = None
  747. try:
  748. image = service.image()
  749. # TODO: pick a digest based on the image tag if there are multiple
  750. # digests
  751. if image['RepoDigests']:
  752. digest = image['RepoDigests'][0]
  753. except NoSuchImageError:
  754. try:
  755. # Fetch the image digest from the registry
  756. distribution = service.get_image_registry_data()
  757. if distribution['Descriptor']['digest']:
  758. digest = '{image_name}@{digest}'.format(
  759. image_name=service.image_name,
  760. digest=distribution['Descriptor']['digest']
  761. )
  762. except NoSuchImageError:
  763. raise UserError(
  764. "Digest not found for service '{service}'. "
  765. "Repository does not exist or may require 'docker login'"
  766. .format(service=service.name))
  767. return digest
  768. class MissingDigests(Exception):
  769. def __init__(self, needs_push, needs_pull):
  770. self.needs_push = needs_push
  771. self.needs_pull = needs_pull
  772. class NeedsPush(Exception):
  773. def __init__(self, image_name):
  774. self.image_name = image_name
  775. class NeedsPull(Exception):
  776. def __init__(self, image_name, service_name):
  777. self.image_name = image_name
  778. self.service_name = service_name
  779. class NoSuchService(Exception):
  780. def __init__(self, name):
  781. if isinstance(name, six.binary_type):
  782. name = name.decode('utf-8')
  783. self.name = name
  784. self.msg = "No such service: %s" % self.name
  785. def __str__(self):
  786. return self.msg
  787. class ProjectError(Exception):
  788. def __init__(self, msg):
  789. self.msg = msg