project.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948
  1. from __future__ import absolute_import
  2. from __future__ import unicode_literals
  3. import datetime
  4. import logging
  5. import operator
  6. import re
  7. from functools import reduce
  8. from os import path
  9. import enum
  10. import six
  11. from docker.errors import APIError
  12. from docker.errors import ImageNotFound
  13. from docker.errors import NotFound
  14. from docker.utils import version_lt
  15. from . import parallel
  16. from .cli.errors import UserError
  17. from .config import ConfigurationError
  18. from .config.config import V1
  19. from .config.sort_services import get_container_name_from_network_mode
  20. from .config.sort_services import get_service_name_from_network_mode
  21. from .const import LABEL_ONE_OFF
  22. from .const import LABEL_PROJECT
  23. from .const import LABEL_SERVICE
  24. from .container import Container
  25. from .network import build_networks
  26. from .network import get_networks
  27. from .network import ProjectNetworks
  28. from .progress_stream import read_status
  29. from .service import BuildAction
  30. from .service import ContainerNetworkMode
  31. from .service import ContainerPidMode
  32. from .service import ConvergenceStrategy
  33. from .service import NetworkMode
  34. from .service import NoSuchImageError
  35. from .service import parse_repository_tag
  36. from .service import PidMode
  37. from .service import Service
  38. from .service import ServiceNetworkMode
  39. from .service import ServicePidMode
  40. from .utils import microseconds_from_time_nano
  41. from .utils import truncate_string
  42. from .volume import ProjectVolumes
  43. log = logging.getLogger(__name__)
  44. @enum.unique
  45. class OneOffFilter(enum.Enum):
  46. include = 0
  47. exclude = 1
  48. only = 2
  49. @classmethod
  50. def update_labels(cls, value, labels):
  51. if value == cls.only:
  52. labels.append('{0}={1}'.format(LABEL_ONE_OFF, "True"))
  53. elif value == cls.exclude:
  54. labels.append('{0}={1}'.format(LABEL_ONE_OFF, "False"))
  55. elif value == cls.include:
  56. pass
  57. else:
  58. raise ValueError("Invalid value for one_off: {}".format(repr(value)))
  59. class Project(object):
  60. """
  61. A collection of services.
  62. """
  63. def __init__(self, name, services, client, networks=None, volumes=None, config_version=None):
  64. self.name = name
  65. self.services = services
  66. self.client = client
  67. self.volumes = volumes or ProjectVolumes({})
  68. self.networks = networks or ProjectNetworks({}, False)
  69. self.config_version = config_version
  70. def labels(self, one_off=OneOffFilter.exclude, legacy=False):
  71. name = self.name
  72. if legacy:
  73. name = re.sub(r'[_-]', '', name)
  74. labels = ['{0}={1}'.format(LABEL_PROJECT, name)]
  75. OneOffFilter.update_labels(one_off, labels)
  76. return labels
  77. @classmethod
  78. def from_config(cls, name, config_data, client, default_platform=None, extra_labels=None):
  79. """
  80. Construct a Project from a config.Config object.
  81. """
  82. extra_labels = extra_labels or []
  83. use_networking = (config_data.version and config_data.version != V1)
  84. networks = build_networks(name, config_data, client)
  85. project_networks = ProjectNetworks.from_services(
  86. config_data.services,
  87. networks,
  88. use_networking)
  89. volumes = ProjectVolumes.from_config(name, config_data, client)
  90. project = cls(name, [], client, project_networks, volumes, config_data.version)
  91. for service_dict in config_data.services:
  92. service_dict = dict(service_dict)
  93. if use_networking:
  94. service_networks = get_networks(service_dict, networks)
  95. else:
  96. service_networks = {}
  97. service_dict.pop('networks', None)
  98. links = project.get_links(service_dict)
  99. network_mode = project.get_network_mode(
  100. service_dict, list(service_networks.keys())
  101. )
  102. pid_mode = project.get_pid_mode(service_dict)
  103. volumes_from = get_volumes_from(project, service_dict)
  104. if config_data.version != V1:
  105. service_dict['volumes'] = [
  106. volumes.namespace_spec(volume_spec)
  107. for volume_spec in service_dict.get('volumes', [])
  108. ]
  109. secrets = get_secrets(
  110. service_dict['name'],
  111. service_dict.pop('secrets', None) or [],
  112. config_data.secrets)
  113. project.services.append(
  114. Service(
  115. service_dict.pop('name'),
  116. client=client,
  117. project=name,
  118. use_networking=use_networking,
  119. networks=service_networks,
  120. links=links,
  121. network_mode=network_mode,
  122. volumes_from=volumes_from,
  123. secrets=secrets,
  124. pid_mode=pid_mode,
  125. platform=service_dict.pop('platform', None),
  126. default_platform=default_platform,
  127. extra_labels=extra_labels,
  128. **service_dict)
  129. )
  130. return project
  131. @property
  132. def service_names(self):
  133. return [service.name for service in self.services]
  134. def get_service(self, name):
  135. """
  136. Retrieve a service by name. Raises NoSuchService
  137. if the named service does not exist.
  138. """
  139. for service in self.services:
  140. if service.name == name:
  141. return service
  142. raise NoSuchService(name)
  143. def validate_service_names(self, service_names):
  144. """
  145. Validate that the given list of service names only contains valid
  146. services. Raises NoSuchService if one of the names is invalid.
  147. """
  148. valid_names = self.service_names
  149. for name in service_names:
  150. if name not in valid_names:
  151. raise NoSuchService(name)
  152. def get_services(self, service_names=None, include_deps=False):
  153. """
  154. Returns a list of this project's services filtered
  155. by the provided list of names, or all services if service_names is None
  156. or [].
  157. If include_deps is specified, returns a list including the dependencies for
  158. service_names, in order of dependency.
  159. Preserves the original order of self.services where possible,
  160. reordering as needed to resolve dependencies.
  161. Raises NoSuchService if any of the named services do not exist.
  162. """
  163. if service_names is None or len(service_names) == 0:
  164. service_names = self.service_names
  165. unsorted = [self.get_service(name) for name in service_names]
  166. services = [s for s in self.services if s in unsorted]
  167. if include_deps:
  168. services = reduce(self._inject_deps, services, [])
  169. uniques = []
  170. [uniques.append(s) for s in services if s not in uniques]
  171. return uniques
  172. def get_services_without_duplicate(self, service_names=None, include_deps=False):
  173. services = self.get_services(service_names, include_deps)
  174. for service in services:
  175. service.remove_duplicate_containers()
  176. return services
  177. def get_links(self, service_dict):
  178. links = []
  179. if 'links' in service_dict:
  180. for link in service_dict.get('links', []):
  181. if ':' in link:
  182. service_name, link_name = link.split(':', 1)
  183. else:
  184. service_name, link_name = link, None
  185. try:
  186. links.append((self.get_service(service_name), link_name))
  187. except NoSuchService:
  188. raise ConfigurationError(
  189. 'Service "%s" has a link to service "%s" which does not '
  190. 'exist.' % (service_dict['name'], service_name))
  191. del service_dict['links']
  192. return links
  193. def get_network_mode(self, service_dict, networks):
  194. network_mode = service_dict.pop('network_mode', None)
  195. if not network_mode:
  196. if self.networks.use_networking:
  197. return NetworkMode(networks[0]) if networks else NetworkMode('none')
  198. return NetworkMode(None)
  199. service_name = get_service_name_from_network_mode(network_mode)
  200. if service_name:
  201. return ServiceNetworkMode(self.get_service(service_name))
  202. container_name = get_container_name_from_network_mode(network_mode)
  203. if container_name:
  204. try:
  205. return ContainerNetworkMode(Container.from_id(self.client, container_name))
  206. except APIError:
  207. raise ConfigurationError(
  208. "Service '{name}' uses the network stack of container '{dep}' which "
  209. "does not exist.".format(name=service_dict['name'], dep=container_name))
  210. return NetworkMode(network_mode)
  211. def get_pid_mode(self, service_dict):
  212. pid_mode = service_dict.pop('pid', None)
  213. if not pid_mode:
  214. return PidMode(None)
  215. service_name = get_service_name_from_network_mode(pid_mode)
  216. if service_name:
  217. return ServicePidMode(self.get_service(service_name))
  218. container_name = get_container_name_from_network_mode(pid_mode)
  219. if container_name:
  220. try:
  221. return ContainerPidMode(Container.from_id(self.client, container_name))
  222. except APIError:
  223. raise ConfigurationError(
  224. "Service '{name}' uses the PID namespace of container '{dep}' which "
  225. "does not exist.".format(name=service_dict['name'], dep=container_name)
  226. )
  227. return PidMode(pid_mode)
  228. def start(self, service_names=None, **options):
  229. containers = []
  230. def start_service(service):
  231. service_containers = service.start(quiet=True, **options)
  232. containers.extend(service_containers)
  233. services = self.get_services(service_names)
  234. def get_deps(service):
  235. return {
  236. (self.get_service(dep), config)
  237. for dep, config in service.get_dependency_configs().items()
  238. }
  239. parallel.parallel_execute(
  240. services,
  241. start_service,
  242. operator.attrgetter('name'),
  243. 'Starting',
  244. get_deps,
  245. fail_check=lambda obj: not obj.containers(),
  246. )
  247. return containers
  248. def stop(self, service_names=None, one_off=OneOffFilter.exclude, **options):
  249. containers = self.containers(service_names, one_off=one_off)
  250. def get_deps(container):
  251. # actually returning inversed dependencies
  252. return {(other, None) for other in containers
  253. if container.service in
  254. self.get_service(other.service).get_dependency_names()}
  255. parallel.parallel_execute(
  256. containers,
  257. self.build_container_operation_with_timeout_func('stop', options),
  258. operator.attrgetter('name'),
  259. 'Stopping',
  260. get_deps,
  261. )
  262. def pause(self, service_names=None, **options):
  263. containers = self.containers(service_names)
  264. parallel.parallel_pause(reversed(containers), options)
  265. return containers
  266. def unpause(self, service_names=None, **options):
  267. containers = self.containers(service_names)
  268. parallel.parallel_unpause(containers, options)
  269. return containers
  270. def kill(self, service_names=None, **options):
  271. parallel.parallel_kill(self.containers(service_names), options)
  272. def remove_stopped(self, service_names=None, one_off=OneOffFilter.exclude, **options):
  273. parallel.parallel_remove(self.containers(
  274. service_names, stopped=True, one_off=one_off
  275. ), options)
  276. def down(
  277. self,
  278. remove_image_type,
  279. include_volumes,
  280. remove_orphans=False,
  281. timeout=None,
  282. ignore_orphans=False):
  283. self.stop(one_off=OneOffFilter.include, timeout=timeout)
  284. if not ignore_orphans:
  285. self.find_orphan_containers(remove_orphans)
  286. self.remove_stopped(v=include_volumes, one_off=OneOffFilter.include)
  287. self.networks.remove()
  288. if include_volumes:
  289. self.volumes.remove()
  290. self.remove_images(remove_image_type)
  291. def remove_images(self, remove_image_type):
  292. for service in self.get_services():
  293. service.remove_image(remove_image_type)
  294. def restart(self, service_names=None, **options):
  295. containers = self.containers(service_names, stopped=True)
  296. parallel.parallel_execute(
  297. containers,
  298. self.build_container_operation_with_timeout_func('restart', options),
  299. operator.attrgetter('name'),
  300. 'Restarting',
  301. )
  302. return containers
  303. def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None,
  304. build_args=None, gzip=False, parallel_build=False, rm=True, silent=False, cli=False,
  305. progress=None):
  306. services = []
  307. for service in self.get_services(service_names):
  308. if service.can_be_built():
  309. services.append(service)
  310. elif not silent:
  311. log.info('%s uses an image, skipping' % service.name)
  312. if cli:
  313. log.warning("Native build is an experimental feature and could change at any time")
  314. if parallel_build:
  315. log.warning("Flag '--parallel' is ignored when building with "
  316. "COMPOSE_DOCKER_CLI_BUILD=1")
  317. if gzip:
  318. log.warning("Flag '--compress' is ignored when building with "
  319. "COMPOSE_DOCKER_CLI_BUILD=1")
  320. def build_service(service):
  321. service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli, progress)
  322. if parallel_build:
  323. _, errors = parallel.parallel_execute(
  324. services,
  325. build_service,
  326. operator.attrgetter('name'),
  327. 'Building',
  328. limit=5,
  329. )
  330. if len(errors):
  331. combined_errors = '\n'.join([
  332. e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values()
  333. ])
  334. raise ProjectError(combined_errors)
  335. else:
  336. for service in services:
  337. build_service(service)
  338. def create(
  339. self,
  340. service_names=None,
  341. strategy=ConvergenceStrategy.changed,
  342. do_build=BuildAction.none,
  343. ):
  344. services = self.get_services_without_duplicate(service_names, include_deps=True)
  345. for svc in services:
  346. svc.ensure_image_exists(do_build=do_build)
  347. plans = self._get_convergence_plans(services, strategy)
  348. for service in services:
  349. service.execute_convergence_plan(
  350. plans[service.name],
  351. detached=True,
  352. start=False)
  353. def _legacy_event_processor(self, service_names):
  354. # Only for v1 files or when Compose is forced to use an older API version
  355. def build_container_event(event, container):
  356. time = datetime.datetime.fromtimestamp(event['time'])
  357. time = time.replace(
  358. microsecond=microseconds_from_time_nano(event['timeNano'])
  359. )
  360. return {
  361. 'time': time,
  362. 'type': 'container',
  363. 'action': event['status'],
  364. 'id': container.id,
  365. 'service': container.service,
  366. 'attributes': {
  367. 'name': container.name,
  368. 'image': event['from'],
  369. },
  370. 'container': container,
  371. }
  372. service_names = set(service_names or self.service_names)
  373. for event in self.client.events(
  374. filters={'label': self.labels()},
  375. decode=True
  376. ):
  377. # This is a guard against some events broadcasted by swarm that
  378. # don't have a status field.
  379. # See https://github.com/docker/compose/issues/3316
  380. if 'status' not in event:
  381. continue
  382. try:
  383. # this can fail if the container has been removed or if the event
  384. # refers to an image
  385. container = Container.from_id(self.client, event['id'])
  386. except APIError:
  387. continue
  388. if container.service not in service_names:
  389. continue
  390. yield build_container_event(event, container)
  391. def events(self, service_names=None):
  392. if version_lt(self.client.api_version, '1.22'):
  393. # New, better event API was introduced in 1.22.
  394. return self._legacy_event_processor(service_names)
  395. def build_container_event(event):
  396. container_attrs = event['Actor']['Attributes']
  397. time = datetime.datetime.fromtimestamp(event['time'])
  398. time = time.replace(
  399. microsecond=microseconds_from_time_nano(event['timeNano'])
  400. )
  401. container = None
  402. try:
  403. container = Container.from_id(self.client, event['id'])
  404. except APIError:
  405. # Container may have been removed (e.g. if this is a destroy event)
  406. pass
  407. return {
  408. 'time': time,
  409. 'type': 'container',
  410. 'action': event['status'],
  411. 'id': event['Actor']['ID'],
  412. 'service': container_attrs.get(LABEL_SERVICE),
  413. 'attributes': dict([
  414. (k, v) for k, v in container_attrs.items()
  415. if not k.startswith('com.docker.compose.')
  416. ]),
  417. 'container': container,
  418. }
  419. def yield_loop(service_names):
  420. for event in self.client.events(
  421. filters={'label': self.labels()},
  422. decode=True
  423. ):
  424. # TODO: support other event types
  425. if event.get('Type') != 'container':
  426. continue
  427. try:
  428. if event['Actor']['Attributes'][LABEL_SERVICE] not in service_names:
  429. continue
  430. except KeyError:
  431. continue
  432. yield build_container_event(event)
  433. return yield_loop(set(service_names) if service_names else self.service_names)
  434. def up(self,
  435. service_names=None,
  436. start_deps=True,
  437. strategy=ConvergenceStrategy.changed,
  438. do_build=BuildAction.none,
  439. timeout=None,
  440. detached=False,
  441. remove_orphans=False,
  442. ignore_orphans=False,
  443. scale_override=None,
  444. rescale=True,
  445. start=True,
  446. always_recreate_deps=False,
  447. reset_container_image=False,
  448. renew_anonymous_volumes=False,
  449. silent=False,
  450. cli=False,
  451. ):
  452. if cli:
  453. log.warning("Native build is an experimental feature and could change at any time")
  454. self.initialize()
  455. if not ignore_orphans:
  456. self.find_orphan_containers(remove_orphans)
  457. if scale_override is None:
  458. scale_override = {}
  459. services = self.get_services_without_duplicate(
  460. service_names,
  461. include_deps=start_deps)
  462. for svc in services:
  463. svc.ensure_image_exists(do_build=do_build, silent=silent, cli=cli)
  464. plans = self._get_convergence_plans(
  465. services, strategy, always_recreate_deps=always_recreate_deps)
  466. def do(service):
  467. return service.execute_convergence_plan(
  468. plans[service.name],
  469. timeout=timeout,
  470. detached=detached,
  471. scale_override=scale_override.get(service.name),
  472. rescale=rescale,
  473. start=start,
  474. reset_container_image=reset_container_image,
  475. renew_anonymous_volumes=renew_anonymous_volumes,
  476. )
  477. def get_deps(service):
  478. return {
  479. (self.get_service(dep), config)
  480. for dep, config in service.get_dependency_configs().items()
  481. }
  482. results, errors = parallel.parallel_execute(
  483. services,
  484. do,
  485. operator.attrgetter('name'),
  486. None,
  487. get_deps,
  488. )
  489. if errors:
  490. raise ProjectError(
  491. 'Encountered errors while bringing up the project.'
  492. )
  493. return [
  494. container
  495. for svc_containers in results
  496. if svc_containers is not None
  497. for container in svc_containers
  498. ]
  499. def initialize(self):
  500. self.networks.initialize()
  501. self.volumes.initialize()
  502. def _get_convergence_plans(self, services, strategy, always_recreate_deps=False):
  503. plans = {}
  504. for service in services:
  505. updated_dependencies = [
  506. name
  507. for name in service.get_dependency_names()
  508. if name in plans and
  509. plans[name].action in ('recreate', 'create')
  510. ]
  511. if updated_dependencies and strategy.allows_recreate:
  512. log.debug('%s has upstream changes (%s)',
  513. service.name,
  514. ", ".join(updated_dependencies))
  515. containers_stopped = any(
  516. service.containers(stopped=True, filters={'status': ['created', 'exited']}))
  517. service_has_links = any(service.get_link_names())
  518. container_has_links = any(c.get('HostConfig.Links') for c in service.containers())
  519. should_recreate_for_links = service_has_links ^ container_has_links
  520. if always_recreate_deps or containers_stopped or should_recreate_for_links:
  521. plan = service.convergence_plan(ConvergenceStrategy.always)
  522. else:
  523. plan = service.convergence_plan(strategy)
  524. else:
  525. plan = service.convergence_plan(strategy)
  526. plans[service.name] = plan
  527. return plans
  528. def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False,
  529. include_deps=False):
  530. services = self.get_services(service_names, include_deps)
  531. if parallel_pull:
  532. self.parallel_pull(services, silent=silent)
  533. else:
  534. must_build = []
  535. for service in services:
  536. try:
  537. service.pull(ignore_pull_failures, silent=silent)
  538. except (ImageNotFound, NotFound):
  539. if service.can_be_built():
  540. must_build.append(service.name)
  541. else:
  542. raise
  543. if len(must_build):
  544. log.warning('Some service image(s) must be built from source by running:\n'
  545. ' docker-compose build {}'
  546. .format(' '.join(must_build)))
  547. def parallel_pull(self, services, ignore_pull_failures=False, silent=False):
  548. msg = 'Pulling' if not silent else None
  549. must_build = []
  550. def pull_service(service):
  551. strm = service.pull(ignore_pull_failures, True, stream=True)
  552. if strm is None: # Attempting to pull service with no `image` key is a no-op
  553. return
  554. try:
  555. writer = parallel.get_stream_writer()
  556. for event in strm:
  557. if 'status' not in event:
  558. continue
  559. status = read_status(event)
  560. writer.write(
  561. msg, service.name, truncate_string(status), lambda s: s
  562. )
  563. except (ImageNotFound, NotFound):
  564. if service.can_be_built():
  565. must_build.append(service.name)
  566. else:
  567. raise
  568. _, errors = parallel.parallel_execute(
  569. services,
  570. pull_service,
  571. operator.attrgetter('name'),
  572. msg,
  573. limit=5,
  574. )
  575. if len(must_build):
  576. log.warning('Some service image(s) must be built from source by running:\n'
  577. ' docker-compose build {}'
  578. .format(' '.join(must_build)))
  579. if len(errors):
  580. combined_errors = '\n'.join([
  581. e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values()
  582. ])
  583. raise ProjectError(combined_errors)
  584. def push(self, service_names=None, ignore_push_failures=False):
  585. unique_images = set()
  586. for service in self.get_services(service_names, include_deps=False):
  587. # Considering <image> and <image:latest> as the same
  588. repo, tag, sep = parse_repository_tag(service.image_name)
  589. service_image_name = sep.join((repo, tag)) if tag else sep.join((repo, 'latest'))
  590. if service_image_name not in unique_images:
  591. service.push(ignore_push_failures)
  592. unique_images.add(service_image_name)
  593. def _labeled_containers(self, stopped=False, one_off=OneOffFilter.exclude):
  594. ctnrs = list(filter(None, [
  595. Container.from_ps(self.client, container)
  596. for container in self.client.containers(
  597. all=stopped,
  598. filters={'label': self.labels(one_off=one_off)})])
  599. )
  600. if ctnrs:
  601. return ctnrs
  602. return list(filter(lambda c: c.has_legacy_proj_name(self.name), filter(None, [
  603. Container.from_ps(self.client, container)
  604. for container in self.client.containers(
  605. all=stopped,
  606. filters={'label': self.labels(one_off=one_off, legacy=True)})])
  607. ))
  608. def containers(self, service_names=None, stopped=False, one_off=OneOffFilter.exclude):
  609. if service_names:
  610. self.validate_service_names(service_names)
  611. else:
  612. service_names = self.service_names
  613. containers = self._labeled_containers(stopped, one_off)
  614. def matches_service_names(container):
  615. return container.labels.get(LABEL_SERVICE) in service_names
  616. return [c for c in containers if matches_service_names(c)]
  617. def find_orphan_containers(self, remove_orphans):
  618. def _find():
  619. containers = set(self._labeled_containers() + self._labeled_containers(stopped=True))
  620. for ctnr in containers:
  621. service_name = ctnr.labels.get(LABEL_SERVICE)
  622. if service_name not in self.service_names:
  623. yield ctnr
  624. orphans = list(_find())
  625. if not orphans:
  626. return
  627. if remove_orphans:
  628. for ctnr in orphans:
  629. log.info('Removing orphan container "{0}"'.format(ctnr.name))
  630. try:
  631. ctnr.kill()
  632. except APIError:
  633. pass
  634. ctnr.remove(force=True)
  635. else:
  636. log.warning(
  637. 'Found orphan containers ({0}) for this project. If '
  638. 'you removed or renamed this service in your compose '
  639. 'file, you can run this command with the '
  640. '--remove-orphans flag to clean it up.'.format(
  641. ', '.join(["{}".format(ctnr.name) for ctnr in orphans])
  642. )
  643. )
  644. def _inject_deps(self, acc, service):
  645. dep_names = service.get_dependency_names()
  646. if len(dep_names) > 0:
  647. dep_services = self.get_services(
  648. service_names=list(set(dep_names)),
  649. include_deps=True
  650. )
  651. else:
  652. dep_services = []
  653. dep_services.append(service)
  654. return acc + dep_services
  655. def build_container_operation_with_timeout_func(self, operation, options):
  656. def container_operation_with_timeout(container):
  657. _options = options.copy()
  658. if _options.get('timeout') is None:
  659. service = self.get_service(container.service)
  660. _options['timeout'] = service.stop_timeout(None)
  661. return getattr(container, operation)(**_options)
  662. return container_operation_with_timeout
  663. def get_volumes_from(project, service_dict):
  664. volumes_from = service_dict.pop('volumes_from', None)
  665. if not volumes_from:
  666. return []
  667. def build_volume_from(spec):
  668. if spec.type == 'service':
  669. try:
  670. return spec._replace(source=project.get_service(spec.source))
  671. except NoSuchService:
  672. pass
  673. if spec.type == 'container':
  674. try:
  675. container = Container.from_id(project.client, spec.source)
  676. return spec._replace(source=container)
  677. except APIError:
  678. pass
  679. raise ConfigurationError(
  680. "Service \"{}\" mounts volumes from \"{}\", which is not the name "
  681. "of a service or container.".format(
  682. service_dict['name'],
  683. spec.source))
  684. return [build_volume_from(vf) for vf in volumes_from]
  685. def get_secrets(service, service_secrets, secret_defs):
  686. secrets = []
  687. for secret in service_secrets:
  688. secret_def = secret_defs.get(secret.source)
  689. if not secret_def:
  690. raise ConfigurationError(
  691. "Service \"{service}\" uses an undefined secret \"{secret}\" "
  692. .format(service=service, secret=secret.source))
  693. if secret_def.get('external'):
  694. log.warning("Service \"{service}\" uses secret \"{secret}\" which is external. "
  695. "External secrets are not available to containers created by "
  696. "docker-compose.".format(service=service, secret=secret.source))
  697. continue
  698. if secret.uid or secret.gid or secret.mode:
  699. log.warning(
  700. "Service \"{service}\" uses secret \"{secret}\" with uid, "
  701. "gid, or mode. These fields are not supported by this "
  702. "implementation of the Compose file".format(
  703. service=service, secret=secret.source
  704. )
  705. )
  706. secret_file = secret_def.get('file')
  707. if not path.isfile(str(secret_file)):
  708. log.warning(
  709. "Service \"{service}\" uses an undefined secret file \"{secret_file}\", "
  710. "the following file should be created \"{secret_file}\"".format(
  711. service=service, secret_file=secret_file
  712. )
  713. )
  714. secrets.append({'secret': secret, 'file': secret_file})
  715. return secrets
  716. def get_image_digests(project):
  717. digests = {}
  718. needs_push = set()
  719. needs_pull = set()
  720. for service in project.services:
  721. try:
  722. digests[service.name] = get_image_digest(service)
  723. except NeedsPush as e:
  724. needs_push.add(e.image_name)
  725. except NeedsPull as e:
  726. needs_pull.add(e.service_name)
  727. if needs_push or needs_pull:
  728. raise MissingDigests(needs_push, needs_pull)
  729. return digests
  730. def get_image_digest(service):
  731. if 'image' not in service.options:
  732. raise UserError(
  733. "Service '{s.name}' doesn't define an image tag. An image name is "
  734. "required to generate a proper image digest. Specify an image repo "
  735. "and tag with the 'image' option.".format(s=service))
  736. _, _, separator = parse_repository_tag(service.options['image'])
  737. # Compose file already uses a digest, no lookup required
  738. if separator == '@':
  739. return service.options['image']
  740. digest = get_digest(service)
  741. if digest:
  742. return digest
  743. if 'build' not in service.options:
  744. raise NeedsPull(service.image_name, service.name)
  745. raise NeedsPush(service.image_name)
  746. def get_digest(service):
  747. digest = None
  748. try:
  749. image = service.image()
  750. # TODO: pick a digest based on the image tag if there are multiple
  751. # digests
  752. if image['RepoDigests']:
  753. digest = image['RepoDigests'][0]
  754. except NoSuchImageError:
  755. try:
  756. # Fetch the image digest from the registry
  757. distribution = service.get_image_registry_data()
  758. if distribution['Descriptor']['digest']:
  759. digest = '{image_name}@{digest}'.format(
  760. image_name=service.image_name,
  761. digest=distribution['Descriptor']['digest']
  762. )
  763. except NoSuchImageError:
  764. raise UserError(
  765. "Digest not found for service '{service}'. "
  766. "Repository does not exist or may require 'docker login'"
  767. .format(service=service.name))
  768. return digest
  769. class MissingDigests(Exception):
  770. def __init__(self, needs_push, needs_pull):
  771. self.needs_push = needs_push
  772. self.needs_pull = needs_pull
  773. class NeedsPush(Exception):
  774. def __init__(self, image_name):
  775. self.image_name = image_name
  776. class NeedsPull(Exception):
  777. def __init__(self, image_name, service_name):
  778. self.image_name = image_name
  779. self.service_name = service_name
  780. class NoSuchService(Exception):
  781. def __init__(self, name):
  782. if isinstance(name, six.binary_type):
  783. name = name.decode('utf-8')
  784. self.name = name
  785. self.msg = "No such service: %s" % self.name
  786. def __str__(self):
  787. return self.msg
  788. class ProjectError(Exception):
  789. def __init__(self, msg):
  790. self.msg = msg