project.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945
  1. import datetime
  2. import enum
  3. import logging
  4. import operator
  5. import re
  6. from functools import reduce
  7. from os import path
  8. import six
  9. from docker.errors import APIError
  10. from docker.errors import ImageNotFound
  11. from docker.errors import NotFound
  12. from docker.utils import version_lt
  13. from . import parallel
  14. from .cli.errors import UserError
  15. from .config import ConfigurationError
  16. from .config.config import V1
  17. from .config.sort_services import get_container_name_from_network_mode
  18. from .config.sort_services import get_service_name_from_network_mode
  19. from .const import LABEL_ONE_OFF
  20. from .const import LABEL_PROJECT
  21. from .const import LABEL_SERVICE
  22. from .container import Container
  23. from .network import build_networks
  24. from .network import get_networks
  25. from .network import ProjectNetworks
  26. from .progress_stream import read_status
  27. from .service import BuildAction
  28. from .service import ContainerNetworkMode
  29. from .service import ContainerPidMode
  30. from .service import ConvergenceStrategy
  31. from .service import NetworkMode
  32. from .service import NoSuchImageError
  33. from .service import parse_repository_tag
  34. from .service import PidMode
  35. from .service import Service
  36. from .service import ServiceNetworkMode
  37. from .service import ServicePidMode
  38. from .utils import microseconds_from_time_nano
  39. from .utils import truncate_string
  40. from .volume import ProjectVolumes
  41. log = logging.getLogger(__name__)
  42. @enum.unique
  43. class OneOffFilter(enum.Enum):
  44. include = 0
  45. exclude = 1
  46. only = 2
  47. @classmethod
  48. def update_labels(cls, value, labels):
  49. if value == cls.only:
  50. labels.append('{0}={1}'.format(LABEL_ONE_OFF, "True"))
  51. elif value == cls.exclude:
  52. labels.append('{0}={1}'.format(LABEL_ONE_OFF, "False"))
  53. elif value == cls.include:
  54. pass
  55. else:
  56. raise ValueError("Invalid value for one_off: {}".format(repr(value)))
  57. class Project(object):
  58. """
  59. A collection of services.
  60. """
  61. def __init__(self, name, services, client, networks=None, volumes=None, config_version=None):
  62. self.name = name
  63. self.services = services
  64. self.client = client
  65. self.volumes = volumes or ProjectVolumes({})
  66. self.networks = networks or ProjectNetworks({}, False)
  67. self.config_version = config_version
  68. def labels(self, one_off=OneOffFilter.exclude, legacy=False):
  69. name = self.name
  70. if legacy:
  71. name = re.sub(r'[_-]', '', name)
  72. labels = ['{0}={1}'.format(LABEL_PROJECT, name)]
  73. OneOffFilter.update_labels(one_off, labels)
  74. return labels
  75. @classmethod
  76. def from_config(cls, name, config_data, client, default_platform=None, extra_labels=None):
  77. """
  78. Construct a Project from a config.Config object.
  79. """
  80. extra_labels = extra_labels or []
  81. use_networking = (config_data.version and config_data.version != V1)
  82. networks = build_networks(name, config_data, client)
  83. project_networks = ProjectNetworks.from_services(
  84. config_data.services,
  85. networks,
  86. use_networking)
  87. volumes = ProjectVolumes.from_config(name, config_data, client)
  88. project = cls(name, [], client, project_networks, volumes, config_data.version)
  89. for service_dict in config_data.services:
  90. service_dict = dict(service_dict)
  91. if use_networking:
  92. service_networks = get_networks(service_dict, networks)
  93. else:
  94. service_networks = {}
  95. service_dict.pop('networks', None)
  96. links = project.get_links(service_dict)
  97. network_mode = project.get_network_mode(
  98. service_dict, list(service_networks.keys())
  99. )
  100. pid_mode = project.get_pid_mode(service_dict)
  101. volumes_from = get_volumes_from(project, service_dict)
  102. if config_data.version != V1:
  103. service_dict['volumes'] = [
  104. volumes.namespace_spec(volume_spec)
  105. for volume_spec in service_dict.get('volumes', [])
  106. ]
  107. secrets = get_secrets(
  108. service_dict['name'],
  109. service_dict.pop('secrets', None) or [],
  110. config_data.secrets)
  111. project.services.append(
  112. Service(
  113. service_dict.pop('name'),
  114. client=client,
  115. project=name,
  116. use_networking=use_networking,
  117. networks=service_networks,
  118. links=links,
  119. network_mode=network_mode,
  120. volumes_from=volumes_from,
  121. secrets=secrets,
  122. pid_mode=pid_mode,
  123. platform=service_dict.pop('platform', None),
  124. default_platform=default_platform,
  125. extra_labels=extra_labels,
  126. **service_dict)
  127. )
  128. return project
  129. @property
  130. def service_names(self):
  131. return [service.name for service in self.services]
  132. def get_service(self, name):
  133. """
  134. Retrieve a service by name. Raises NoSuchService
  135. if the named service does not exist.
  136. """
  137. for service in self.services:
  138. if service.name == name:
  139. return service
  140. raise NoSuchService(name)
  141. def validate_service_names(self, service_names):
  142. """
  143. Validate that the given list of service names only contains valid
  144. services. Raises NoSuchService if one of the names is invalid.
  145. """
  146. valid_names = self.service_names
  147. for name in service_names:
  148. if name not in valid_names:
  149. raise NoSuchService(name)
  150. def get_services(self, service_names=None, include_deps=False):
  151. """
  152. Returns a list of this project's services filtered
  153. by the provided list of names, or all services if service_names is None
  154. or [].
  155. If include_deps is specified, returns a list including the dependencies for
  156. service_names, in order of dependency.
  157. Preserves the original order of self.services where possible,
  158. reordering as needed to resolve dependencies.
  159. Raises NoSuchService if any of the named services do not exist.
  160. """
  161. if service_names is None or len(service_names) == 0:
  162. service_names = self.service_names
  163. unsorted = [self.get_service(name) for name in service_names]
  164. services = [s for s in self.services if s in unsorted]
  165. if include_deps:
  166. services = reduce(self._inject_deps, services, [])
  167. uniques = []
  168. [uniques.append(s) for s in services if s not in uniques]
  169. return uniques
  170. def get_services_without_duplicate(self, service_names=None, include_deps=False):
  171. services = self.get_services(service_names, include_deps)
  172. for service in services:
  173. service.remove_duplicate_containers()
  174. return services
  175. def get_links(self, service_dict):
  176. links = []
  177. if 'links' in service_dict:
  178. for link in service_dict.get('links', []):
  179. if ':' in link:
  180. service_name, link_name = link.split(':', 1)
  181. else:
  182. service_name, link_name = link, None
  183. try:
  184. links.append((self.get_service(service_name), link_name))
  185. except NoSuchService:
  186. raise ConfigurationError(
  187. 'Service "%s" has a link to service "%s" which does not '
  188. 'exist.' % (service_dict['name'], service_name))
  189. del service_dict['links']
  190. return links
  191. def get_network_mode(self, service_dict, networks):
  192. network_mode = service_dict.pop('network_mode', None)
  193. if not network_mode:
  194. if self.networks.use_networking:
  195. return NetworkMode(networks[0]) if networks else NetworkMode('none')
  196. return NetworkMode(None)
  197. service_name = get_service_name_from_network_mode(network_mode)
  198. if service_name:
  199. return ServiceNetworkMode(self.get_service(service_name))
  200. container_name = get_container_name_from_network_mode(network_mode)
  201. if container_name:
  202. try:
  203. return ContainerNetworkMode(Container.from_id(self.client, container_name))
  204. except APIError:
  205. raise ConfigurationError(
  206. "Service '{name}' uses the network stack of container '{dep}' which "
  207. "does not exist.".format(name=service_dict['name'], dep=container_name))
  208. return NetworkMode(network_mode)
  209. def get_pid_mode(self, service_dict):
  210. pid_mode = service_dict.pop('pid', None)
  211. if not pid_mode:
  212. return PidMode(None)
  213. service_name = get_service_name_from_network_mode(pid_mode)
  214. if service_name:
  215. return ServicePidMode(self.get_service(service_name))
  216. container_name = get_container_name_from_network_mode(pid_mode)
  217. if container_name:
  218. try:
  219. return ContainerPidMode(Container.from_id(self.client, container_name))
  220. except APIError:
  221. raise ConfigurationError(
  222. "Service '{name}' uses the PID namespace of container '{dep}' which "
  223. "does not exist.".format(name=service_dict['name'], dep=container_name)
  224. )
  225. return PidMode(pid_mode)
  226. def start(self, service_names=None, **options):
  227. containers = []
  228. def start_service(service):
  229. service_containers = service.start(quiet=True, **options)
  230. containers.extend(service_containers)
  231. services = self.get_services(service_names)
  232. def get_deps(service):
  233. return {
  234. (self.get_service(dep), config)
  235. for dep, config in service.get_dependency_configs().items()
  236. }
  237. parallel.parallel_execute(
  238. services,
  239. start_service,
  240. operator.attrgetter('name'),
  241. 'Starting',
  242. get_deps,
  243. fail_check=lambda obj: not obj.containers(),
  244. )
  245. return containers
  246. def stop(self, service_names=None, one_off=OneOffFilter.exclude, **options):
  247. containers = self.containers(service_names, one_off=one_off)
  248. def get_deps(container):
  249. # actually returning inversed dependencies
  250. return {(other, None) for other in containers
  251. if container.service in
  252. self.get_service(other.service).get_dependency_names()}
  253. parallel.parallel_execute(
  254. containers,
  255. self.build_container_operation_with_timeout_func('stop', options),
  256. operator.attrgetter('name'),
  257. 'Stopping',
  258. get_deps,
  259. )
  260. def pause(self, service_names=None, **options):
  261. containers = self.containers(service_names)
  262. parallel.parallel_pause(reversed(containers), options)
  263. return containers
  264. def unpause(self, service_names=None, **options):
  265. containers = self.containers(service_names)
  266. parallel.parallel_unpause(containers, options)
  267. return containers
  268. def kill(self, service_names=None, **options):
  269. parallel.parallel_kill(self.containers(service_names), options)
  270. def remove_stopped(self, service_names=None, one_off=OneOffFilter.exclude, **options):
  271. parallel.parallel_remove(self.containers(
  272. service_names, stopped=True, one_off=one_off
  273. ), options)
  274. def down(
  275. self,
  276. remove_image_type,
  277. include_volumes,
  278. remove_orphans=False,
  279. timeout=None,
  280. ignore_orphans=False):
  281. self.stop(one_off=OneOffFilter.include, timeout=timeout)
  282. if not ignore_orphans:
  283. self.find_orphan_containers(remove_orphans)
  284. self.remove_stopped(v=include_volumes, one_off=OneOffFilter.include)
  285. self.networks.remove()
  286. if include_volumes:
  287. self.volumes.remove()
  288. self.remove_images(remove_image_type)
  289. def remove_images(self, remove_image_type):
  290. for service in self.get_services():
  291. service.remove_image(remove_image_type)
  292. def restart(self, service_names=None, **options):
  293. containers = self.containers(service_names, stopped=True)
  294. parallel.parallel_execute(
  295. containers,
  296. self.build_container_operation_with_timeout_func('restart', options),
  297. operator.attrgetter('name'),
  298. 'Restarting',
  299. )
  300. return containers
  301. def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None,
  302. build_args=None, gzip=False, parallel_build=False, rm=True, silent=False, cli=False,
  303. progress=None):
  304. services = []
  305. for service in self.get_services(service_names):
  306. if service.can_be_built():
  307. services.append(service)
  308. elif not silent:
  309. log.info('%s uses an image, skipping' % service.name)
  310. if cli:
  311. log.warning("Native build is an experimental feature and could change at any time")
  312. if parallel_build:
  313. log.warning("Flag '--parallel' is ignored when building with "
  314. "COMPOSE_DOCKER_CLI_BUILD=1")
  315. if gzip:
  316. log.warning("Flag '--compress' is ignored when building with "
  317. "COMPOSE_DOCKER_CLI_BUILD=1")
  318. def build_service(service):
  319. service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli, progress)
  320. if parallel_build:
  321. _, errors = parallel.parallel_execute(
  322. services,
  323. build_service,
  324. operator.attrgetter('name'),
  325. 'Building',
  326. limit=5,
  327. )
  328. if len(errors):
  329. combined_errors = '\n'.join([
  330. e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values()
  331. ])
  332. raise ProjectError(combined_errors)
  333. else:
  334. for service in services:
  335. build_service(service)
  336. def create(
  337. self,
  338. service_names=None,
  339. strategy=ConvergenceStrategy.changed,
  340. do_build=BuildAction.none,
  341. ):
  342. services = self.get_services_without_duplicate(service_names, include_deps=True)
  343. for svc in services:
  344. svc.ensure_image_exists(do_build=do_build)
  345. plans = self._get_convergence_plans(services, strategy)
  346. for service in services:
  347. service.execute_convergence_plan(
  348. plans[service.name],
  349. detached=True,
  350. start=False)
  351. def _legacy_event_processor(self, service_names):
  352. # Only for v1 files or when Compose is forced to use an older API version
  353. def build_container_event(event, container):
  354. time = datetime.datetime.fromtimestamp(event['time'])
  355. time = time.replace(
  356. microsecond=microseconds_from_time_nano(event['timeNano'])
  357. )
  358. return {
  359. 'time': time,
  360. 'type': 'container',
  361. 'action': event['status'],
  362. 'id': container.id,
  363. 'service': container.service,
  364. 'attributes': {
  365. 'name': container.name,
  366. 'image': event['from'],
  367. },
  368. 'container': container,
  369. }
  370. service_names = set(service_names or self.service_names)
  371. for event in self.client.events(
  372. filters={'label': self.labels()},
  373. decode=True
  374. ):
  375. # This is a guard against some events broadcasted by swarm that
  376. # don't have a status field.
  377. # See https://github.com/docker/compose/issues/3316
  378. if 'status' not in event:
  379. continue
  380. try:
  381. # this can fail if the container has been removed or if the event
  382. # refers to an image
  383. container = Container.from_id(self.client, event['id'])
  384. except APIError:
  385. continue
  386. if container.service not in service_names:
  387. continue
  388. yield build_container_event(event, container)
  389. def events(self, service_names=None):
  390. if version_lt(self.client.api_version, '1.22'):
  391. # New, better event API was introduced in 1.22.
  392. return self._legacy_event_processor(service_names)
  393. def build_container_event(event):
  394. container_attrs = event['Actor']['Attributes']
  395. time = datetime.datetime.fromtimestamp(event['time'])
  396. time = time.replace(
  397. microsecond=microseconds_from_time_nano(event['timeNano'])
  398. )
  399. container = None
  400. try:
  401. container = Container.from_id(self.client, event['id'])
  402. except APIError:
  403. # Container may have been removed (e.g. if this is a destroy event)
  404. pass
  405. return {
  406. 'time': time,
  407. 'type': 'container',
  408. 'action': event['status'],
  409. 'id': event['Actor']['ID'],
  410. 'service': container_attrs.get(LABEL_SERVICE),
  411. 'attributes': dict([
  412. (k, v) for k, v in container_attrs.items()
  413. if not k.startswith('com.docker.compose.')
  414. ]),
  415. 'container': container,
  416. }
  417. def yield_loop(service_names):
  418. for event in self.client.events(
  419. filters={'label': self.labels()},
  420. decode=True
  421. ):
  422. # TODO: support other event types
  423. if event.get('Type') != 'container':
  424. continue
  425. try:
  426. if event['Actor']['Attributes'][LABEL_SERVICE] not in service_names:
  427. continue
  428. except KeyError:
  429. continue
  430. yield build_container_event(event)
  431. return yield_loop(set(service_names) if service_names else self.service_names)
  432. def up(self,
  433. service_names=None,
  434. start_deps=True,
  435. strategy=ConvergenceStrategy.changed,
  436. do_build=BuildAction.none,
  437. timeout=None,
  438. detached=False,
  439. remove_orphans=False,
  440. ignore_orphans=False,
  441. scale_override=None,
  442. rescale=True,
  443. start=True,
  444. always_recreate_deps=False,
  445. reset_container_image=False,
  446. renew_anonymous_volumes=False,
  447. silent=False,
  448. cli=False,
  449. ):
  450. if cli:
  451. log.warning("Native build is an experimental feature and could change at any time")
  452. self.initialize()
  453. if not ignore_orphans:
  454. self.find_orphan_containers(remove_orphans)
  455. if scale_override is None:
  456. scale_override = {}
  457. services = self.get_services_without_duplicate(
  458. service_names,
  459. include_deps=start_deps)
  460. for svc in services:
  461. svc.ensure_image_exists(do_build=do_build, silent=silent, cli=cli)
  462. plans = self._get_convergence_plans(
  463. services, strategy, always_recreate_deps=always_recreate_deps)
  464. def do(service):
  465. return service.execute_convergence_plan(
  466. plans[service.name],
  467. timeout=timeout,
  468. detached=detached,
  469. scale_override=scale_override.get(service.name),
  470. rescale=rescale,
  471. start=start,
  472. reset_container_image=reset_container_image,
  473. renew_anonymous_volumes=renew_anonymous_volumes,
  474. )
  475. def get_deps(service):
  476. return {
  477. (self.get_service(dep), config)
  478. for dep, config in service.get_dependency_configs().items()
  479. }
  480. results, errors = parallel.parallel_execute(
  481. services,
  482. do,
  483. operator.attrgetter('name'),
  484. None,
  485. get_deps,
  486. )
  487. if errors:
  488. raise ProjectError(
  489. 'Encountered errors while bringing up the project.'
  490. )
  491. return [
  492. container
  493. for svc_containers in results
  494. if svc_containers is not None
  495. for container in svc_containers
  496. ]
  497. def initialize(self):
  498. self.networks.initialize()
  499. self.volumes.initialize()
  500. def _get_convergence_plans(self, services, strategy, always_recreate_deps=False):
  501. plans = {}
  502. for service in services:
  503. updated_dependencies = [
  504. name
  505. for name in service.get_dependency_names()
  506. if name in plans and
  507. plans[name].action in ('recreate', 'create')
  508. ]
  509. if updated_dependencies and strategy.allows_recreate:
  510. log.debug('%s has upstream changes (%s)',
  511. service.name,
  512. ", ".join(updated_dependencies))
  513. containers_stopped = any(
  514. service.containers(stopped=True, filters={'status': ['created', 'exited']}))
  515. service_has_links = any(service.get_link_names())
  516. container_has_links = any(c.get('HostConfig.Links') for c in service.containers())
  517. should_recreate_for_links = service_has_links ^ container_has_links
  518. if always_recreate_deps or containers_stopped or should_recreate_for_links:
  519. plan = service.convergence_plan(ConvergenceStrategy.always)
  520. else:
  521. plan = service.convergence_plan(strategy)
  522. else:
  523. plan = service.convergence_plan(strategy)
  524. plans[service.name] = plan
  525. return plans
  526. def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False,
  527. include_deps=False):
  528. services = self.get_services(service_names, include_deps)
  529. if parallel_pull:
  530. self.parallel_pull(services, silent=silent)
  531. else:
  532. must_build = []
  533. for service in services:
  534. try:
  535. service.pull(ignore_pull_failures, silent=silent)
  536. except (ImageNotFound, NotFound):
  537. if service.can_be_built():
  538. must_build.append(service.name)
  539. else:
  540. raise
  541. if len(must_build):
  542. log.warning('Some service image(s) must be built from source by running:\n'
  543. ' docker-compose build {}'
  544. .format(' '.join(must_build)))
  545. def parallel_pull(self, services, ignore_pull_failures=False, silent=False):
  546. msg = 'Pulling' if not silent else None
  547. must_build = []
  548. def pull_service(service):
  549. strm = service.pull(ignore_pull_failures, True, stream=True)
  550. if strm is None: # Attempting to pull service with no `image` key is a no-op
  551. return
  552. try:
  553. writer = parallel.get_stream_writer()
  554. for event in strm:
  555. if 'status' not in event:
  556. continue
  557. status = read_status(event)
  558. writer.write(
  559. msg, service.name, truncate_string(status), lambda s: s
  560. )
  561. except (ImageNotFound, NotFound):
  562. if service.can_be_built():
  563. must_build.append(service.name)
  564. else:
  565. raise
  566. _, errors = parallel.parallel_execute(
  567. services,
  568. pull_service,
  569. operator.attrgetter('name'),
  570. msg,
  571. limit=5,
  572. )
  573. if len(must_build):
  574. log.warning('Some service image(s) must be built from source by running:\n'
  575. ' docker-compose build {}'
  576. .format(' '.join(must_build)))
  577. if len(errors):
  578. combined_errors = '\n'.join([
  579. e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values()
  580. ])
  581. raise ProjectError(combined_errors)
  582. def push(self, service_names=None, ignore_push_failures=False):
  583. unique_images = set()
  584. for service in self.get_services(service_names, include_deps=False):
  585. # Considering <image> and <image:latest> as the same
  586. repo, tag, sep = parse_repository_tag(service.image_name)
  587. service_image_name = sep.join((repo, tag)) if tag else sep.join((repo, 'latest'))
  588. if service_image_name not in unique_images:
  589. service.push(ignore_push_failures)
  590. unique_images.add(service_image_name)
  591. def _labeled_containers(self, stopped=False, one_off=OneOffFilter.exclude):
  592. ctnrs = list(filter(None, [
  593. Container.from_ps(self.client, container)
  594. for container in self.client.containers(
  595. all=stopped,
  596. filters={'label': self.labels(one_off=one_off)})])
  597. )
  598. if ctnrs:
  599. return ctnrs
  600. return list(filter(lambda c: c.has_legacy_proj_name(self.name), filter(None, [
  601. Container.from_ps(self.client, container)
  602. for container in self.client.containers(
  603. all=stopped,
  604. filters={'label': self.labels(one_off=one_off, legacy=True)})])
  605. ))
  606. def containers(self, service_names=None, stopped=False, one_off=OneOffFilter.exclude):
  607. if service_names:
  608. self.validate_service_names(service_names)
  609. else:
  610. service_names = self.service_names
  611. containers = self._labeled_containers(stopped, one_off)
  612. def matches_service_names(container):
  613. return container.labels.get(LABEL_SERVICE) in service_names
  614. return [c for c in containers if matches_service_names(c)]
  615. def find_orphan_containers(self, remove_orphans):
  616. def _find():
  617. containers = set(self._labeled_containers() + self._labeled_containers(stopped=True))
  618. for ctnr in containers:
  619. service_name = ctnr.labels.get(LABEL_SERVICE)
  620. if service_name not in self.service_names:
  621. yield ctnr
  622. orphans = list(_find())
  623. if not orphans:
  624. return
  625. if remove_orphans:
  626. for ctnr in orphans:
  627. log.info('Removing orphan container "{0}"'.format(ctnr.name))
  628. try:
  629. ctnr.kill()
  630. except APIError:
  631. pass
  632. ctnr.remove(force=True)
  633. else:
  634. log.warning(
  635. 'Found orphan containers ({0}) for this project. If '
  636. 'you removed or renamed this service in your compose '
  637. 'file, you can run this command with the '
  638. '--remove-orphans flag to clean it up.'.format(
  639. ', '.join(["{}".format(ctnr.name) for ctnr in orphans])
  640. )
  641. )
  642. def _inject_deps(self, acc, service):
  643. dep_names = service.get_dependency_names()
  644. if len(dep_names) > 0:
  645. dep_services = self.get_services(
  646. service_names=list(set(dep_names)),
  647. include_deps=True
  648. )
  649. else:
  650. dep_services = []
  651. dep_services.append(service)
  652. return acc + dep_services
  653. def build_container_operation_with_timeout_func(self, operation, options):
  654. def container_operation_with_timeout(container):
  655. _options = options.copy()
  656. if _options.get('timeout') is None:
  657. service = self.get_service(container.service)
  658. _options['timeout'] = service.stop_timeout(None)
  659. return getattr(container, operation)(**_options)
  660. return container_operation_with_timeout
  661. def get_volumes_from(project, service_dict):
  662. volumes_from = service_dict.pop('volumes_from', None)
  663. if not volumes_from:
  664. return []
  665. def build_volume_from(spec):
  666. if spec.type == 'service':
  667. try:
  668. return spec._replace(source=project.get_service(spec.source))
  669. except NoSuchService:
  670. pass
  671. if spec.type == 'container':
  672. try:
  673. container = Container.from_id(project.client, spec.source)
  674. return spec._replace(source=container)
  675. except APIError:
  676. pass
  677. raise ConfigurationError(
  678. "Service \"{}\" mounts volumes from \"{}\", which is not the name "
  679. "of a service or container.".format(
  680. service_dict['name'],
  681. spec.source))
  682. return [build_volume_from(vf) for vf in volumes_from]
  683. def get_secrets(service, service_secrets, secret_defs):
  684. secrets = []
  685. for secret in service_secrets:
  686. secret_def = secret_defs.get(secret.source)
  687. if not secret_def:
  688. raise ConfigurationError(
  689. "Service \"{service}\" uses an undefined secret \"{secret}\" "
  690. .format(service=service, secret=secret.source))
  691. if secret_def.get('external'):
  692. log.warning("Service \"{service}\" uses secret \"{secret}\" which is external. "
  693. "External secrets are not available to containers created by "
  694. "docker-compose.".format(service=service, secret=secret.source))
  695. continue
  696. if secret.uid or secret.gid or secret.mode:
  697. log.warning(
  698. "Service \"{service}\" uses secret \"{secret}\" with uid, "
  699. "gid, or mode. These fields are not supported by this "
  700. "implementation of the Compose file".format(
  701. service=service, secret=secret.source
  702. )
  703. )
  704. secret_file = secret_def.get('file')
  705. if not path.isfile(str(secret_file)):
  706. log.warning(
  707. "Service \"{service}\" uses an undefined secret file \"{secret_file}\", "
  708. "the following file should be created \"{secret_file}\"".format(
  709. service=service, secret_file=secret_file
  710. )
  711. )
  712. secrets.append({'secret': secret, 'file': secret_file})
  713. return secrets
  714. def get_image_digests(project):
  715. digests = {}
  716. needs_push = set()
  717. needs_pull = set()
  718. for service in project.services:
  719. try:
  720. digests[service.name] = get_image_digest(service)
  721. except NeedsPush as e:
  722. needs_push.add(e.image_name)
  723. except NeedsPull as e:
  724. needs_pull.add(e.service_name)
  725. if needs_push or needs_pull:
  726. raise MissingDigests(needs_push, needs_pull)
  727. return digests
  728. def get_image_digest(service):
  729. if 'image' not in service.options:
  730. raise UserError(
  731. "Service '{s.name}' doesn't define an image tag. An image name is "
  732. "required to generate a proper image digest. Specify an image repo "
  733. "and tag with the 'image' option.".format(s=service))
  734. _, _, separator = parse_repository_tag(service.options['image'])
  735. # Compose file already uses a digest, no lookup required
  736. if separator == '@':
  737. return service.options['image']
  738. digest = get_digest(service)
  739. if digest:
  740. return digest
  741. if 'build' not in service.options:
  742. raise NeedsPull(service.image_name, service.name)
  743. raise NeedsPush(service.image_name)
  744. def get_digest(service):
  745. digest = None
  746. try:
  747. image = service.image()
  748. # TODO: pick a digest based on the image tag if there are multiple
  749. # digests
  750. if image['RepoDigests']:
  751. digest = image['RepoDigests'][0]
  752. except NoSuchImageError:
  753. try:
  754. # Fetch the image digest from the registry
  755. distribution = service.get_image_registry_data()
  756. if distribution['Descriptor']['digest']:
  757. digest = '{image_name}@{digest}'.format(
  758. image_name=service.image_name,
  759. digest=distribution['Descriptor']['digest']
  760. )
  761. except NoSuchImageError:
  762. raise UserError(
  763. "Digest not found for service '{service}'. "
  764. "Repository does not exist or may require 'docker login'"
  765. .format(service=service.name))
  766. return digest
  767. class MissingDigests(Exception):
  768. def __init__(self, needs_push, needs_pull):
  769. self.needs_push = needs_push
  770. self.needs_pull = needs_pull
  771. class NeedsPush(Exception):
  772. def __init__(self, image_name):
  773. self.image_name = image_name
  774. class NeedsPull(Exception):
  775. def __init__(self, image_name, service_name):
  776. self.image_name = image_name
  777. self.service_name = service_name
  778. class NoSuchService(Exception):
  779. def __init__(self, name):
  780. if isinstance(name, six.binary_type):
  781. name = name.decode('utf-8')
  782. self.name = name
  783. self.msg = "No such service: %s" % self.name
  784. def __str__(self):
  785. return self.msg
  786. class ProjectError(Exception):
  787. def __init__(self, msg):
  788. self.msg = msg