Create volume for compose app

Signed-off-by: Nicolas De Loof <[email protected]>

ecs/aws.go

@@ -68,9 +68,12 @@ type API interface {
 	GetPublicIPs(ctx context.Context, interfaces ...string) (map[string]string, error)
 	LoadBalancerType(ctx context.Context, arn string) (string, error)
 	GetLoadBalancerURL(ctx context.Context, arn string) (string, error)
-	WithVolumeSecurityGroups(ctx context.Context, id string, fn func(securityGroups []string) error) error
 	GetParameter(ctx context.Context, name string) (string, error)
 	SecurityGroupExists(ctx context.Context, sg string) (bool, error)
 	DeleteCapacityProvider(ctx context.Context, arn string) error
 	DeleteAutoscalingGroup(ctx context.Context, arn string) error
+	FileSystemExists(ctx context.Context, id string) (bool, error)
+	FindFileSystem(ctx context.Context, tags map[string]string) (string, error)
+	CreateFileSystem(ctx context.Context, tags map[string]string) (string, error)
+	DeleteFileSystem(ctx context.Context, id string) error
 }

ecs/awsResources.go

@@ -20,13 +20,17 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/aws/aws-sdk-go/service/elbv2"
-	"github.com/awslabs/goformation/v4/cloudformation/ec2"
-	"github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2"
+	"github.com/docker/compose-cli/api/compose"
 
+	"github.com/docker/compose-cli/errdefs"
+
+	"github.com/aws/aws-sdk-go/service/elbv2"
 	"github.com/awslabs/goformation/v4/cloudformation"
+	"github.com/awslabs/goformation/v4/cloudformation/ec2"
 	"github.com/awslabs/goformation/v4/cloudformation/ecs"
+	"github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2"
 	"github.com/compose-spec/compose-go/types"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -77,6 +81,10 @@ func (b *ecsAPIService) parse(ctx context.Context, project *types.Project) (awsR
 	if err != nil {
 		return r, err
 	}
+	r.filesystems, err = b.parseExternalVolumes(ctx, project)
+	if err != nil {
+		return r, err
+	}
 	return r, nil
 }
 
@@ -88,7 +96,7 @@ func (b *ecsAPIService) parseClusterExtension(ctx context.Context, project *type
 			return "", err
 		}
 		if !ok {
-			return "", fmt.Errorf("cluster does not exist: %s", cluster)
+			return "", errors.Wrapf(errdefs.ErrNotFound, "cluster %q does not exist", cluster)
 		}
 		return cluster, nil
 	}
@@ -143,42 +151,62 @@ func (b *ecsAPIService) parseLoadBalancerExtension(ctx context.Context, project
 func (b *ecsAPIService) parseExternalNetworks(ctx context.Context, project *types.Project) (map[string]string, error) {
 	securityGroups := make(map[string]string, len(project.Networks))
 	for name, net := range project.Networks {
-		if !net.External.External {
-			continue
-		}
-		sg := net.Name
+		// FIXME remove this for G.A
 		if x, ok := net.Extensions[extensionSecurityGroup]; ok {
 			logrus.Warn("to use an existing security-group, use `network.external` and `network.name` in your compose file")
 			logrus.Debugf("Security Group for network %q set by user to %q", net.Name, x)
-			sg = x.(string)
+			net.External.External = true
+			net.Name = x.(string)
+			project.Networks[name] = net
+		}
+
+		if !net.External.External {
+			continue
 		}
-		exists, err := b.aws.SecurityGroupExists(ctx, sg)
+		exists, err := b.aws.SecurityGroupExists(ctx, net.Name)
 		if err != nil {
 			return nil, err
 		}
 		if !exists {
-			return nil, fmt.Errorf("security group %s doesn't exist", sg)
+			return nil, errors.Wrapf(errdefs.ErrNotFound, "security group %q doesn't exist", net.Name)
 		}
-		securityGroups[name] = sg
+		securityGroups[name] = net.Name
 	}
 	return securityGroups, nil
 }
 
 func (b *ecsAPIService) parseExternalVolumes(ctx context.Context, project *types.Project) (map[string]string, error) {
 	filesystems := make(map[string]string, len(project.Volumes))
-	// project.Volumes.filter(|v| v.External.External).first(|v| b.SDK.FileSystemExists(ctx, vol.Name))?
 	for name, vol := range project.Volumes {
-		if !vol.External.External {
+		if vol.External.External {
+			exists, err := b.aws.FileSystemExists(ctx, vol.Name)
+			if err != nil {
+				return nil, err
+			}
+			if !exists {
+				return nil, errors.Wrapf(errdefs.ErrNotFound, "EFS file system %q doesn't exist", vol.Name)
+			}
+			filesystems[name] = vol.Name
 			continue
 		}
-		exists, err := b.SDK.FileSystemExists(ctx, vol.Name)
+
+		logrus.Debugf("searching for existing filesystem as volume %q", name)
+		tags := map[string]string{
+			compose.ProjectTag: project.Name,
+			compose.VolumeTag:  name,
+		}
+		id, err := b.aws.FindFileSystem(ctx, tags)
 		if err != nil {
 			return nil, err
 		}
-		if !exists {
-			return nil, fmt.Errorf("EFS file system %s doesn't exist", vol.Name)
+		if id == "" {
+			logrus.Debug("no EFS filesystem found, create a fresh new one")
+			id, err = b.aws.CreateFileSystem(ctx, tags)
+			if err != nil {
+				return nil, err
+			}
 		}
-		filesystems[name] = vol.Name
+		filesystems[name] = id
 	}
 	return filesystems, nil
 }
@@ -206,11 +234,9 @@ func (b *ecsAPIService) ensureNetworks(r *awsResources, project *types.Project,
 		r.securityGroups = make(map[string]string, len(project.Networks))
 	}
 	for name, net := range project.Networks {
-		if net.External.External {
-			r.securityGroups[name] = net.Name
+		if _, ok := r.securityGroups[name]; ok {
 			continue
 		}
-
 		securityGroup := networkResourceName(name)
 		template.Resources[securityGroup] = &ec2.SecurityGroup{
 			GroupDescription: fmt.Sprintf("%s Security Group for %s network", project.Name, name),
@@ -230,12 +256,6 @@ func (b *ecsAPIService) ensureNetworks(r *awsResources, project *types.Project,
 	}
 }
 
-func (b *ecsAPIService) ensureVolumes(r *awsResources, project *types.Project, template *cloudformation.Template) {
-	if r.filesystems == nil {
-		r.filesystems = make(map[string]string, len(project.Volumes))
-	}
-}
-
 func (b *ecsAPIService) ensureLoadBalancer(r *awsResources, project *types.Project, template *cloudformation.Template) {
 	if r.loadBalancer != "" {
 		return

ecs/aws_mock.go

@@ -110,6 +110,21 @@ func (mr *MockAPIMockRecorder) CreateCluster(arg0, arg1 interface{}) *gomock.Cal
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateCluster", reflect.TypeOf((*MockAPI)(nil).CreateCluster), arg0, arg1)
 }
 
+// CreateFileSystem mocks base method
+func (m *MockAPI) CreateFileSystem(arg0 context.Context, arg1 map[string]string) (string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CreateFileSystem", arg0, arg1)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// CreateFileSystem indicates an expected call of CreateFileSystem
+func (mr *MockAPIMockRecorder) CreateFileSystem(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFileSystem", reflect.TypeOf((*MockAPI)(nil).CreateFileSystem), arg0, arg1)
+}
+
 // CreateSecret mocks base method
 func (m *MockAPI) CreateSecret(arg0 context.Context, arg1 secrets.Secret) (string, error) {
 	m.ctrl.T.Helper()
@@ -167,6 +182,20 @@ func (mr *MockAPIMockRecorder) DeleteCapacityProvider(arg0, arg1 interface{}) *g
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCapacityProvider", reflect.TypeOf((*MockAPI)(nil).DeleteCapacityProvider), arg0, arg1)
 }
 
+// DeleteFileSystem mocks base method
+func (m *MockAPI) DeleteFileSystem(arg0 context.Context, arg1 string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DeleteFileSystem", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DeleteFileSystem indicates an expected call of DeleteFileSystem
+func (mr *MockAPIMockRecorder) DeleteFileSystem(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteFileSystem", reflect.TypeOf((*MockAPI)(nil).DeleteFileSystem), arg0, arg1)
+}
+
 // DeleteSecret mocks base method
 func (m *MockAPI) DeleteSecret(arg0 context.Context, arg1 string, arg2 bool) error {
 	m.ctrl.T.Helper()
@@ -225,6 +254,36 @@ func (mr *MockAPIMockRecorder) DescribeStackEvents(arg0, arg1 interface{}) *gomo
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeStackEvents", reflect.TypeOf((*MockAPI)(nil).DescribeStackEvents), arg0, arg1)
 }
 
+// FileSystemExists mocks base method
+func (m *MockAPI) FileSystemExists(arg0 context.Context, arg1 string) (bool, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "FileSystemExists", arg0, arg1)
+	ret0, _ := ret[0].(bool)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// FileSystemExists indicates an expected call of FileSystemExists
+func (mr *MockAPIMockRecorder) FileSystemExists(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileSystemExists", reflect.TypeOf((*MockAPI)(nil).FileSystemExists), arg0, arg1)
+}
+
+// FindFileSystem mocks base method
+func (m *MockAPI) FindFileSystem(arg0 context.Context, arg1 map[string]string) (string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "FindFileSystem", arg0, arg1)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// FindFileSystem indicates an expected call of FindFileSystem
+func (mr *MockAPIMockRecorder) FindFileSystem(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindFileSystem", reflect.TypeOf((*MockAPI)(nil).FindFileSystem), arg0, arg1)
+}
+
 // GetDefaultVPC mocks base method
 func (m *MockAPI) GetDefaultVPC(arg0 context.Context) (string, error) {
 	m.ctrl.T.Helper()
@@ -587,20 +646,6 @@ func (mr *MockAPIMockRecorder) WaitStackComplete(arg0, arg1, arg2 interface{}) *
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitStackComplete", reflect.TypeOf((*MockAPI)(nil).WaitStackComplete), arg0, arg1, arg2)
 }
 
-// WithVolumeSecurityGroups mocks base method
-func (m *MockAPI) WithVolumeSecurityGroups(arg0 context.Context, arg1 string, arg2 func([]string) error) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "WithVolumeSecurityGroups", arg0, arg1, arg2)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// WithVolumeSecurityGroups indicates an expected call of WithVolumeSecurityGroups
-func (mr *MockAPIMockRecorder) WithVolumeSecurityGroups(arg0, arg1, arg2 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithVolumeSecurityGroups", reflect.TypeOf((*MockAPI)(nil).WithVolumeSecurityGroups), arg0, arg1, arg2)
-}
-
 // getURLWithPortMapping mocks base method
 func (m *MockAPI) getURLWithPortMapping(arg0 context.Context, arg1 []string) ([]compose.PortPublisher, error) {
 	m.ctrl.T.Helper()

ecs/cloudformation.go

@@ -72,6 +72,8 @@ func (b *ecsAPIService) convert(ctx context.Context, project *types.Project) (*c
 	// Private DNS namespace will allow DNS name for the services to be <service>.<project>.local
 	b.createCloudMap(project, template, resources.vpc)
 
+	b.createNFSMountTarget(project, resources, template)
+
 	for _, service := range project.Services {
 		err := b.createService(project, service, template, resources)
 		if err != nil {
@@ -81,17 +83,6 @@ func (b *ecsAPIService) convert(ctx context.Context, project *types.Project) (*c
 		b.createAutoscalingPolicy(project, resources, template, service)
 	}
 
-	// Create a NFS inbound rule on each mount target for volumes
-	// as "source security group" use an arbitrary network attached to service(s) who mounts target volume
-	for n, vol := range project.Volumes {
-		err := b.aws.WithVolumeSecurityGroups(ctx, vol.Name, func(securityGroups []string) error {
-			return b.createNFSmountIngress(securityGroups, project, n, template)
-		})
-		if err != nil {
-			return nil, err
-		}
-	}
-
 	err = b.createCapacityProvider(ctx, project, template, resources)
 	if err != nil {
 		return nil, err
@@ -104,7 +95,7 @@ func (b *ecsAPIService) createService(project *types.Project, service types.Serv
 	taskExecutionRole := b.createTaskExecutionRole(project, service, template)
 	taskRole := b.createTaskRole(project, service, template)
 
-	definition, err := b.createTaskDefinition(project, service)
+	definition, err := b.createTaskDefinition(project, service, resources)
 	if err != nil {
 		return err
 	}
@@ -152,6 +143,10 @@ func (b *ecsAPIService) createService(project *types.Project, service types.Serv
 		dependsOn = append(dependsOn, serviceResourceName(dependency))
 	}
 
+	for _, s := range service.Volumes {
+		dependsOn = append(dependsOn, b.mountTargets(s.Source, resources)...)
+	}
+
 	minPercent, maxPercent, err := computeRollingUpdateLimits(service)
 	if err != nil {
 		return err
@@ -326,12 +321,11 @@ func (b *ecsAPIService) createTargetGroup(project *types.Project, service types.
 		port.Published,
 	)
 	template.Resources[targetGroupName] = &elasticloadbalancingv2.TargetGroup{
-		HealthCheckEnabled: false,
-		Port:               int(port.Target),
-		Protocol:           protocol,
-		Tags:               projectTags(project),
-		TargetType:         elbv2.TargetTypeEnumIp,
-		VpcId:              vpc,
+		Port:       int(port.Target),
+		Protocol:   protocol,
+		Tags:       projectTags(project),
+		TargetType: elbv2.TargetTypeEnumIp,
+		VpcId:      vpc,
 	}
 	return targetGroupName
 }

ecs/cloudformation_test.go

@@ -23,6 +23,8 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/awslabs/goformation/v4/cloudformation/efs"
+
 	"github.com/golang/mock/gomock"
 
 	"github.com/docker/compose-cli/api/compose"
@@ -336,7 +338,7 @@ services:
 	assert.Check(t, loadBalancer.Type == elbv2.LoadBalancerTypeEnumNetwork)
 }
 
-func TestUseCustomNetwork(t *testing.T) {
+func TestUseExternalNetwork(t *testing.T) {
 	template := convertYaml(t, `
 services:
   test:
@@ -355,6 +357,65 @@ networks:
 	assert.Check(t, s.NetworkConfiguration.AwsvpcConfiguration.SecurityGroups[0] == "sg-123abc") //nolint:staticcheck
 }
 
+func testVolume(t *testing.T, yaml string, fn ...func(m *MockAPIMockRecorder)) {
+	template := convertYaml(t, yaml, fn...)
+
+	s := template.Resources["DbdataNFSMountTargetOnSubnet1"].(*efs.MountTarget)
+	assert.Check(t, s != nil)
+	assert.Equal(t, s.FileSystemId, "fs-123abc") //nolint:staticcheck
+
+	s = template.Resources["DbdataNFSMountTargetOnSubnet2"].(*efs.MountTarget)
+	assert.Check(t, s != nil)
+	assert.Equal(t, s.FileSystemId, "fs-123abc") //nolint:staticcheck
+}
+
+func TestUseExternalVolume(t *testing.T) {
+	testVolume(t, `
+services:
+  test:
+    image: nginx
+volumes:
+  db-data:
+    external: true
+    name: fs-123abc
+`, useDefaultVPC, func(m *MockAPIMockRecorder) {
+		m.FileSystemExists(gomock.Any(), "fs-123abc").Return(true, nil)
+	})
+}
+
+func TestCreateVolume(t *testing.T) {
+	tags := map[string]string{
+		compose.ProjectTag: t.Name(),
+		compose.VolumeTag:  "db-data",
+	}
+	testVolume(t, `
+services:
+  test:
+    image: nginx
+volumes:
+  db-data: {}
+`, useDefaultVPC, func(m *MockAPIMockRecorder) {
+		m.FindFileSystem(gomock.Any(), tags).Return("", nil)
+		m.CreateFileSystem(gomock.Any(), tags).Return("fs-123abc", nil)
+	})
+}
+
+func TestReusePreviousVolume(t *testing.T) {
+	tags := map[string]string{
+		compose.ProjectTag: t.Name(),
+		compose.VolumeTag:  "db-data",
+	}
+	testVolume(t, `
+services:
+  test:
+    image: nginx
+volumes:
+  db-data: {}
+`, useDefaultVPC, func(m *MockAPIMockRecorder) {
+		m.FindFileSystem(gomock.Any(), tags).Return("fs-123abc", nil)
+	})
+}
+
 func TestServiceMapping(t *testing.T) {
 	template := convertYaml(t, `
 services:

ecs/compatibility.go

@@ -97,7 +97,9 @@ var compatibleComposeAttributes = []string{
 	"secrets.file",
 	"volumes",
 	"volumes.external",
+	"volumes.name",
 	"networks.external",
+	"networks.name",
 }
 
 func (c *fargateCompatibilityChecker) CheckImage(service *types.ServiceConfig) {
@@ -133,9 +135,3 @@ func (c *fargateCompatibilityChecker) CheckLoggingDriver(config *types.LoggingCo
 		c.Unsupported("services.logging.driver %s is not supported", config.Driver)
 	}
 }
-
-func (c *fargateCompatibilityChecker) CheckVolumeConfigExternal(config *types.VolumeConfig) {
-	if !config.External.External {
-		c.Unsupported("non-external volumes are not supported")
-	}
-}

ecs/convert.go

@@ -39,7 +39,7 @@ import (
 const secretsInitContainerImage = "docker/ecs-secrets-sidecar"
 const searchDomainInitContainerImage = "docker/ecs-searchdomain-sidecar"
 
-func (b *ecsAPIService) createTaskDefinition(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefinition, error) {
+func (b *ecsAPIService) createTaskDefinition(project *types.Project, service types.ServiceConfig, resources awsResources) (*ecs.TaskDefinition, error) {
 	cpu, mem, err := toLimits(service)
 	if err != nil {
 		return nil, err
@@ -81,11 +81,11 @@ func (b *ecsAPIService) createTaskDefinition(project *types.Project, service typ
 	}
 
 	for _, v := range service.Volumes {
-		source := project.Volumes[v.Source]
+		volume := project.Volumes[v.Source]
 		volumes = append(volumes, ecs.TaskDefinition_Volume{
 			EFSVolumeConfiguration: &ecs.TaskDefinition_EFSVolumeConfiguration{
-				FilesystemId:  source.Name,
-				RootDirectory: source.DriverOpts["root_directory"],
+				FilesystemId:  resources.filesystems[v.Source],
+				RootDirectory: volume.DriverOpts["root_directory"],
 			},
 			Name: v.Source,
 		})
@@ -100,7 +100,6 @@ func (b *ecsAPIService) createTaskDefinition(project *types.Project, service typ
 	if err != nil {
 		return nil, err
 	}
-
 	var reservations *types.Resource
 	if service.Deploy != nil && service.Deploy.Resources.Reservations != nil {
 		reservations = service.Deploy.Resources.Reservations

ecs/sdk.go

@@ -787,28 +787,6 @@ func (s sdk) GetLoadBalancerURL(ctx context.Context, arn string) (string, error)
 	return dnsName, nil
 }
 
-func (s sdk) WithVolumeSecurityGroups(ctx context.Context, id string, fn func(securityGroups []string) error) error {
-	mounts, err := s.EFS.DescribeMountTargetsWithContext(ctx, &efs.DescribeMountTargetsInput{
-		FileSystemId: aws.String(id),
-	})
-	if err != nil {
-		return err
-	}
-	for _, mount := range mounts.MountTargets {
-		groups, err := s.EFS.DescribeMountTargetSecurityGroupsWithContext(ctx, &efs.DescribeMountTargetSecurityGroupsInput{
-			MountTargetId: mount.MountTargetId,
-		})
-		if err != nil {
-			return err
-		}
-		err = fn(aws.StringValueSlice(groups.SecurityGroups))
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func (s sdk) GetParameter(ctx context.Context, name string) (string, error) {
 	parameter, err := s.SSM.GetParameterWithContext(ctx, &ssm.GetParameterInput{
 		Name: aws.String(name),
@@ -869,19 +847,58 @@ func (s sdk) FileSystemExists(ctx context.Context, id string) (bool, error) {
 	return len(desc.FileSystems) > 0, nil
 }
 
-func (s sdk) CreateFileSystem(ctx context.Context, name string) (string, error) {
+func (s sdk) FindFileSystem(ctx context.Context, tags map[string]string) (string, error) {
+	var token *string
+	for {
+		desc, err := s.EFS.DescribeFileSystemsWithContext(ctx, &efs.DescribeFileSystemsInput{
+			Marker: token,
+		})
+		if err != nil {
+			return "", err
+		}
+		for _, filesystem := range desc.FileSystems {
+			if containsAll(filesystem.Tags, tags) {
+				return aws.StringValue(filesystem.FileSystemId), nil
+			}
+		}
+		if desc.NextMarker == token {
+			return "", nil
+		}
+		token = desc.NextMarker
+	}
+}
+
+func containsAll(tags []*efs.Tag, required map[string]string) bool {
+TAGS:
+	for key, value := range required {
+		for _, t := range tags {
+			if aws.StringValue(t.Key) == key && aws.StringValue(t.Value) == value {
+				continue TAGS
+			}
+		}
+		return false
+	}
+	return true
+}
+
+func (s sdk) CreateFileSystem(ctx context.Context, tags map[string]string) (string, error) {
+	var efsTags []*efs.Tag
+	for k, v := range tags {
+		efsTags = append(efsTags, &efs.Tag{
+			Key:   aws.String(k),
+			Value: aws.String(v),
+		})
+	}
 	res, err := s.EFS.CreateFileSystemWithContext(ctx, &efs.CreateFileSystemInput{
-		Tags: []*efs.Tag{
-			{
-				Key:   aws.String(compose.VolumeTag),
-				Value: aws.String(name),
-			},
-		},
+		Encrypted: aws.Bool(true),
+		Tags:      efsTags,
 	})
 	if err != nil {
 		return "", err
 	}
-	return aws.StringValue(res.FileSystemId), nil
+	id := aws.StringValue(res.FileSystemId)
+	logrus.Debugf("Created file system %q", id)
+	return id, nil
 }
 
 func (s sdk) DeleteFileSystem(ctx context.Context, id string) error {

ecs/volumes.go

@@ -20,40 +20,27 @@ import (
 	"fmt"
 
 	"github.com/awslabs/goformation/v4/cloudformation"
-	"github.com/awslabs/goformation/v4/cloudformation/ec2"
-	"github.com/awslabs/goformation/v4/cloudformation/ecs"
+	"github.com/awslabs/goformation/v4/cloudformation/efs"
 	"github.com/compose-spec/compose-go/types"
 )
 
-func (b *ecsAPIService) createNFSmountIngress(securityGroups []string, project *types.Project, n string, template *cloudformation.Template) error {
-	target := securityGroups[0]
-	for _, s := range project.Services {
-		for _, v := range s.Volumes {
-			if v.Source != n {
-				continue
+func (b *ecsAPIService) createNFSMountTarget(project *types.Project, resources awsResources, template *cloudformation.Template) {
+	for volume := range project.Volumes {
+		for _, subnet := range resources.subnets {
+			name := fmt.Sprintf("%sNFSMountTargetOn%s", normalizeResourceName(volume), normalizeResourceName(subnet))
+			template.Resources[name] = &efs.MountTarget{
+				FileSystemId:   resources.filesystems[volume],
+				SecurityGroups: resources.allSecurityGroups(),
+				SubnetId:       subnet,
 			}
-			var source string
-			for net := range s.Networks {
-				network := project.Networks[net]
-				if ext, ok := network.Extensions[extensionSecurityGroup]; ok {
-					source = ext.(string)
-				} else {
-					source = networkResourceName(net)
-				}
-				break
-			}
-			name := fmt.Sprintf("%sNFSMount%s", normalizeResourceName(s.Name), normalizeResourceName(n))
-			template.Resources[name] = &ec2.SecurityGroupIngress{
-				Description:           fmt.Sprintf("Allow NFS mount for %s on %s", s.Name, n),
-				GroupId:               target,
-				SourceSecurityGroupId: cloudformation.Ref(source),
-				IpProtocol:            "tcp",
-				FromPort:              2049,
-				ToPort:                2049,
-			}
-			service := template.Resources[serviceResourceName(s.Name)].(*ecs.Service)
-			service.AWSCloudFormationDependsOn = append(service.AWSCloudFormationDependsOn, name)
 		}
 	}
-	return nil
+}
+
+func (b *ecsAPIService) mountTargets(volume string, resources awsResources) []string {
+	var refs []string
+	for _, subnet := range resources.subnets {
+		refs = append(refs, fmt.Sprintf("%sNFSMountTargetOn%s", normalizeResourceName(volume), normalizeResourceName(subnet)))
+	}
+	return refs
 }

go.mod

@@ -6,8 +6,6 @@ go 1.15
 // we need to create a new release tag for docker/distribution
 replace github.com/docker/distribution => github.com/docker/distribution v0.0.0-20200708230824-53e18a9d9bfe
 
-replace github.com/awslabs/goformation/v4 => github.com/ndeloof/goformation/v4 v4.8.1-0.20200827081523-b7a7ac375adf
-
 require (
 	github.com/AlecAivazis/survey/v2 v2.1.1
 	github.com/Azure/azure-sdk-for-go v43.3.0+incompatible
@@ -21,10 +19,10 @@ require (
 	github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
 	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5
 	github.com/Microsoft/hcsshim v0.8.9 // indirect
-	github.com/aws/aws-sdk-go v1.34.8
-	github.com/awslabs/goformation/v4 v4.14.0
+	github.com/aws/aws-sdk-go v1.35.7
+	github.com/awslabs/goformation/v4 v4.15.2
 	github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129
-	github.com/compose-spec/compose-go v0.0.0-20200907084823-057e1edc5b6f
+	github.com/compose-spec/compose-go v0.0.0-20201005072614-3b6106793209
 	github.com/containerd/console v1.0.0
 	github.com/containerd/containerd v1.3.5 // indirect
 	github.com/docker/cli v0.0.0-20200528204125-dd360c7c0de8
@@ -53,7 +51,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/tsdb v0.7.1
 	github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b
-	github.com/sirupsen/logrus v1.6.0
+	github.com/sirupsen/logrus v1.7.0
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
@@ -63,6 +61,7 @@ require (
 	golang.org/x/net v0.0.0-20200822124328-c89045814202
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
 	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
+	golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 // indirect
 	google.golang.org/grpc v1.32.0
 	google.golang.org/protobuf v1.25.0
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect

go.sum

@@ -96,8 +96,10 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
-github.com/aws/aws-sdk-go v1.34.8 h1:GDfVeXG8XQDbpOeAj7415F8qCQZwvY/k/fj+HBqUnBA=
-github.com/aws/aws-sdk-go v1.34.8/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.35.7 h1:FHMhVhyc/9jljgFAcGkQDYjpC9btM0B8VfkLBfctdNE=
+github.com/aws/aws-sdk-go v1.35.7/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
+github.com/awslabs/goformation/v4 v4.15.2 h1:sRfSdC1FnSBhsrz5G0XZZxapEtmJSlkNpnFQJf8ylfs=
+github.com/awslabs/goformation/v4 v4.15.2/go.mod h1:GcJULxCJfloT+3pbqCluXftdEK2AD/UqpS3hkaaBntg=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -120,8 +122,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/compose-spec/compose-go v0.0.0-20200907084823-057e1edc5b6f h1:YsU3/17YA/skXpCQbRcrzWJxslWZ2lmvQK0bRiCyC38=
-github.com/compose-spec/compose-go v0.0.0-20200907084823-057e1edc5b6f/go.mod h1:voTGL1mRFcKRaFbi1lXGlR1YffS/9YD1jnVl4N/rYzw=
+github.com/compose-spec/compose-go v0.0.0-20201005072614-3b6106793209 h1:PLZiS7hjkiAqZYBRAEq3tbGlhCh6/R14dO1ahwbEIBg=
+github.com/compose-spec/compose-go v0.0.0-20201005072614-3b6106793209/go.mod h1:rNXXqhdClEljsNb6QDIOqTQaRfigwTgGZZM6Zpr3LeY=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmYntojat8lljbt1mgKNpTxUZJsSzJ9Y1s=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
@@ -193,8 +195,6 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
-github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
@@ -296,8 +296,10 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
-github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -314,8 +316,6 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -364,8 +364,6 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
-github.com/ndeloof/goformation/v4 v4.8.1-0.20200827081523-b7a7ac375adf h1:jdmD8L6TKRZpa7B4qUmjiWRBMkgbfUF/7pi/Kgba5lA=
-github.com/ndeloof/goformation/v4 v4.8.1-0.20200827081523-b7a7ac375adf/go.mod h1:GcJULxCJfloT+3pbqCluXftdEK2AD/UqpS3hkaaBntg=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
@@ -429,8 +427,8 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
-github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
@@ -459,7 +457,6 @@ github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -616,6 +613,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -633,6 +631,8 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 h1:bNEHhJCnrwMKNMmOx3yAynp5vs5/gRy+XWFtZFu7NBM=
+golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=