Merge pull request #622 from docker/efs

Introduce support for external EFS volumes

ecs/cloudformation.go

@@ -56,6 +56,47 @@ func (b *ecsAPIService) Convert(ctx context.Context, project *types.Project) ([]
 	if err != nil {
 		return nil, err
 	}
+
+	// Create a NFS inbound rule on each mount target for volumes
+	// as "source security group" use an arbitrary network attached to service(s) who mounts target volume
+	for n, vol := range project.Volumes {
+		err := b.SDK.WithVolumeSecurityGroups(ctx, vol.Name, func(securityGroups []string) error {
+			target := securityGroups[0]
+			for _, s := range project.Services {
+				for _, v := range s.Volumes {
+					if v.Source != n {
+						continue
+					}
+					var source string
+					for net := range s.Networks {
+						network := project.Networks[net]
+						if ext, ok := network.Extensions[extensionSecurityGroup]; ok {
+							source = ext.(string)
+						} else {
+							source = networkResourceName(project, net)
+						}
+						break
+					}
+					name := fmt.Sprintf("%sNFSMount%s", s.Name, n)
+					template.Resources[name] = &ec2.SecurityGroupIngress{
+						Description:           fmt.Sprintf("Allow NFS mount for %s on %s", s.Name, n),
+						GroupId:               target,
+						SourceSecurityGroupId: cloudformation.Ref(source),
+						IpProtocol:            "tcp",
+						FromPort:              2049,
+						ToPort:                2049,
+					}
+					service := template.Resources[serviceResourceName(s.Name)].(*ecs.Service)
+					service.AWSCloudFormationDependsOn = append(service.AWSCloudFormationDependsOn, name)
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return marshall(template)
 }
 
@@ -111,7 +152,7 @@ func (b *ecsAPIService) convert(project *types.Project) (*cloudformation.Templat
 		Description: "Name of the LoadBalancer to connect to (optional)",
 	}
 
-	// Create Cluster is `ParameterClusterName` parameter is not set
+	// Createmount.nfs4: Connection timed out : unsuccessful EFS utils command execution; code: 32 Cluster is `ParameterClusterName` parameter is not set
 	template.Conditions["CreateCluster"] = cloudformation.Equals("", cloudformation.Ref(parameterClusterName))
 
 	cluster := createCluster(project, template)
@@ -240,6 +281,7 @@ func (b *ecsAPIService) convert(project *types.Project) (*cloudformation.Templat
 					},
 				},
 			},
+			PlatformVersion:    "1.4.0", // LATEST which is set to 1.3.0 (?) which doesn’t allow efs volumes.
 			PropagateTags:      ecsapi.PropagateTagsService,
 			SchedulingStrategy: ecsapi.SchedulingStrategyReplica,
 			ServiceRegistries:  []ecs.Service_ServiceRegistry{serviceRegistry},
@@ -579,8 +621,8 @@ func networkResourceName(project *types.Project, network string) string {
 	return fmt.Sprintf("%s%sNetwork", normalizeResourceName(project.Name), normalizeResourceName(network))
 }
 
-func serviceResourceName(dependency string) string {
-	return fmt.Sprintf("%sService", normalizeResourceName(dependency))
+func serviceResourceName(service string) string {
+	return fmt.Sprintf("%sService", normalizeResourceName(service))
 }
 
 func normalizeResourceName(s string) string {

ecs/compatibility.go

@@ -62,10 +62,16 @@ var compatibleComposeAttributes = []string{
 	"services.secrets.source",
 	"services.secrets.target",
 	"services.user",
+	"services.volumes",
+	"services.volumes.read_only",
+	"services.volumes.source",
+	"services.volumes.target",
 	"services.working_dir",
 	"secrets.external",
 	"secrets.name",
 	"secrets.file",
+	"volumes",
+	"volumes.external",
 }
 
 func (c *fargateCompatibilityChecker) CheckImage(service *types.ServiceConfig) {
@@ -101,3 +107,9 @@ func (c *fargateCompatibilityChecker) CheckLoggingDriver(config *types.LoggingCo
 		c.Unsupported("services.logging.driver %s is not supported", config.Driver)
 	}
 }
+
+func (c *fargateCompatibilityChecker) CheckVolumeConfigExternal(config *types.VolumeConfig) {
+	if !config.External.External {
+		c.Unsupported("non-external volumes are not supported")
+	}
+}

ecs/convert.go

@@ -81,6 +81,22 @@ func convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi
 		})
 	}
 
+	for _, v := range service.Volumes {
+		source := project.Volumes[v.Source]
+		volumes = append(volumes, ecs.TaskDefinition_Volume{
+			EFSVolumeConfiguration: &ecs.TaskDefinition_EFSVolumeConfiguration{
+				FilesystemId:  source.Name,
+				RootDirectory: source.DriverOpts["root_directory"],
+			},
+			Name: v.Source,
+		})
+		mounts = append(mounts, ecs.TaskDefinition_MountPoint{
+			ContainerPath: v.Target,
+			ReadOnly:      v.ReadOnly,
+			SourceVolume:  v.Source,
+		})
+	}
+
 	pairs, err := createEnvironment(project, service)
 	if err != nil {
 		return nil, err

ecs/sdk.go

@@ -36,19 +36,21 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
 	"github.com/aws/aws-sdk-go/service/ecs"
 	"github.com/aws/aws-sdk-go/service/ecs/ecsiface"
+	"github.com/aws/aws-sdk-go/service/efs"
+	"github.com/aws/aws-sdk-go/service/efs/efsiface"
 	"github.com/aws/aws-sdk-go/service/elbv2"
 	"github.com/aws/aws-sdk-go/service/elbv2/elbv2iface"
 	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/aws/aws-sdk-go/service/iam/iamiface"
 	"github.com/aws/aws-sdk-go/service/secretsmanager"
 	"github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface"
-	cf "github.com/awslabs/goformation/v4/cloudformation"
 	"github.com/sirupsen/logrus"
 )
 
 type sdk struct {
 	ECS ecsiface.ECSAPI
 	EC2 ec2iface.EC2API
+	EFS efsiface.EFSAPI
 	ELB elbv2iface.ELBV2API
 	CW  cloudwatchlogsiface.CloudWatchLogsAPI
 	IAM iamiface.IAMAPI
@@ -63,6 +65,7 @@ func newSDK(sess *session.Session) sdk {
 	return sdk{
 		ECS: ecs.New(sess),
 		EC2: ec2.New(sess),
+		EFS: efs.New(sess),
 		ELB: elbv2.New(sess),
 		CW:  cloudwatchlogs.New(sess),
 		IAM: iam.New(sess),
@@ -187,12 +190,8 @@ func (s sdk) StackExists(ctx context.Context, name string) (bool, error) {
 	return len(stacks.Stacks) > 0, nil
 }
 
-func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template, parameters map[string]string) error {
+func (s sdk) CreateStack(ctx context.Context, name string, template []byte, parameters map[string]string) error {
 	logrus.Debug("Create CloudFormation stack")
-	json, err := marshall(template)
-	if err != nil {
-		return err
-	}
 
 	param := []*cloudformation.Parameter{}
 	for name, value := range parameters {
@@ -202,10 +201,10 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template
 		})
 	}
 
-	_, err = s.CF.CreateStackWithContext(ctx, &cloudformation.CreateStackInput{
+	_, err := s.CF.CreateStackWithContext(ctx, &cloudformation.CreateStackInput{
 		OnFailure:        aws.String("DELETE"),
 		StackName:        aws.String(name),
-		TemplateBody:     aws.String(string(json)),
+		TemplateBody:     aws.String(string(template)),
 		Parameters:       param,
 		TimeoutInMinutes: nil,
 		Capabilities: []*string{
@@ -221,12 +220,8 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template
 	return err
 }
 
-func (s sdk) CreateChangeSet(ctx context.Context, name string, template *cf.Template, parameters map[string]string) (string, error) {
+func (s sdk) CreateChangeSet(ctx context.Context, name string, template []byte, parameters map[string]string) (string, error) {
 	logrus.Debug("Create CloudFormation Changeset")
-	json, err := marshall(template)
-	if err != nil {
-		return "", err
-	}
 
 	param := []*cloudformation.Parameter{}
 	for name := range parameters {
@@ -241,7 +236,7 @@ func (s sdk) CreateChangeSet(ctx context.Context, name string, template *cf.Temp
 		ChangeSetName: aws.String(update),
 		ChangeSetType: aws.String(cloudformation.ChangeSetTypeUpdate),
 		StackName:     aws.String(name),
-		TemplateBody:  aws.String(string(json)),
+		TemplateBody:  aws.String(string(template)),
 		Parameters:    param,
 		Capabilities: []*string{
 			aws.String(cloudformation.CapabilityCapabilityIam),
@@ -671,3 +666,25 @@ func (s sdk) GetLoadBalancerURL(ctx context.Context, arn string) (string, error)
 	}
 	return dnsName, nil
 }
+
+func (s sdk) WithVolumeSecurityGroups(ctx context.Context, id string, fn func(securityGroups []string) error) error {
+	mounts, err := s.EFS.DescribeMountTargetsWithContext(ctx, &efs.DescribeMountTargetsInput{
+		FileSystemId: aws.String(id),
+	})
+	if err != nil {
+		return err
+	}
+	for _, mount := range mounts.MountTargets {
+		groups, err := s.EFS.DescribeMountTargetSecurityGroupsWithContext(ctx, &efs.DescribeMountTargetSecurityGroupsInput{
+			MountTargetId: mount.MountTargetId,
+		})
+		if err != nil {
+			return err
+		}
+		err = fn(aws.StringValueSlice(groups.SecurityGroups))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

ecs/testdata/simple/simple-cloudformation-conversion.golden

@@ -123,6 +123,7 @@
             ]
           }
         },
+        "PlatformVersion": "1.4.0",
         "PropagateTags": "SERVICE",
         "SchedulingStrategy": "REPLICA",
         "ServiceRegistries": [

ecs/up.go

@@ -37,7 +37,7 @@ func (b *ecsAPIService) Up(ctx context.Context, project *types.Project) error {
 		return err
 	}
 
-	template, err := b.convert(project)
+	template, err := b.Convert(ctx, project)
 	if err != nil {
 		return err
 	}

go.mod

@@ -6,6 +6,8 @@ go 1.15
 // we need to create a new release tag for docker/distribution
 replace github.com/docker/distribution => github.com/docker/distribution v0.0.0-20200708230824-53e18a9d9bfe
 
+replace github.com/awslabs/goformation/v4 => github.com/ndeloof/goformation/v4 v4.8.1-0.20200827081523-b7a7ac375adf
+
 require (
 	github.com/AlecAivazis/survey/v2 v2.1.1
 	github.com/Azure/azure-sdk-for-go v43.3.0+incompatible

go.sum

@@ -66,8 +66,6 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.34.8 h1:GDfVeXG8XQDbpOeAj7415F8qCQZwvY/k/fj+HBqUnBA=
 github.com/aws/aws-sdk-go v1.34.8/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
-github.com/awslabs/goformation/v4 v4.14.0 h1:E2Pet9eIqA4qzt3dzzzE4YN83V4Kyfbcio0VokBC9TA=
-github.com/awslabs/goformation/v4 v4.14.0/go.mod h1:GcJULxCJfloT+3pbqCluXftdEK2AD/UqpS3hkaaBntg=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -287,6 +285,8 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
+github.com/ndeloof/goformation/v4 v4.8.1-0.20200827081523-b7a7ac375adf h1:jdmD8L6TKRZpa7B4qUmjiWRBMkgbfUF/7pi/Kgba5lA=
+github.com/ndeloof/goformation/v4 v4.8.1-0.20200827081523-b7a7ac375adf/go.mod h1:GcJULxCJfloT+3pbqCluXftdEK2AD/UqpS3hkaaBntg=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=