Merge pull request #866 from convox/20160713

[RELEASE] 20160713

api/controllers/instances.go

@@ -4,7 +4,8 @@ import (
 	"net/http"
 	"strconv"
 
-	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/convox/rack/api/httperr"
 	"github.com/convox/rack/api/models"
 	"github.com/convox/rack/api/provider"
@@ -70,8 +71,9 @@ func InstanceTerminate(rw http.ResponseWriter, r *http.Request) *httperr.Error {
 
 	instanceId := mux.Vars(r)["id"]
 
-	_, err = models.EC2().TerminateInstances(&ec2.TerminateInstancesInput{
-		InstanceIds: []*string{&instanceId},
+	_, err = models.AutoScaling().TerminateInstanceInAutoScalingGroup(&autoscaling.TerminateInstanceInAutoScalingGroupInput{
+		InstanceId: aws.String(instanceId),
+		ShouldDecrementDesiredCapacity: aws.Bool(false),
 	})
 
 	if err != nil {

api/dist/kernel.json

@@ -11,7 +11,14 @@
     "BlankRegistryHost": { "Fn::Equals": [ { "Ref": "RegistryHost" }, "" ] },
     "Development": { "Fn::Equals": [ { "Ref": "Development" }, "Yes" ] },
     "Private": { "Fn::Equals": [ { "Ref": "Private" }, "Yes" ] },
-    "PrivateApi": { "Fn::Equals": [ { "Ref": "PrivateApi" }, "Yes" ] }
+    "PrivateApi": { "Fn::Equals": [ { "Ref": "PrivateApi" }, "Yes" ] },
+    "RegionHasEFS": {
+        "Fn::Or": [
+          { "Fn::Equals": [ { "Ref": "AWS::Region" }, "us-east-1" ]},
+          { "Fn::Equals": [ { "Ref": "AWS::Region" }, "us-west-2" ]},
+          { "Fn::Equals": [ { "Ref": "AWS::Region" }, "eu-west-1" ]}
+        ]
+    }
   },
   "Mappings": {
     "RegionConfig": {
@@ -156,6 +163,11 @@
       "Description": "SSL certificate",
       "Default": ""
     },
+    "ContainerDisk": {
+      "Type": "Number",
+      "Description": "Default container disk size in GB",
+      "Default": "10"
+    },
     "Development": {
       "Type": "String",
       "Description": "Development mode",
@@ -897,12 +909,25 @@
             "bootcmd:\n",
             "  - mkswap /dev/xvdb\n",
             "  - swapon /dev/xvdb\n",
+            "  - yum install -y nfs-utils\n",
+            "  - mkdir /volumes\n",
+            { "Fn::If": [ "RegionHasEFS",
+              { "Fn::Join": [ "", [
+                "  - while true; do mount -t nfs -o nfsvers=4.1 $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone).",
+                { "Ref": "VolumeFilesystem" },
+                ".efs.",
+                { "Ref": "AWS::Region" },
+                ".amazonaws.com:/ /volumes && break; sleep 5; done\n"
+              ] ] },
+              ""
+            ] },
             "  - [ cloud-init-per, instance, docker_storage_setup, /usr/bin/docker-storage-setup ]\n",
             "  - echo ECS_CLUSTER=", { "Ref": "Cluster" }, " >> /etc/ecs/ecs.config\n",
             "  - echo ECS_ENGINE_AUTH_TYPE=docker >> /etc/ecs/ecs.config\n",
             "  - head -n -1 /etc/sysconfig/docker >> /etc/sysconfig/docker-tmp\n",
             "  - mv /etc/sysconfig/docker-tmp /etc/sysconfig/docker\n",
             "  - echo 'OPTIONS=\"--default-ulimit nofile=1024000:1024000\"' >> /etc/sysconfig/docker\n",
+            { "Fn::Join": [ "", [ "  - echo 'OPTIONS=\"${OPTIONS} --storage-opt dm.basesize=", { "Ref": "ContainerDisk" }, "G\"' >> /etc/sysconfig/docker\n" ] ] },
             { "Fn::If": [ "BlankCertificate",
               { "Fn::Join": [ "", [
                 "  - echo 'ECS_ENGINE_AUTH_DATA={\"index.docker.io\":{\"username\":\"\",\"password\":\"\",\"email\":\"\"},\"", { "Fn::Join": [ ":", [ { "Fn::GetAtt": [ "Balancer", "DNSName" ] }, "5000" ] ] }, "\":{\"username\":\"convox\",\"password\":\"", { "Ref": "Password" }, "\",\"email\":\"user@convox.io\"}}' >> /etc/ecs/ecs.config\n",
@@ -1437,6 +1462,53 @@
         "ProvisionedThroughput": { "ReadCapacityUnits": "5", "WriteCapacityUnits": "5" }
       }
     },
+    "VolumeFilesystem": {
+      "Type": "AWS::EFS::FileSystem",
+      "Condition": "RegionHasEFS",
+      "Properties": {
+        "FileSystemTags": [
+          { "Key": "Name", "Value": { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName" }, "shared-volumes" ] ] } }
+        ]
+      }
+    },
+    "VolumeSecurity": {
+      "Type": "AWS::EC2::SecurityGroup",
+      "Condition": "RegionHasEFS",
+      "Properties": {
+        "GroupDescription": "volume security group",
+        "SecurityGroupIngress": [
+          { "IpProtocol": "tcp", "FromPort": "2049", "ToPort": "2049", "CidrIp": { "Ref": "VPCCIDR" } }
+        ],
+        "VpcId": { "Ref": "Vpc" }
+      }
+    },
+    "VolumeTarget0": {
+      "Type": "AWS::EFS::MountTarget",
+      "Condition": "RegionHasEFS",
+      "Properties": {
+        "FileSystemId": { "Ref": "VolumeFilesystem" },
+        "SubnetId": { "Ref": "Subnet0" },
+        "SecurityGroups": [ { "Ref": "VolumeSecurity" } ]
+      }
+    },
+    "VolumeTarget1": {
+      "Type": "AWS::EFS::MountTarget",
+      "Condition": "RegionHasEFS",
+      "Properties": {
+        "FileSystemId": { "Ref": "VolumeFilesystem" },
+        "SubnetId": { "Ref": "Subnet1" },
+        "SecurityGroups": [ { "Ref": "VolumeSecurity" } ]
+      }
+    },
+    "VolumeTarget2": {
+      "Type": "AWS::EFS::MountTarget",
+      "Condition": "RegionHasEFS",
+      "Properties": {
+        "FileSystemId": { "Ref": "VolumeFilesystem" },
+        "SubnetId": { "Ref": "Subnet2" },
+        "SecurityGroups": [ { "Ref": "VolumeSecurity" } ]
+      }
+    },
     "Settings": {
       "Properties": {
         "AccessControl": "Private",

api/manifest/manifest.go

@@ -191,7 +191,7 @@ func (m Manifest) Validate() error {
 	regexValidCronLabel := regexp.MustCompile(`\A[a-zA-Z][-a-zA-Z0-9]{3,29}\z`)
 
 	for _, entry := range map[string]ManifestEntry(m) {
-		labels := entry.labelsByPrefix("convox.cron")
+		labels := entry.LabelsByPrefix("convox.cron")
 		for k := range labels {
 			parts := strings.Split(k, ".")
 			if len(parts) != 3 {
@@ -998,9 +998,11 @@ func (me ManifestEntry) Label(key string) string {
 	return ""
 }
 
-func (me ManifestEntry) labelsByPrefix(prefix string) map[string]string {
+// LabelsByPrefix filters Docker Compose label names by prefixes and returns
+// a map of label names to values that match.
+func (e ManifestEntry) LabelsByPrefix(prefix string) map[string]string {
 	returnLabels := make(map[string]string)
-	switch labels := me.Labels.(type) {
+	switch labels := e.Labels.(type) {
 	case map[interface{}]interface{}:
 		for k, v := range labels {
 			ks, ok := k.(string)
@@ -1097,6 +1099,17 @@ func (me ManifestEntry) syncAdds(app, process string) error {
 				remote = filepath.Join(wd, remote)
 			}
 
+			local := filepath.Join(me.Build, parts[1])
+
+			info, err := os.Stat(local)
+			if err != nil {
+				return err
+			}
+
+			if !info.IsDir() && strings.HasSuffix(remote, "/") {
+				remote = filepath.Join(remote, filepath.Base(local))
+			}
+
 			registerSync(containerName(app, process), filepath.Join(me.Build, parts[1]), remote)
 		}
 	}
@@ -1385,6 +1398,12 @@ func processAdds(prefix string, adds map[string]bool, lock sync.Mutex, syncs []S
 
 		fmt.Printf(system("%s uploading %d files\n"), prefix, len(adds))
 
+		if os.Getenv("CONVOX_DEBUG") != "" {
+			for add := range adds {
+				fmt.Printf(system("%s uploading %s\n"), prefix, add)
+			}
+		}
+
 		for _, sync := range syncs {
 			var buf bytes.Buffer
 
@@ -1683,6 +1702,12 @@ func (m *Manifest) downloadRemoteAdds(container string, adds map[string]string)
 
 	fmt.Printf(system("%s downloading %d files\n"), m.systemPrefix(), len(remotes))
 
+	if os.Getenv("CONVOX_DEBUG") != "" {
+		for _, remote := range remotes {
+			fmt.Printf(system("%s downloading %s\n"), m.systemPrefix(), remote)
+		}
+	}
+
 	args := append([]string{"exec", "-i", container, "tar", "czf", "-"}, remotes...)
 
 	data, err := Execer("docker", args...).Output()

api/models/manifest.go

@@ -7,6 +7,7 @@ import (
 	"html/template"
 	"math/rand"
 	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 
@@ -163,9 +164,10 @@ func (m Manifest) AppName() string {
 	return m[0].app.Name
 }
 
-func (me ManifestEntry) LabelsByPrefix(prefix string) map[string]string {
+// LabelsByPrefix will return the labels that have a given prefix
+func (e ManifestEntry) LabelsByPrefix(prefix string) map[string]string {
 	returnLabels := make(map[string]string)
-	switch labels := me.Labels.(type) {
+	switch labels := e.Labels.(type) {
 	case map[interface{}]interface{}:
 		for k, v := range labels {
 			ks, ok := k.(string)
@@ -468,13 +470,38 @@ func (me ManifestEntry) EnvMap() map[string]string {
 	return envs
 }
 
-func (me ManifestEntry) MountableVolumes() []string {
-	volumes := []string{}
+// MountableVolume describes a mountable volume
+type MountableVolume struct {
+	Host      string
+	Container string
+}
+
+// MountableVolumes return the mountable volumes for a manifest entry
+func (e ManifestEntry) MountableVolumes() []MountableVolume {
+	volumes := []MountableVolume{}
+
+	for _, volume := range e.Volumes {
+		parts := strings.Split(volume, ":")
 
-	for _, volume := range me.Volumes {
-		if strings.HasPrefix(volume, "/var/run/docker.sock") {
-			volumes = append(volumes, volume)
+		// if only one volume part use it for both sides
+		if len(parts) == 1 {
+			parts = append(parts, parts[0])
 		}
+
+		// if we dont have two volume parts bail
+		if len(parts) != 2 {
+			continue
+		}
+
+		// only support absolute paths for volume source
+		if !filepath.IsAbs(parts[0]) {
+			continue
+		}
+
+		volumes = append(volumes, MountableVolume{
+			Host:      parts[0],
+			Container: parts[1],
+		})
 	}
 
 	return volumes

api/models/templates/app.tmpl

@@ -605,6 +605,7 @@
 {{ end }}
 
 {{ define "entry_task" }}
+{{ $e := . }}
 { "Fn::If": [ "Blank{{ upper .Name }}Service",
   {
     "Name": "{{ .Name }}",
@@ -633,15 +634,18 @@
     },
     "Volumes": [
       {{ range .MountableVolumes }}
-        "{{ . }}",
+        {{ if eq .Host "/var/run/docker.sock" }}
+          "{{.Host}}:{{.Container}}",
+        {{ else }}
+          { "Fn::Join": [ "", [ "/volumes/", { "Ref": "AWS::StackName" }, "/{{$e.Name}}{{.Host}}:{{.Container}}" ] ] },
+        {{ end }}
       {{ end }}
       { "Ref" : "AWS::NoValue" }
     ],
     "Services": [
       { "Ref" : "AWS::NoValue" }
     ],
     "PortMappings": [
-      {{ $e := . }}
       {{ range .PortMappings }}
         { "Fn::Join": [ ":", [
           { "Ref": "{{ upper $e.Name }}Port{{ .Balancer }}Host" },