[AWS] Fix: override disk when mounted on same device name (#97)

internal/plan/mappings/aws/ec2_asg.yaml

@@ -62,35 +62,69 @@ compute_resource:
  storage:
  - type: list
  item:
- - paths: '${ami}.values.block_device_mappings[].ebs | select(length > 0)'
+ - paths: '${ami}.values.block_device_mappings[] | select(length > 0)'
  properties:
  size:
- - paths: ".volume_size"
+ - paths: ".ebs.volume_size"
  default: 8
  unit: gb
  type:
- - paths: ".volume_type"
+ - paths: ".ebs.volume_type"
  default: standard
  reference:
  general: disk_types 
+ key:
+ - paths: ".device_name | cbf::extract_disk_key"
+ override_priority: 
+ - default: 2
  - paths: 
  - '${launch_configuration}.values.ebs_block_device[] | select(length > 0)'
- - '${launch_configuration}.values.block_device_mappings[] | select(length > 0) | select(.virtual_name == null or (.virtual_name | startswith("ephemeral") | not)) | .ebs'
  properties:
  size:
- - paths: ".volume_size"
+ - paths: 
+ - ".volume_size"
  unit: gb
- - paths: ".snapshot_id"
+ - paths: 
+ - ".snapshot_id"
  reference:
  paths: .prior_state.values.root_module.resources[] | select(.values.id == "${key}") | .values
  property: ".volume_size"
  - default: 8
  unit: gb
  type:
- - paths: ".volume_type"
+ - paths: 
+ - ".volume_type"
  default: standard
  reference:
  general: disk_types
+ key:
+ - paths: ".device_name | cbf::extract_disk_key"
+ override_priority: 
+ - default: 0
+ - paths:
+ - '${launch_configuration}.values.block_device_mappings[] | select(length > 0) | select(.virtual_name == null or (.virtual_name | startswith("ephemeral") | not)) | select(.ebs != null)'
+ properties:
+ size:
+ - paths: 
+ - ".ebs[0].volume_size"
+ unit: gb
+ - paths: 
+ - ".ebs[0].snapshot_id"
+ reference:
+ paths: .prior_state.values.root_module.resources[] | select(.values.id == "${key}") | .values
+ property: ".volume_size"
+ - default: 8
+ unit: gb
+ type:
+ - paths: 
+ - ".ebs[0].volume_type"
+ default: standard
+ reference:
+ general: disk_types
+ key:
+ - paths: ".device_name | cbf::extract_disk_key"
+ override_priority: 
+ - default: 1
  - paths: 
  - '${launch_configuration}.values.ephemeral_block_device[] | select(length > 0)'
  - '${launch_configuration}.values.block_device_mappings[] | select(length > 0) | select(.virtual_name != null and (.virtual_name | startswith("ephemeral")))'

internal/plan/mappings/aws/ec2_instance.yaml

@@ -65,35 +65,69 @@ compute_resource:
  - type: list
  item:
  - paths: 
- - '${ami}.values.block_device_mappings[] | select(length > 0) | .ebs'
+ - '${ami}.values.block_device_mappings[] | select(length > 0)'
  properties:
  size:
- - paths: ".volume_size"
+ - paths: ".ebs.volume_size"
  default: 8
  unit: gb
  type:
- - paths: ".volume_type"
+ - paths: ".ebs.volume_type"
  default: standard
  reference:
- general: disk_types 
+ general: disk_types
+ key:
+ - paths: ".device_name | cbf::extract_disk_key"
+ override_priority: 
+ - default: 2
  - paths: 
  - '.values.ebs_block_device[] | select(length > 0)'
- - '${launch_template}.values.block_device_mappings[] | select(length > 0) | select(.virtual_name == null or .virtual_name == "" or (.virtual_name | startswith("ephemeral") | not)) | .ebs'
  properties:
  size:
- - paths: ".volume_size"
+ - paths: 
+ - ".volume_size"
  unit: gb
- - paths: ".snapshot_id"
+ - paths: 
+ - ".snapshot_id"
  reference:
  paths: .prior_state.values.root_module.resources[] | select(.values.id == "${key}") | .values
  property: ".volume_size"
  - default: 8
  unit: gb
  type:
- - paths: ".volume_type"
+ - paths: 
+ - ".volume_type"
+ default: standard
+ reference:
+ general: disk_types
+ key:
+ - paths: ".device_name | cbf::extract_disk_key"
+ override_priority: 
+ - default : 0
+ - paths:
+ - '${launch_template}.values.block_device_mappings[] | select(length > 0) | select(.virtual_name == null or .virtual_name == "" or (.virtual_name | startswith("ephemeral") | not)) | select(.ebs != null)'
+ properties:
+ size:
+ - paths: 
+ - ".ebs[0].volume_size"
+ unit: gb
+ - paths: 
+ - ".ebs[0].snapshot_id"
+ reference:
+ paths: .prior_state.values.root_module.resources[] | select(.values.id == "${key}") | .values
+ property: ".volume_size"
+ - default: 8
+ unit: gb
+ type:
+ - paths: 
+ - ".ebs[0].volume_type"
  default: standard
  reference:
  general: disk_types
+ key:
+ - paths: ".device_name | cbf::extract_disk_key"
+ override_priority: 
+ - default : 1
  - paths: 
  - '.values.ephemeral_block_device[] | select(length > 0)'
  - '${launch_template}.values.block_device_mappings[] | select(length > 0) | select(.virtual_name != null and (.virtual_name | startswith("ephemeral")))'

internal/plan/resolver.go

@@ -13,8 +13,10 @@ import (
 )
 
 type storage struct {
- SizeGb decimal.Decimal
- IsSSD bool
+ SizeGb decimal.Decimal
+ IsSSD bool
+ OverridePriority int
+ Key string
 }
 
 func applyReference(valueFound string, propertyMapping *PropertyDefinition, context *tfContext) (interface{}, error) {

internal/plan/resources.go

@@ -3,6 +3,7 @@ package plan
 import (
  "fmt"
  "regexp"
+ "sort"
  "strings"
 
  "github.com/carboniferio/carbonifer/internal/providers"
@@ -305,20 +306,9 @@ func GetComputeResource(resourceI interface{}, resourceMapping *ResourceMapping,
  return nil, errors.Wrapf(err, "Cannot get storages for %v", resourceAddress)
  }
 
- for i, storageI := range storages {
- if storageI == nil {
- continue
- }
- storage, err := getStorage(storageI.(map[string]interface{}))
- if err != nil {
- return nil, errors.Wrapf(err, "Cannot get storage[%v] for %v", i, resourceAddress)
- }
- size := storage.SizeGb
- if storage.IsSSD {
- computeResource.Specs.SsdStorage = computeResource.Specs.SsdStorage.Add(size)
- } else {
- computeResource.Specs.HddStorage = computeResource.Specs.HddStorage.Add(size)
- }
+ err = processStorages(storages, &computeResource, context)
+ if err != nil {
+ return nil, errors.Wrapf(err, "Cannot process storages for %v", resourceAddress)
  }
 
  resourcesResult = append(resourcesResult, computeResource)
@@ -346,6 +336,68 @@ func getGPU(gpu map[string]interface{}) ([]string, error) {
  return gpuTypes, nil
 }
 
+func processStorages(storagesI []interface{}, computeResource *resources.ComputeResource, context *tfContext) error {
+ storagesByKey := map[string][]*storage{}
+ for i, storageI := range storagesI {
+ if storageI == nil {
+ continue
+ }
+ storageItem, err := getStorage(storageI.(map[string]interface{}))
+ if err != nil {
+ return errors.Wrapf(err, "Cannot get storage[%v] for %v", i, context.ResourceAddress)
+ }
+ if storagesByKey[storageItem.Key] == nil {
+ storagesByKey[storageItem.Key] = []*storage{storageItem}
+ } else {
+ storagesByKey[storageItem.Key] = append(storagesByKey[storageItem.Key], storageItem)
+ }
+ }
+
+ // Print the map nicely
+ for key, storages := range storagesByKey {
+ log.Debugf("Storage Key: %v", key)
+ for _, storage := range storages {
+ log.Debugf(" %v\n", storage)
+ }
+ }
+ log.Debug("------")
+
+ storagesOverriden := []*storage{}
+
+ for key, storages := range storagesByKey {
+ // Take the storage with the lower priority in the list
+ if key != "" {
+ storages = sortStorages(storages)
+ storageItem := storages[0]
+ storagesOverriden = append(storagesOverriden, storageItem)
+ } else {
+ storagesOverriden = append(storagesOverriden, storages...)
+ }
+ }
+
+ for _, storageItem := range storagesOverriden {
+ size := storageItem.SizeGb
+ if storageItem.IsSSD {
+ computeResource.Specs.SsdStorage = computeResource.Specs.SsdStorage.Add(size)
+ } else {
+ computeResource.Specs.HddStorage = computeResource.Specs.HddStorage.Add(size)
+ }
+ }
+
+ return nil
+}
+
+func sortStorages(storages []*storage) []*storage {
+ if len(storages) == 0 {
+ return storages
+ }
+ // sort by override_priority, lowest priority first
+ sort.Slice(storages, func(i, j int) bool {
+ return storages[i].OverridePriority < storages[j].OverridePriority
+ })
+ return storages
+}
+
 func getStorage(storageMap map[string]interface{}) (*storage, error) {
  storageSize, ok := storageMap["size"].(*valueWithUnit)
  if !ok {
@@ -400,9 +452,24 @@ func getStorage(storageMap map[string]interface{}) (*storage, error) {
  isSSD = true
  }
  }
+ overridePriority := 0
+ overridePriorityI := storageMap["override_priority"]
+ if overridePriorityI != nil {
+ overridePriority = overridePriorityI.(*valueWithUnit).Value.(int)
+ }
+
+ key := ""
+ keyI := storageMap["key"]
+ if keyI != nil {
+ keyString := keyI.(*valueWithUnit).Value.(string)
+ key = keyString
+ }
+
  storage := storage{
- SizeGb: storageSizeGb,
- IsSSD: isSSD,
+ SizeGb: storageSizeGb,
+ IsSSD: isSSD,
+ OverridePriority: overridePriority,
+ Key: key,
  }
  return &storage, nil
 }

internal/plan/test/resources_aws_asg_test.go

@@ -58,7 +58,7 @@ func TestGetResource_AWSASG(t *testing.T) {
  MemoryMb: int32(16384),
 
  HddStorage: decimal.NewFromInt(300),
- SsdStorage: decimal.NewFromInt(30 + 150),
+ SsdStorage: decimal.NewFromInt(150),
  },
  },
  }

internal/plan/test/resources_aws_test.go

@@ -90,8 +90,26 @@ func TestGetResource_EC2(t *testing.T) {
  VCPUs: int32(4),
  MemoryMb: int32(16384),
 
+ HddStorage: decimal.Zero,
+ SsdStorage: decimal.NewFromInt(180),
+ },
+ },
+ "aws_instance.ec2_with_lt_disk_override": resources.ComputeResource{
+ Identification: &resources.ResourceIdentification{
+ Address: "aws_instance.ec2_with_lt_disk_override",
+ Name: "ec2_with_lt_disk_override",
+ ResourceType: "aws_instance",
+ Provider: providers.AWS,
+ Region: "eu-west-3",
+ Count: 1,
+ ReplicationFactor: 1,
+ },
+ Specs: &resources.ComputeResourceSpecs{
+ VCPUs: int32(4),
+ MemoryMb: int32(16384),
+
  HddStorage: decimal.NewFromInt(300),
- SsdStorage: decimal.NewFromInt(30 + 150),
+ SsdStorage: decimal.NewFromInt(150),
  },
  },
  }

internal/utils/jsonQuery.go

@@ -18,6 +18,15 @@ func (*moduleLoader) LoadModule(name string) (*gojq.Query, error) {
 
  def all_select(a; b):
  .planned_values | .. | objects | select(has("resources")) | .resources[] | select(.[a] == b);
+
+ def extract_disk_key:
+ if test("^/dev/sd[a-z]+") or test("^/dev/xvd[a-z]+") then
+ capture("^/dev/(?:sd|xvd)(?<letter>[a-z]+)").letter
+ elif test("^/dev/nvme[0-2]?[0-9]n") then
+ capture("^/dev/nvme(?<number>[0-2]?[0-9])n").number | tonumber | (96 + .) | [.] | implode
+ else
+ "Unknown format"
+ end;
  `)
  }
  return nil, fmt.Errorf("module not found: %q", name)

test/terraform/aws_ec2/ec2_launch_template.tf

@@ -3,6 +3,18 @@ resource "aws_launch_template" "my_launch_template" {
  image_id = "${data.aws_ami.ubuntu.id}"
  instance_type = "m5d.xlarge"
 
+ block_device_mappings {
+ device_name = "/dev/sdb"
+ virtual_name = "ephemeral0"
+ }
+
+}
+
+resource "aws_launch_template" "my_launch_template_disk_override" {
+ name_prefix = "my_launch_template"
+ image_id = "${data.aws_ami.ubuntu.id}"
+ instance_type = "m5d.xlarge"
+
  block_device_mappings {
  device_name = "/dev/sda1"
 
@@ -25,4 +37,11 @@ resource "aws_instance" "ec2_with_lt" {
  id = "${aws_launch_template.my_launch_template.id}"
  version = "$$Latest"
  }
+}
+
+resource "aws_instance" "ec2_with_lt_disk_override" {
+ launch_template {
+ id = "${aws_launch_template.my_launch_template_disk_override.id}"
+ version = "$$Latest"
+ }
 }