Allowed multiple values of secrets (#34)

models/examples/if-else-statement.json

@@ -5,6 +5,14 @@
  {
  "value": "10",
  "target": "max"
+ },
+ {
+ "value": "SECRET_PASS",
+ "target": "secretL1"
+ },
+ {
+ "value": "SECRET_PASS",
+ "target": "secretL2"
  }
  ],
  "workflow":
@@ -18,7 +26,9 @@
  "uid": "192161d7-e3f2-4991-adc0-a99c88c144c0",
  "description": "Graph component",
  "inputs": [
- { "name": "max", "mediatype": ["integer"], "type": "parameter" }
+ { "name": "max", "mediatype": ["integer"], "type": "parameter" },
+ { "name": "secretL1", "type": "env_secret" },
+ { "name": "secretL2", "type": "env_secret" }
  ],
  "outputs": [
  { "name": "description", "type": "parameter" }
@@ -30,6 +40,14 @@
  {
  "source": { "port": "max" },
  "target": { "node": "N1", "port": "value" }
+ },
+ {
+ "source": { "port": "secretL1" },
+ "target": { "node": "N1", "port": "secretB1" }
+ },
+ {
+ "source": { "port": "secretL2" },
+ "target": { "node": "If", "port": "secretPass" }
  }
  ],
  "outputMappings": [
@@ -46,7 +64,8 @@
  "uid": "192161d7-e3f2-4991-adc0-a99c88c144b0",
  "description": "Generate",
  "inputs": [
- { "name": "value", "type": "parameter" }
+ { "name": "value", "type": "parameter" },
+ { "name": "secretB1", "type": "env_secret" }
  ],
  "outputs": [
  { "name": "rand", "type": "parameter" }
@@ -57,7 +76,7 @@
  "container": {
  "name": "containername_n1_b1",
  "image": "bash:latest",
- "command": ["bash", "-c", "shuf -i 0-$0 -n1 > /tmp/out"],
+ "command": ["bash", "-c", "echo SecretB1 $secretB1; shuf -i 0-$0 -n1 > /tmp/out"],
  "args": []
  },
  "args": [
@@ -85,7 +104,8 @@
  "name": "valFromParam",
  "mediatype": ["number"],
  "type": "parameter"
- }
+ },
+ { "name": "secretPass", "type": "env_secret" }
  ],
  "outputs": [
  { "name": "ifOut", "type": "parameter" }
@@ -97,6 +117,10 @@
  {
  "source": { "port": "valFromParam" },
  "target": { "port": "valParam" }
+ },
+ {
+ "source": { "port": "secretPass" },
+ "target": { "port": "envValue" }
  }
  ],
  "outputMappings": [
@@ -113,6 +137,10 @@
  "name": "valParam",
  "mediatype": ["number"],
  "type": "parameter"
+ },
+ {
+ "name": "envValue",
+ "type": "env_secret"
  }
  ],
  "outputs": [
@@ -127,7 +155,7 @@
  "command": [
  "sh",
  "-c",
- "echo value $0 is huge > /tmp/out"
+ "echo env $envValue; echo value $0 is huge > /tmp/out"
  ],
  "args": []
  },
@@ -153,6 +181,10 @@
  "name": "valParam",
  "mediatype": ["number"],
  "type": "parameter"
+ },
+ {
+ "name": "envValue",
+ "type": "env_secret"
  }
  ],
  "outputs": [
@@ -167,7 +199,7 @@
  "command": [
  "sh",
  "-c",
- "echo value $0 is small > /tmp/out"
+ "echo env $envValue; echo value $0 is small > /tmp/out"
  ],
  "args": []
  },

transpiler/helpers.go

@@ -52,8 +52,7 @@ func getNodeSecretMap(targetNodeId string, cmpSecrets secretMap, cmpInputs []mod
  for _, m := range inputsMap {
  mt, ok := checkInputType(cmpInputs, m.Source.Port)
  if ok && mt == models.FlowifySecretType && m.Target.Node == targetNodeId {
- k, _ := findKeyFor(nSecrets, m.Source.Port)
- nSecrets[k] = m.Target.Port
+ nSecrets[m.Target.Port] = cmpSecrets[m.Source.Port]
  }
  }
  return nSecrets

transpiler/transpiler.go

@@ -72,9 +72,9 @@ func AddBrick(name string, cmp *models.Brick, outputs wfv1.Outputs, inputs wfv1.
  container.Args = args
  envs := []corev1.EnvVar{}
  for k, e := range sMap {
- s := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: secret.DefaultObjectName}, Key: k}
+ s := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: secret.DefaultObjectName}, Key: e}
  vf := corev1.EnvVarSource{SecretKeyRef: &s}
- env := corev1.EnvVar{Name: e, ValueFrom: &vf}
+ env := corev1.EnvVar{Name: k, ValueFrom: &vf}
  envs = append(envs, env)
  }
  container.Env = envs
@@ -155,7 +155,7 @@ func AddGraph(name string, cmp *models.Graph, outputs wfv1.Outputs, inputs wfv1.
  errors.Errorf("Unrecognized implementation type: %s", impCmp)
  }
  case models.FlowifyVolumeType:
- logrus.Info("Ref volume edge: ", e)
+ logrus.Debug("Ref volume edge:", e)
  default:
  return errors.Errorf("Unrecognized edge target-type, '%s', for target port-address '%s.%s'", it, e.Target.Node, e.Target.Port)
  }
@@ -425,12 +425,11 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks
  }
  inParams = append(inParams, param)
  case models.FlowifySecretType:
- _, ok := findKeyFor(cmpSecrets, i.Name)
- if !ok {
- cmpSecrets[i.Name] = i.Name
+ if _, ok := cmpSecrets[i.Name]; !ok {
+ return nil, fmt.Errorf("missing secret <%s> for component: %s", i.Name, cmp.Uid)
  }
  case models.FlowifyVolumeType:
- logrus.Info("Ref mount: ", i.Name, volumes[i.Name])
+ logrus.Debugf("Ref mount: %s %s", i.Name, volumes[i.Name])
  default:
  return nil, fmt.Errorf("cannot append input data (name: %s, type %s) at node %s", i.Name, i.Type, cmpName)
  }
@@ -468,8 +467,7 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks
  if mt, ok := checkInputType(cmp.Inputs, m.Source.Port); ok {
  switch mt {
  case models.FlowifySecretType:
- k, _ := findKeyFor(nodeSecrets, m.Source.Port)
- nodeSecrets[k] = m.Target.Port
+ nodeSecrets[m.Target.Port] = cmpSecrets[m.Source.Port]
  case models.FlowifyVolumeType:
  if val, ok := volumes[m.Source.Port]; ok {
  nodeVolumes[m.Target.Port] = val
@@ -508,12 +506,11 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks
  if mt, ok := checkInputType(cmp.Inputs, m.Source.Port); ok {
  switch mt {
  case models.FlowifySecretType:
- k, _ := findKeyFor(nodeSecrets, m.Source.Port)
- nodeSecrets[k] = m.Target.Port
+ nodeSecrets[m.Target.Port] = cmpSecrets[m.Source.Port]
  case models.FlowifyVolumeType:
  if val, ok := volumes[m.Source.Port]; ok {
  nodeVolumes[m.Target.Port] = val
- logrus.Infof("Rewriting nodeVolumes for %s: %s -> %s", cmp.Name, m.Source.Port, m.Target.Port)
+ logrus.Debugf("Rewriting nodeVolumes for %s: %s -> %s", cmp.Name, m.Source.Port, m.Target.Port)
  }
  }
  }
@@ -538,7 +535,7 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks
  scopeVolumes[m.Target.Node] = make(map[string]corev1.Volume, 0)
  }
  scopeVolumes[m.Target.Node][m.Target.Port] = val
- logrus.Infof("Rewriting nodeVolumes for %s: %s -> %s", cmp.Name, m.Source.Port, m.Target.Port)
+ logrus.Debugf("Rewriting nodeVolumes for %s: %s -> %s", cmp.Name, m.Source.Port, m.Target.Port)
  }
  }
  }
@@ -551,17 +548,17 @@ func TraverseComponent(cmp *models.Component, templates *[]wfv1.Template, tasks
  }
  nodeSecrets := getNodeSecretMap(c.Id, cmpSecrets, cmp.Inputs, impCmp.InputMappings)
  volumes := getConnectedVolumeMap(c.Id, &tc, impCmp.Edges, impCmp.Nodes, scopeVolumes)
- logrus.Info(volumes)
+ logrus.Debug(volumes)
  _, err := TraverseComponent(&tc, templates, tasks, nodeSecrets, volumes)
  if err != nil {
  return nil, err
  }
  }
  case models.Brick:
- for k, elem := range cmpSecrets {
+ for k, _ := range cmpSecrets {
  ex := false
  for _, i := range cmp.Inputs {
- if elem == i.Name {
+ if k == i.Name {
  ex = true
  break
  }
@@ -605,15 +602,20 @@ func GetArgoWorkflow(job models.Job) (*wfv1.Workflow, error) {
  secretMapValues := make(secretMap)
  for _, cmpInput := range wf.Component.Inputs {
  if cmpInput.Type == models.FlowifySecretType {
+ missing := true
  for _, jobInput := range job.InputValues {
  if cmpInput.Name == jobInput.Target {
  val, ok := jobInput.Value.(string)
  if !ok {
  return nil, errors.Errorf("Cannot convert flowify secret '%s' to string.", jobInput.Target)
  }
- secretMapValues[val] = cmpInput.Name
+ secretMapValues[cmpInput.Name] = val
+ missing = false
  }
  }
+ if missing {
+ return nil, errors.Errorf("Missing input secret <%s> for job: %s", cmpInput.Name, job.Name)
+ }
  }
  }
 
@@ -637,7 +639,7 @@ func GetArgoWorkflow(job models.Job) (*wfv1.Workflow, error) {
  if err != nil {
  return nil, err
  }
- logrus.Infof("Appending volume from config: %s -> (%s) ", config, v.Target)
+ logrus.Debugf("Appending volume from config: %s -> (%s) ", config, v.Target)
  volumeMap[tgt.Name] = volume // add volume with top level input name
  } else {
  return nil, fmt.Errorf("mount config must be json encoded string")

transpiler/transpiler_test.go

@@ -187,6 +187,8 @@ const (
  }`
 )
 
+var secrets = secretMap{"secretWF1": "wf1_secret_value", "secretWF2": "wf2_secret_value"}
+
 func init() {
 }
 
@@ -202,7 +204,7 @@ func Test_ParseComponentTree(t *testing.T) {
  assert.Equal(t, 3, len(wf.Component.Inputs))
  assert.Equal(t, 0, len(wf.Component.Outputs))
 
- argoWF, err := ParseComponentTree(wf, secretMap{}, volumeMap{}, map[string]string{}, map[string]string{})
+ argoWF, err := ParseComponentTree(wf, secrets, volumeMap{}, map[string]string{}, map[string]string{})
  assert.Nil(t, err)
  assert.True(t, len(argoWF.Spec.Templates) == 4, "Expected no. of templates is 4.")
  // assert.Equal(t, "seedWF", argoWF.Spec.Arguments.Parameters[0].Name)
@@ -222,17 +224,17 @@ func Test_ParseComponentTree(t *testing.T) {
  for _, env := range template.Container.Env {
  switch env.Name {
  case "secretN1":
- assert.Equal(t, "secretWF1", env.ValueFrom.SecretKeyRef.Key)
+ assert.Equal(t, secrets["secretWF1"], env.ValueFrom.SecretKeyRef.Key)
  case "secretN2":
- assert.Equal(t, "secretWF2", env.ValueFrom.SecretKeyRef.Key)
+ assert.Equal(t, secrets["secretWF2"], env.ValueFrom.SecretKeyRef.Key)
  default:
  t.Errorf("Unexpected secret %s in brick %s.", env.Name, template.Name)
  }
  }
  case "192161d7-e3f2-4991-adc0-a99c88c144b2":
  assert.Equal(t, 1, len(template.Container.Env))
  assert.Equal(t, "secretN1", template.Container.Env[0].Name)
- assert.Equal(t, "secretWF2", template.Container.Env[0].ValueFrom.SecretKeyRef.Key)
+ assert.Equal(t, secrets["secretWF2"], template.Container.Env[0].ValueFrom.SecretKeyRef.Key)
  default:
  t.Errorf("Unexpected template %s.", template.Name)
  }
@@ -244,7 +246,8 @@ func Test_RemoveDuplicatedTemplates(t *testing.T) {
  err := json.Unmarshal([]byte(minimalExampleJSON), &wf)
  assert.Nil(t, err)
 
- job := models.Job{Metadata: models.Metadata{Description: "test job"}, Type: "job", InputValues: nil, Workflow: wf}
+ inputValues := []models.Value{{Value: secrets["secretWF1"], Target: "secretWF1"}, {Value: secrets["secretWF2"], Target: "secretWF2"}}
+ job := models.Job{Metadata: models.Metadata{Description: "test job"}, Type: "job", InputValues: inputValues, Workflow: wf}
  argoWF, err := GetArgoWorkflow(job)
  assert.Nil(t, err)
  assert.True(t, len(argoWF.Spec.Templates) == 3, "Expected no. of templates is 3.")