From d1e6d42cefa7887cc7fbba46229598b2b4abb4cb Mon Sep 17 00:00:00 2001 From: AndriiMysko Date: Thu, 6 Jan 2022 17:55:56 +0200 Subject: [PATCH 1/6] Add retrieving custom LXD images attempt --- backend/lxd.go | 13 +++++++++++++ backend/start_attributes.go | 7 +++++-- image/manager.go | 19 +++++++++++++++++++ job.go | 1 + 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/backend/lxd.go b/backend/lxd.go index a9a791da4..ff5e4478a 100644 --- a/backend/lxd.go +++ b/backend/lxd.go @@ -509,6 +509,19 @@ func (p *lxdProvider) Start(ctx gocontext.Context, startAttributes *StartAttribu // Select the image if startAttributes.ImageName != "" { imageName = startAttributes.ImageName + } else if startAttributes.OSCustom != "" { + imageName = startAttributes.OSCustom + + var imgManager *image.Manager + imgManager, err = image.NewManager(ctx, nil, p.imageBaseURL) + if err != nil { + return nil, err + } + err = imgManager.LoadCustom(imageName, startAttributes.TamToken) + + if err != nil { + return nil, err + } } else { imageArch := startAttributes.Arch if p.archOverride != "" { diff --git a/backend/start_attributes.go b/backend/start_attributes.go index 075efbdb9..040bc748e 100644 --- a/backend/start_attributes.go +++ b/backend/start_attributes.go @@ -20,6 +20,7 @@ type StartAttributes struct { Group string `json:"group"` OS string `json:"os"` ImageName string `json:"image_name"` + OSCustom string `json:"os_custom"` // The VMType isn't stored in the config directly, but in the top level of // the job payload, see the worker.JobPayload struct. @@ -29,12 +30,10 @@ type StartAttributes struct { // the job payload, see the worker.JobPayload struct. VMConfig VmConfig `json:"-"` - // The VMSize isn't stored in the config directly, but in the top level of // the job payload, see the worker.JobPayload struct. VMSize string `json:"-"` - // Warmer isn't stored in the config directly, but in the top level of // the job payload, see the worker.JobPayload struct. Warmer bool `json:"-"` @@ -46,6 +45,10 @@ type StartAttributes struct { // ProgressType isn't stored in the config directly, but is injected from // the processor ProgressType string `json:"-"` + + // TamToken isn't stored in the config directly, but in the top level of + // the job payload, see the worker.JobPayload struct. + TamToken string `json:"-"` } // SetDefaults sets any missing required attributes to the default values provided diff --git a/image/manager.go b/image/manager.go index b360227cd..afe71a51b 100644 --- a/image/manager.go +++ b/image/manager.go @@ -68,6 +68,19 @@ func (m *Manager) Load(imageName string) error { return m.importImage(imageName, imageURL) } +func (m *Manager) LoadCustom(imageName string, tamToken string) error { + // TODO: Image cleanup + + imageURL := m.customImageUrl(imageName, tamToken) + + m.logger.WithFields(logrus.Fields{ + "image_name": imageName, + "image_url": imageURL, + }).Info("importing image") + + return m.importImage(imageName, imageURL) +} + func (m *Manager) Exists(imageName string) (bool, error) { images, err := m.client.GetImages() if err != nil { @@ -206,3 +219,9 @@ func (m *Manager) imageUrl(name string) string { u.Path = fmt.Sprintf("/images/travis/%s", name) return u.String() } + +func (m *Manager) customImageUrl(name string, tamToken string) string { + u := *m.imagesServerURL + u.Path = fmt.Sprintf("/images/custom/%s/%s", name, tamToken) + return u.String() +} diff --git a/job.go b/job.go index 00b4b79c7..e3dcae8e6 100644 --- a/job.go +++ b/job.go @@ -41,6 +41,7 @@ type JobPayload struct { Queue string `json:"queue"` Trace bool `json:"trace"` Warmer bool `json:"warmer"` + TamToken string `json:"tam_token"` } // JobMetaPayload contains meta information about the job. From a290ecc03bdc434390393fb273ea50b49fe58df9 Mon Sep 17 00:00:00 2001 From: Stanislav Kolotinskiy Date: Tue, 25 Jan 2022 14:37:29 +0200 Subject: [PATCH 2/6] Debug --- backend/lxd.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/lxd.go b/backend/lxd.go index ff5e4478a..93a070142 100644 --- a/backend/lxd.go +++ b/backend/lxd.go @@ -506,6 +506,7 @@ func (p *lxdProvider) Start(ctx gocontext.Context, startAttributes *StartAttribu jobID, _ := context.JobIDFromContext(ctx) repo, _ := context.RepositoryFromContext(ctx) + logger.WithField("ImageName", startAttributes.ImageName).WithField("OSCustom", startAttributes.OSCustom).Info("DEBUG") // Select the image if startAttributes.ImageName != "" { imageName = startAttributes.ImageName From 252e8706791faea46cc4e9dece170f91284e8f50 Mon Sep 17 00:00:00 2001 From: Andrii Mysko Date: Tue, 25 Jan 2022 15:23:54 +0200 Subject: [PATCH 3/6] Debug --- backend/lxd.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/lxd.go b/backend/lxd.go index ff5e4478a..d6d8f9fdc 100644 --- a/backend/lxd.go +++ b/backend/lxd.go @@ -506,6 +506,7 @@ func (p *lxdProvider) Start(ctx gocontext.Context, startAttributes *StartAttribu jobID, _ := context.JobIDFromContext(ctx) repo, _ := context.RepositoryFromContext(ctx) + logger.WithField("ImageName", startAttributes.ImageName).WithField("OSCustom", startAttributes.OSCustom).Info("DEBUG") // Select the image if startAttributes.ImageName != "" { imageName = startAttributes.ImageName From db1f8d81bf3f2a34ae3352048b67e6c969bfeadb Mon Sep 17 00:00:00 2001 From: Andrii Mysko Date: Tue, 25 Jan 2022 16:56:26 +0200 Subject: [PATCH 4/6] Add TamToken --- amqp_job_queue.go | 1 + 1 file changed, 1 insertion(+) diff --git a/amqp_job_queue.go b/amqp_job_queue.go index 8566095c8..ce3bc5dfb 100644 --- a/amqp_job_queue.go +++ b/amqp_job_queue.go @@ -216,6 +216,7 @@ func (q *AMQPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err erro buildJob.startAttributes.VMSize = buildJob.payload.VMSize buildJob.startAttributes.VMConfig = buildJob.payload.VMConfig buildJob.startAttributes.Warmer = buildJob.payload.Warmer + buildJob.startAttributes.TamToken = buildJob.payload.TamToken buildJob.startAttributes.SetDefaults(q.DefaultLanguage, q.DefaultDist, q.DefaultArch, q.DefaultGroup, q.DefaultOS, VMTypeDefault, VMConfigDefault) buildJob.conn = q.conn buildJob.stateCount = buildJob.payload.Meta.StateUpdateCount From f73bfb382777ff411e07fcd95975f8c932117197 Mon Sep 17 00:00:00 2001 From: Stanislav Kolotinskiy Date: Fri, 28 Jan 2022 15:02:31 +0200 Subject: [PATCH 5/6] Write separate cloud init --- backend/lxd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/lxd.go b/backend/lxd.go index c8584b23e..24672a83c 100644 --- a/backend/lxd.go +++ b/backend/lxd.go @@ -758,7 +758,7 @@ iface eth0 inet static mtu %s `, address, p.networkGateway, strings.Join(p.networkDNS, " "), p.networkMTU) default: - fileName = "/etc/netplan/50-cloud-init.yaml" + fileName = "/etc/netplan/51-cloud-init.yaml" content = fmt.Sprintf(`network: version: 2 ethernets: From 2758d3073915c6ccd52b02b4616b252306e6755b Mon Sep 17 00:00:00 2001 From: Andrii Mysko Date: Thu, 3 Feb 2022 16:11:18 +0200 Subject: [PATCH 6/6] Escape image name for lxd server --- image/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/manager.go b/image/manager.go index 1ff28f1f0..0b14a4cd5 100644 --- a/image/manager.go +++ b/image/manager.go @@ -234,6 +234,6 @@ func (m *Manager) imageUrl(name string) string { func (m *Manager) customImageUrl(name string, tamToken string) string { u := *m.imagesServerURL - u.Path = fmt.Sprintf("/images/custom/%s/%s", name, tamToken) + u.Path = fmt.Sprintf("/images/custom/%s/%s", url.QueryEscape(name), tamToken) return u.String() }