add pulp single repo

Signed-off-by: Jonathan Holloway <jholloway@redhat.com>

pkg/jobs/memory.go

@@ -60,7 +60,7 @@ func NewMemoryClient() *MemoryWorker {
 		SlowQueueSize: 50,
 		FastWorkers:   100,
 		SlowWorkers:   10,
-		Timeout:       2 * time.Hour,
+		Timeout:       6 * time.Hour,
 		IntSignal:     []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGSTOP},
 	})
 }

pkg/models/commits.go

@@ -26,7 +26,7 @@ const (
 	RepoStatusError = "ERROR"
 	// RepoStatusPending is for when the repo process is starting
 	RepoStatusPending = "PENDING"
-	// RepoStatusSkipped is for when a Repo is available to the user
+	// RepoStatusSkipped is for when a Repo is available to the user (post commit build)
 	RepoStatusSkipped = "SKIPPED"
 	// RepoStatusSuccess is for when a Repo is available to the user
 	RepoStatusSuccess = "SUCCESS"
@@ -63,6 +63,7 @@ type Repo struct {
 	Model
 	URL        string `json:"RepoURL"`          // AWS repo URL
 	Status     string `json:"RepoStatus"`       // AWS repo upload status
+	PulpID     string `json:"pulp_repo_id"`     // Pulp Repo ID (used for updates)
 	PulpURL    string `json:"pulp_repo_url"`    // Distribution URL returned from Pulp
 	PulpStatus string `json:"pulp_repo_status"` // Status of Pulp repo import
 }

pkg/routes/images_test.go

@@ -1802,6 +1802,7 @@ var _ = Describe("Images Route Integration Tests", func() {
 						InstalledPackages: []models.InstalledPackage{
 							{Name: "vim"},
 						},
+						Repo: &models.Repo{},
 					},
 				}
 

pkg/services/images.go

@@ -371,6 +371,8 @@ func (s *ImageService) CreateImage(image *models.Image) error {
 		image.Installer.OrgID = image.OrgID
 	}
 
+	image.Commit.Repo = &models.Repo{}
+
 	if result := db.DB.Create(&image); result.Error != nil {
 		return result.Error
 	}
@@ -578,7 +580,13 @@ func (s *ImageService) UpdateImage(image *models.Image, previousImage *models.Im
 				err := errors.NewBadRequest(fmt.Sprintf("Commit repo wasn't found in the database: #%v", image.Commit.ID))
 				return err
 			}
+			// NOTE: this is storing the URL of the parent commit instead of the DBID of the parent commit
 			image.Commit.OSTreeParentCommit = repo.ContentURL()
+
+			image.Commit.Repo = &models.Repo{
+				PulpID:  repo.PulpID,
+				PulpURL: repo.PulpURL,
+			}
 		}
 
 		if config.DistributionsRefs[previousSuccessfulImage.Distribution] != config.DistributionsRefs[image.Distribution] {
@@ -676,7 +684,7 @@ func (s *ImageService) processCommit(ctx context.Context, image *models.Image, l
 	imageID := image.ID
 	for {
 		// reload the image from database, for a long-running process
-		if err := db.DB.Joins("Commit").Joins("Installer").First(&image, imageID).Error; err != nil {
+		if err := db.DB.Joins("Commit.Repo").Joins("Commit").Joins("Installer").First(&image, imageID).Error; err != nil {
 			log.WithContext(ctx).WithField("error", err.Error()).Error("error occurred when reloading image from database")
 			if goErrors.Is(err, gorm.ErrRecordNotFound) {
 				return nil, new(ImageNotFoundError)
@@ -903,24 +911,17 @@ func (s *ImageService) runAWSRepoProcess(repo *models.Repo) (*models.Repo, error
 // CreateRepoForImage creates the OSTree repo to host that image
 func (s *ImageService) CreateRepoForImage(ctx context.Context, img *models.Image) (*models.Repo, error) {
 	s.log.Info("Creating OSTree repo for image")
-	repo := &models.Repo{
-		Status:     models.RepoStatusPending,
-		PulpStatus: models.RepoStatusPending,
-	}
-	tx := db.DB.Create(repo)
-	if tx.Error != nil {
-		return nil, tx.Error
-	}
-	s.log = s.log.WithField("repoID", repo.ID)
-	s.log.Debug("OSTree repo is created on the database")
 
-	img.Commit.Repo = repo
-	img.Commit.RepoID = &repo.ID
+	img.Commit.Repo.Status = models.RepoStatusPending
+	img.Commit.Repo.PulpStatus = models.RepoStatusPending
 
-	tx = db.DB.Save(img.Commit)
+	repo := img.Commit.Repo
+
+	tx := db.DB.Save(img.Commit)
 	if tx.Error != nil {
 		return nil, tx.Error
 	}
+	s.log = s.log.WithField("repoID", repo.ID)
 	s.log.Debug("OSTree repo was saved to commit")
 
 	var err error

pkg/services/images_test.go

@@ -2577,22 +2577,14 @@ var _ = Describe("Image Service Test", func() {
 				Name:      faker.UUIDHyphenated(),
 				RequestID: faker.UUIDHyphenated(),
 				Installer: &models.Installer{OrgID: orgID},
-				Commit:    &models.Commit{OrgID: orgID},
+				Commit: &models.Commit{
+					OrgID: orgID,
+					Repo:  &models.Repo{}},
 			}
 			err := db.DB.Create(image).Error
 			Expect(err).ToNot(HaveOccurred())
 		})
 
-		It("should create image repo successfully", func() {
-			expectedRepo := models.Repo{URL: faker.URL(), Status: models.RepoStatusSuccess}
-			err := db.DB.Create(&expectedRepo).Error
-			Expect(err).ToNot(HaveOccurred())
-			mockRepoBuilder.EXPECT().ImportRepo(gomock.AssignableToTypeOf(&models.Repo{})).Return(&expectedRepo, nil)
-			repo, err := service.CreateRepoForImage(context.Background(), image)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(repo.ID).To(Equal(expectedRepo.ID))
-		})
-
 		It("should return successfully if AWS repo status is success", func() {
 			expectedRepo := models.Repo{
 				URL:        faker.URL(),

pkg/services/repobuilder.go

@@ -376,7 +376,8 @@ func (rb *RepoBuilder) StoreRepo(ctx context.Context, repo *models.Repo) (*model
 
 	var err error
 	log.WithContext(ctx).Debug("Storing repo via Pulp")
-	repo.PulpURL, err = repostore.PulpRepoStore(ctx, cmt.OrgID, *cmt.RepoID, cmt.ImageBuildTarURL)
+	repo.PulpID, repo.PulpURL, err = repostore.PulpRepoStore(ctx, cmt.OrgID, *cmt.RepoID, cmt.ImageBuildTarURL,
+		repo.PulpID, repo.PulpURL, cmt.OSTreeParentRef)
 	if err != nil {
 		log.WithContext(ctx).WithField("error", err.Error()).Error("Error storing Image Builder commit in Pulp OSTree repo")
 
@@ -397,9 +398,6 @@ func (rb *RepoBuilder) StoreRepo(ctx context.Context, repo *models.Repo) (*model
 		return repo, fmt.Errorf("error saving status :: %s", result.Error.Error())
 	}
 
-	redactedURL, _ := url.Parse(repo.PulpURL)
-	log.WithContext(ctx).WithField("repo_url", redactedURL.Redacted()).Info("Commit stored in Pulp OSTree repo")
-
 	return repo, nil
 }
 

pkg/services/repostore/pulpstore.go

@@ -14,55 +14,83 @@ import (
 )
 
 // PulpRepoStore imports an OSTree repo into a Pulp repository
-func PulpRepoStore(ctx context.Context, orgID string, sourceRepoID uint, sourceURL string) (string, error) {
+func PulpRepoStore(ctx context.Context, orgID string, edgeRepoID uint,
+	imageBuilderTarURL string, pulpParentID string, pulpParentURL string,
+	pulpParentRef string) (string, string, error) {
 	// create a pulp service with a pre-defined name
 	pserv, err := domainService(ctx, orgID)
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 	log.WithContext(ctx).Info("Service for domain created")
 
 	// Import the commit tarfile into an initial Pulp file repo
-	fileRepoArtifact, fileRepoVersion, err := fileRepoImport(ctx, pserv, sourceURL)
+	fileRepoArtifact, fileRepoVersion, err := fileRepoImport(ctx, pserv, imageBuilderTarURL)
 	if err != nil {
 		log.WithContext(ctx).Error("Error importing tarfile to initial pulp file repository")
 
-		return "", err
+		return "", "", err
 	}
 	log.WithContext(ctx).WithFields(log.Fields{
 		"artifact": fileRepoArtifact,
 		"version":  fileRepoArtifact,
 	}).Info("Pulp artifact uploaded")
 
-	// create an OSTree repository in Pulp
-	repoName := fmt.Sprintf("repo-%s-%d", orgID, sourceRepoID)
+	var pulpID string
+	var distBaseURL string
+	var pulpHref string
+	if pulpParentID != "" {
+		// update a Pulp OSTree repository
+		err = ostreeRepoImportUpdate(ctx, pserv, pulpParentID, fileRepoArtifact, pulpParentRef)
+		if err != nil {
+			log.WithContext(ctx).Error("Error importing tarfile into pulp ostree repository")
 
-	distBaseURL, pulpHref, err := createOSTreeRepository(ctx, pserv, orgID, repoName)
-	if err != nil {
-		log.WithContext(ctx).Error("Error creating pulp ostree repository")
+			return "", "", err
+		}
 
-		return "", err
-	}
-	log.WithContext(ctx).WithFields(log.Fields{
-		"dist_base_url": distBaseURL,
-		"pulp_href":     pulpHref,
-	}).Info("Pulp OSTree Repo created with Content Guard and Distribution")
+		pulpID = pulpParentID
+		distBaseURL = pulpParentURL
 
-	repoURL, err := ostreeRepoImport(ctx, pserv, pulpHref, repoName, distBaseURL, fileRepoArtifact)
-	if err != nil {
-		log.WithContext(ctx).Error("Error importing tarfile into pulp ostree repository")
+		log.WithFields(log.Fields{
+			"parent_id":     pulpParentID,
+			"parent_url":    pulpParentURL,
+			"dist_base_url": distBaseURL,
+			"pulp_href":     pulpHref},
+		).Info("Existing repo updated with a new commit")
+	} else {
+		// create a Pulp OSTree repository
+		repoName := fmt.Sprintf("repo-%s-%d", orgID, edgeRepoID)
+
+		distBaseURL, pulpHref, err = createOSTreeRepository(ctx, pserv, orgID, repoName)
+		if err != nil {
+			log.WithContext(ctx).Error("Error creating pulp ostree repository")
 
-		return "", err
+			return "", "", err
+		}
+
+		log.WithContext(ctx).WithFields(log.Fields{
+			"dist_base_url": distBaseURL,
+			"pulp_href":     pulpHref,
+		}).Info("Pulp OSTree Repo created with Content Guard and Distribution")
+		err = ostreeRepoImport(ctx, pserv, pulpHref, fileRepoArtifact)
+		if err != nil {
+			log.WithContext(ctx).Error("Error importing tarfile into pulp ostree repository")
+
+			return "", "", err
+		}
+
+		pulpID = pulp.ScanUUID(&pulpHref).String()
 	}
-	parsedURL, _ := url.Parse(repoURL)
+
+	parsedURL, _ := url.Parse(distBaseURL)
 	log.WithContext(ctx).WithField("repo_url", parsedURL.Redacted()).Info("Image Builder commit tarfile imported into pulp ostree repo from pulp file repo")
 
 	if err = pserv.FileRepositoriesVersionDelete(ctx, pulp.ScanUUID(&fileRepoVersion), pulp.ScanRepoFileVersion(&fileRepoVersion)); err != nil {
-		return "", err
+		return "", "", err
 	}
 	log.WithContext(ctx).WithField("filerepo_version", fileRepoVersion).Info("Artifact version deleted")
 
-	return repoURL, nil
+	return pulpID, distBaseURL, nil
 }
 
 // looks up a domain, creates one if it does not exist, and returns a service using that domain
@@ -198,25 +226,64 @@ func fileRepoImport(ctx context.Context, pulpService *pulp.PulpService, sourceUR
 }
 
 // ostreeRepoImport imports an artifact into a Pulp ostree repo
-func ostreeRepoImport(ctx context.Context, pulpService *pulp.PulpService, pulpHref string, pulpRepoName string,
-	distBaseURL string, fileRepoArtifact string) (string, error) {
+func ostreeRepoImport(ctx context.Context, pulpService *pulp.PulpService,
+	pulpHref string, fileRepoArtifact string) error {
 
 	log.WithContext(ctx).WithFields(log.Fields{
-		"pulp_href":            pulp.ScanUUID(&pulpHref),
-		"pulp_reponame":        pulpRepoName,
-		"distribution_baseurl": distBaseURL,
-		"artifact_href":        fileRepoArtifact,
+		"pulp_href_uuid":     pulp.ScanUUID(&pulpHref),
+		"pulp_href":          pulpHref,
+		"artifact_href_uuid": pulp.ScanUUID(&fileRepoArtifact),
+		"artifact_href":      fileRepoArtifact,
 	}).Debug("Starting tarfile import into Pulp OSTree repository")
 	repoImported, err := pulpService.RepositoriesImport(ctx, pulp.ScanUUID(&pulpHref), "repo", fileRepoArtifact)
+
 	if err != nil {
-		return "", err
+		return err
+	}
+
+	log.WithContext(ctx).WithFields(log.Fields{
+		"repo_href": *repoImported.PulpHref,
+		"repo_name": repoImported.Name,
+	}).Info("Repository imported")
+
+	return nil
+}
+
+// ostreeRepoImportUpdate imports an artifact into a Pulp ostree repo
+func ostreeRepoImportUpdate(ctx context.Context, pulpService *pulp.PulpService,
+	parentID string, fileRepoArtifact string, parentRef string) error {
+
+	log.WithContext(ctx).WithFields(log.Fields{
+		"pulp_parent_uuid":   parentID,
+		"artifact_href_uuid": pulp.ScanUUID(&fileRepoArtifact),
+		"artifact_href":      fileRepoArtifact,
+		"pulp_parent_ref":    parentRef,
+	}).Debug("Starting tarfile update into Pulp OSTree repository")
+
+	var repoImported *pulp.OstreeOstreeRepositoryResponse
+	var err error
+
+	pulpUUID, err := uuid.Parse(parentID)
+	if err != nil {
+		log.WithField("pulp_id", pulpUUID).Error("Unable to parse Pulp ID into UUID")
+
+		return err
+	}
+
+	repoImported, err = pulpService.RepositoriesImportCommit(ctx,
+		pulpUUID,
+		"repo",
+		fileRepoArtifact,
+		parentRef)
+
+	if err != nil {
+		return err
 	}
-	log.WithContext(ctx).WithField("repo_href", *repoImported.PulpHref).Info("Repository imported")
 
-	parsedURL, _ := url.Parse(distBaseURL)
 	log.WithContext(ctx).WithFields(log.Fields{
-		"repo_distribution_url": parsedURL.Redacted(),
-	}).Debug("Repo import into Pulp complete")
+		"repo_href": *repoImported.PulpHref,
+		"repo_name": repoImported.Name,
+	}).Info("Repository imported")
 
-	return distBaseURL, nil
+	return nil
 }

podman/container-compose-dbmigrate.yml

@@ -1,4 +1,7 @@
 version: '2'
+secrets:
+  edgemgmt_config:
+    file: $PWD/podman/env/edgemgmt_config.json
 services:
     edge-api-dbmigrate:
       image: localhost/edge-api:localdev
@@ -8,3 +11,6 @@ services:
         - ~/dev/git/RedHatInsights/edge-api:/opt/app-root/src:Z
       working_dir: /opt/app-root/src
       command:  go run cmd/migrate/main.go
+      secrets:
+        - source: edgemgmt_config
+          target: /tmp/edgemgmt_config.json