Skip to content

Commit

Permalink
Add digest check for chart download (#46)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hangyan
hangyan authored Feb 15, 2020
1 parent 5f53e44 commit 3b8bc95
Show file tree
Hide file tree
Showing 5 changed files with 64 additions and 118 deletions.
35 changes: 23 additions & 12 deletions controllers/chartrepo_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,7 @@ func ignoreNotFound(err error) error {

// syncChartRepo sync ChartRepo to helm repo store
func (r *ChartRepoReconciler) syncChartRepo(cr *alaudaiov1alpha1.ChartRepo, ctx context.Context) error {
return r.createCharts(cr, ctx)

return r.syncCharts(cr, ctx)
}

// DownloadIndexFile fetches the index from a repository.
Expand Down Expand Up @@ -175,12 +174,14 @@ func loadIndex(data []byte) (*repo.IndexFile, error) {
return i, nil
}

// createCharts create charts resource for a repo
// syncCharts create charts resource for a repo
// TODO: ace.ACE
func (r *ChartRepoReconciler) createCharts(cr *alaudaiov1alpha1.ChartRepo, ctx context.Context) error {
func (r *ChartRepoReconciler) syncCharts(cr *alaudaiov1alpha1.ChartRepo, ctx context.Context) error {
log := r.Log.WithValues("chartrepo", cr.GetName())

checked := map[string]bool{}
existCharts := map[string]alaudaiov1alpha1.Chart{}

index, err := r.GetIndex(cr, ctx)
if err != nil {
return err
Expand All @@ -190,17 +191,13 @@ func (r *ChartRepoReconciler) createCharts(cr *alaudaiov1alpha1.ChartRepo, ctx c
checked[strings.ToLower(name)] = true
}

existCharts := map[string]alaudaiov1alpha1.Chart{}

var charts alaudaiov1alpha1.ChartList
labels := client.MatchingLabels{
"repo": cr.GetName(),
}

if err := r.List(ctx, &charts, labels, client.InNamespace(r.Namespace)); err != nil {
return err
}

for _, item := range charts.Items {
name := strings.Split(item.GetName(), ".")[0]
existCharts[name] = item
Expand All @@ -222,12 +219,12 @@ func (r *ChartRepoReconciler) createCharts(cr *alaudaiov1alpha1.ChartRepo, ctx c
}

old := existCharts[name]

if compareChart(old, chart) {
chart.SetResourceVersion(old.GetResourceVersion())
if err := r.Update(ctx, chart); err != nil {
return err
}
log.Info("update chart", "name", old.Name, "repo", cr.Name)
}

}
Expand All @@ -241,18 +238,28 @@ func (r *ChartRepoReconciler) createCharts(cr *alaudaiov1alpha1.ChartRepo, ctx c
}
log.Info("delete charts", "name", item.GetName())
}

}

return nil

}

// compareChart simply compare versions list length
// compareChart compare if a Chart need update
// 1. If length not equal, update
// 2. compare all digest

func compareChart(old alaudaiov1alpha1.Chart, new *alaudaiov1alpha1.Chart) bool {
if len(old.Spec.Versions) != len(new.Spec.Versions) {
return true
}

for _, o := range old.Spec.Versions {
for _, n := range new.Spec.Versions {
if o.Version == n.Version && o.Digest != n.Digest {
return true
}
}
}

return false
}

Expand Down Expand Up @@ -330,6 +337,10 @@ func (r *ChartRepoReconciler) updateChartRepoStatus(ctx context.Context, cr *ala
func (r *ChartRepoReconciler) isReadyForResync(cr *alaudaiov1alpha1.ChartRepo) bool {
log := r.Log.WithValues("chartrepo", cr.GetName())

if cr.Status.Phase != "Synced" {
return true
}

if cr.GetAnnotations() != nil && cr.GetAnnotations()["alauda.io/last-sync-at"] != "" {
last := cr.GetAnnotations()["alauda.io/last-sync-at"]
// see: https://stackoverflow.com/questions/25845172/parsing-date-string-in-go
Expand Down
12 changes: 6 additions & 6 deletions pkg/chartrepo/list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,23 +118,23 @@ func GetChartRepo(name string, ns string, cfg *rest.Config) (*repo.Entry, error)

}

// get chart
func GetChart(name, version, ns string, cfg *rest.Config) (string, error) {
// get chart info, url and digest is the info we want
func GetChart(name, version, ns string, cfg *rest.Config) (*repo.ChartVersion, error) {
client, err := clientset.NewForConfig(cfg)
if err != nil {
return "", err
return nil, err
}

chart, err := client.AppV1alpha1().Charts(ns).Get(name, metav1.GetOptions{})
if err != nil {
return "", err
return nil, err
}

for _, item := range chart.Spec.Versions {
if version == "" || version == item.Version {
return item.URLs[0], nil
return &item.ChartVersion, nil
}
}
return "", errors.New(fmt.Sprintf("cannot find version %s for chart %s", version, name))
return nil, errors.New(fmt.Sprintf("cannot find version %s for chart %s", version, name))

}
44 changes: 15 additions & 29 deletions pkg/helm/download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,57 +62,41 @@ func (d *Downloader) getRepoInfo(name string, ns string) (*repo.Entry, error) {
return entry, err
}

// cal the local path for a chart
// name: <repo>/<name>
func getChartPath(name, version string) string {
new := strings.Replace(name, "/", "-", -1)
return fmt.Sprintf("%s/%s-%s.tgz", ChartsDir, new, version)
}

// downloadChart download a chart from helm repo to local disk and return the path
// name: <repo>/<chart>
func (d *Downloader) downloadChart(name string, version string) (string, error) {
log := d.log

repo, chart := getRepoAndChart(name)
if repo == "" && chart == "" {
repoName, chart := getRepoAndChart(name)
if repoName == "" && chart == "" {
return "", errors.New("cannot parse chart name")
}

log.Info("get chart", "name", name, "version", version)

dir := ChartsDir

if version != "" {
// we can check the path now
checkPath := getChartPath(name, version)
if _, err := os.Stat(checkPath); !os.IsNotExist(err) {
log.Info("chart already downloaded, use it", "path", checkPath)
return checkPath, nil
if _, err := os.Stat(dir); os.IsNotExist(err) {
if err = os.MkdirAll(dir, 0755); err != nil {
return "", err
}
log.Info("helm charts dir not exist, create it: ", "dir", dir)
}

entry, err := d.getRepoInfo(repo, d.ns)
entry, err := d.getRepoInfo(repoName, d.ns)
if err != nil {
log.Error(err, "get chartrepo error")
return "", err
}

chartResourceName := fmt.Sprintf("%s.%s", strings.ToLower(chart), repo)

path, err := chartrepo.GetChart(chartResourceName, version, d.ns, d.cfg)
chartResourceName := fmt.Sprintf("%s.%s", strings.ToLower(chart), repoName)
cv, err := chartrepo.GetChart(chartResourceName, version, d.ns, d.cfg)
if err != nil {
log.Error(err, "get chart error")
return "", err
}

if _, err := os.Stat(dir); os.IsNotExist(err) {
if err = os.MkdirAll(dir, 0755); err != nil {
return "", err
}
log.Info("dir not exist, create it: ", "dir", dir)
}

path := cv.URLs[0]
fileName := strings.Split(path, "/")[1]
filePath := fmt.Sprintf("%s/%s-%s", dir, repo, fileName)
filePath := fmt.Sprintf("%s/%s-%s-%s", dir, repoName, cv.Digest, fileName)

if _, err := os.Stat(filePath); !os.IsNotExist(err) {
log.Info("chart already downloaded, use it", "path", filePath)
Expand All @@ -124,6 +108,8 @@ func (d *Downloader) downloadChart(name string, version string) (string, error)
return "", err
}

log.Info("download chart to disk", "path", filePath)

return filePath, nil

}
Expand Down
45 changes: 10 additions & 35 deletions pkg/helm/install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,42 +54,18 @@ func (d *Deploy) install() (*release.Release, error) {
}
}

//cp, err := client.ChartPathOptions.LocateChart(chrt, settings)
//if err != nil {
// klog.Errorf("locate chart %s error: %s", cp, err.Error())
// // a simple string match
// if client.Version == "" && strings.Contains(err.Error(), " no chart version found for") {
// klog.Info("no normal version found, try using devel flag")
// client.Version = ">0.0.0-0"
// cp, err = client.ChartPathOptions.LocateChart(chrt, settings)
// if err != nil {
// return nil, err
// }
// } else {
// return nil, err
// }
//}

// load from cache first, then from disk
var chartRequested *chart.Chart
chartPath := getChartPath(hr.Spec.Chart, hr.Spec.Version)
d.Log.Info("chart path", "path", chartPath)
result, ok := chartCache.Get(chartPath)
if ok {
log.Info("load charts from cache", "path", chartPath)
chartRequested = result.(*chart.Chart)
} else {
dl := NewDownloader(systemNamespace, inCluster.ToRestConfig(), d.Log)
chartPath, err := dl.downloadChart(hr.Spec.Chart, hr.Spec.Version)
if err != nil {
return nil, err
}
log.Info("load charts from disk", "path", chartPath)
chartRequested, err = loader.Load(chartPath)
if err != nil {
return nil, err
}
chartCache.SetDefault(chartPath, chartRequested)

dl := NewDownloader(systemNamespace, inCluster.ToRestConfig(), d.Log)
chartPath, err := dl.downloadChart(hr.Spec.Chart, hr.Spec.Version)
if err != nil {
return nil, err
}
log.Info("load charts from disk", "path", chartPath)
chartRequested, err = loader.Load(chartPath)
if err != nil {
return nil, err
}

values, err := getValues(hr, inCluster.ToRestConfig())
Expand All @@ -98,7 +74,6 @@ func (d *Deploy) install() (*release.Release, error) {
}

client.Namespace = hr.Spec.Namespace
// klog.Infof("load chart request: %s client: %+v", chartRequested.Name(), client)
validInstallableChart, err := isChartInstallable(chartRequested)
if !validInstallableChart {
log.Error(err, "not installable error")
Expand Down
46 changes: 10 additions & 36 deletions pkg/helm/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,44 +88,18 @@ func (d *Deploy) Sync() (*release.Release, error) {
}
client.ResetValues = true

// locate chart
//chrt := hr.Spec.Chart
//chartPath, err := client.ChartPathOptions.LocateChart(chrt, settings)
//if err != nil {
// klog.Errorf("locate chart %s error: %s", chartPath, err.Error())
// // a simple string match
// if client.Version == "" && strings.Contains(err.Error(), " no chart version found for") {
// klog.Info("no normal version found, try using devel flag")
// client.Version = ">0.0.0-0"
// chartPath, err = client.ChartPathOptions.LocateChart(chrt, settings)
// if err != nil {
// return nil, err
// }
// } else {
// return nil, err
// }
//}

// load from cache first, then from disk
var ch *chart.Chart
chartPath := getChartPath(hr.Spec.Chart, hr.Spec.Version)
log.Info("chart path", "path", chartPath)
result, ok := chartCache.Get(chartPath)
if ok {
log.Info("load charts from cache", "path", chartPath)
ch = result.(*chart.Chart)
} else {
downloader := NewDownloader(d.SystemNamespace, d.InCluster.ToRestConfig(), d.Log)
chartPath, err := downloader.downloadChart(hr.Spec.Chart, hr.Spec.Version)
if err != nil {
return nil, err
}
log.Info("load charts from disk", "path", chartPath)
ch, err = loader.Load(chartPath)
if err != nil {
return nil, err
}
chartCache.SetDefault(chartPath, ch)

downloader := NewDownloader(d.SystemNamespace, d.InCluster.ToRestConfig(), d.Log)
chartPath, err := downloader.downloadChart(hr.Spec.Chart, hr.Spec.Version)
if err != nil {
return nil, err
}
log.Info("load charts from disk", "path", chartPath)
ch, err = loader.Load(chartPath)
if err != nil {
return nil, err
}

if req := ch.Metadata.Dependencies; req != nil {
Expand Down

0 comments on commit 3b8bc95

Please sign in to comment.