Skip to content

Commit

Permalink
Optimize release notes generation to use GitHub Milestones (#13398)
Browse files Browse the repository at this point in the history 
Signed-off-by: Florent Poinsard <florent.poinsard@outlook.fr>
  • Loading branch information
@frouioui
frouioui committed Jul 26, 2023
1 parent 4c5a512 commit 0ff8986
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 260 deletions.
236 changes: 35 additions & 201 deletions go/tools/release-notes/release_notes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"regexp"
"sort"
"strings"
"sync"
"text/template"

"github.com/spf13/pflag"
Expand All @@ -40,24 +39,24 @@ type (

labels []label

author struct {
Login string `json:"login"`
pullRequestAuthor struct {
Login string
}

prInfo struct {
Labels labels `json:"labels"`
Number int `json:"number"`
Title string `json:"title"`
Author author `json:"author"`
pullRequestInformation struct {
Number int
Title string
Labels labels
Author pullRequestAuthor
}

prsByComponent = map[string][]prInfo
prsByComponent = map[string][]pullRequestInformation

prsByType = map[string]prsByComponent

sortedPRComponent struct {
Name string
PrInfos []prInfo
PrInfos []pullRequestInformation
}

sortedPRType struct {
Expand All @@ -82,7 +81,6 @@ type (

var (
releaseNotesPath = `changelog/`
numberOfThreads = 10
)

const (
Expand Down Expand Up @@ -188,61 +186,27 @@ func loadKnownIssues(release string) ([]knownIssue, error) {
return knownIssues, nil
}

func loadMergedPRs(from, to string) (prs []string, authors []string, commitCount int, err error) {
// load the git log with "author \t title \t parents"
out, err := execCmd("git", "log", `--pretty=format:%ae%x09%s%x09%P%x09%h`, fmt.Sprintf("%s..%s", from, to))

func loadMergedPRsAndAuthors(name string) (pris []pullRequestInformation, authors []string, err error) {
out, err := execCmd("gh", "pr", "list", "-s", "merged", "-S", fmt.Sprintf("milestone:%s", name), "--json", "number,title,labels,author", "--limit", "5000")
if err != nil {
return
}

return parseGitLog(string(out))
}

func parseGitLog(s string) (prs []string, authorCommits []string, commitCount int, err error) {
rx := regexp.MustCompile(`(.+)\t(.+)\t(.+)\t(.+)`)
mergePR := regexp.MustCompile(`Merge pull request #(\d+)`)
squashPR := regexp.MustCompile(`\(#(\d+)\)`)
authMap := map[string]string{} // here we will store email <-> gh user mappings
lines := strings.Split(s, "\n")
for _, line := range lines {
lineInfo := rx.FindStringSubmatch(line)
if len(lineInfo) != 5 {
log.Fatalf("failed to parse the output from git log: %s", line)
}
authorEmail := lineInfo[1]
title := lineInfo[2]
parents := lineInfo[3]
sha := lineInfo[4]
merged := mergePR.FindStringSubmatch(title)
if len(merged) == 2 {
// this is a merged PR. remember the PR #
prs = append(prs, merged[1])
continue
}

if len(parents) <= lengthOfSingleSHA {
// we have a single parent, and the commit counts
commitCount++
if _, exists := authMap[authorEmail]; !exists {
authMap[authorEmail] = sha
}
}

squashed := squashPR.FindStringSubmatch(title)
if len(squashed) == 2 {
// this is a merged PR. remember the PR #
prs = append(prs, squashed[1])
continue
}
err = json.Unmarshal(out, &pris)
if err != nil {
return nil, nil, err
}

for _, author := range authMap {
authorCommits = append(authorCommits, author)
// Get the full list of distinct PRs authors and sort them
authorMap := map[string]bool{}
for _, pri := range pris {
login := pri.Author.Login
if ok := authorMap[login]; !ok {
authors = append(authors, login)
authorMap[login] = true
}
}

sort.Strings(prs)
sort.Strings(authorCommits) // not really needed, but makes testing easier
sort.Strings(authors)

return
}
Expand All @@ -262,133 +226,10 @@ func execCmd(name string, arg ...string) ([]byte, error) {
return out, nil
}

func loadPRInfo(pr string) (prInfo, error) {
out, err := execCmd("gh", "pr", "view", pr, "--json", "title,number,labels,author")
if err != nil {
return prInfo{}, err
}
var prInfo prInfo
err = json.Unmarshal(out, &prInfo)
return prInfo, err
}

func loadAuthorInfo(sha string) (string, error) {
out, err := execCmd("gh", "api", "/repos/vitessio/vitess/commits/"+sha)
if err != nil {
return "", err
}
var prInfo prInfo
err = json.Unmarshal(out, &prInfo)
if err != nil {
return "", err
}
return prInfo.Author.Login, nil
}

type req struct {
isPR bool
key string
}

func loadAllPRs(prs, authorCommits []string) ([]prInfo, []string, error) {
errChan := make(chan error)
wgDone := make(chan bool)
prChan := make(chan req, len(prs)+len(authorCommits))
// fill the work queue
for _, s := range prs {
prChan <- req{isPR: true, key: s}
}
for _, s := range authorCommits {
prChan <- req{isPR: false, key: s}
}
close(prChan)

var prInfos []prInfo
var authors []string
fmt.Printf("Found %d merged PRs. Loading PR info", len(prs))
wg := sync.WaitGroup{}
mu := sync.Mutex{}

shouldLoad := func(in string) bool {
if in == "" {
return false
}
mu.Lock()
defer mu.Unlock()

for _, existing := range authors {
if existing == in {
return false
}
}
return true
}
addAuthor := func(in string) {
mu.Lock()
defer mu.Unlock()
authors = append(authors, in)
}
addPR := func(in prInfo) {
mu.Lock()
defer mu.Unlock()
prInfos = append(prInfos, in)
}

for i := 0; i < numberOfThreads; i++ {
wg.Add(1)
go func() {
// load meta data about PRs
defer wg.Done()

for b := range prChan {
fmt.Print(".")
if b.isPR {
prInfo, err := loadPRInfo(b.key)
if err != nil {
errChan <- err
break
}
addPR(prInfo)
continue
}
author, err := loadAuthorInfo(b.key)
if err != nil {
errChan <- err
break
}
if shouldLoad(author) {
addAuthor(author)
}

}
}()
}

go func() {
// wait for the loading to finish
wg.Wait()
close(wgDone)
}()

var err error
select {
case <-wgDone:
break
case err = <-errChan:
break
}

fmt.Println()

sort.Strings(authors)

return prInfos, authors, err
}

func groupPRs(prInfos []prInfo) prsByType {
func groupPRs(pris []pullRequestInformation) prsByType {
prPerType := prsByType{}

for _, info := range prInfos {
for _, info := range pris {
var typ, component string
for _, lbl := range info.Labels {
switch {
Expand Down Expand Up @@ -478,11 +319,11 @@ func getStringForKnownIssues(issues []knownIssue) (string, error) {
return buff.String(), nil
}

func groupAndStringifyPullRequest(pr []prInfo) (string, error) {
if len(pr) == 0 {
func groupAndStringifyPullRequest(pris []pullRequestInformation) (string, error) {
if len(pris) == 0 {
return "", nil
}
prPerType := groupPRs(pr)
prPerType := groupPRs(pris)
prStr, err := getStringForPullRequestInfos(prPerType)
if err != nil {
return "", err
Expand All @@ -492,14 +333,10 @@ func groupAndStringifyPullRequest(pr []prInfo) (string, error) {

func main() {
var (
from, versionName, summaryFile string
to = "HEAD"
versionName, summaryFile string
)
pflag.StringVarP(&from, "from", "f", "", "from sha/tag/branch")
pflag.StringVarP(&to, "to", "t", to, "to sha/tag/branch")
pflag.StringVarP(&versionName, "version", "v", "", "name of the version (has to be the following format: v11.0.0)")
pflag.StringVarP(&summaryFile, "summary", "s", "", "readme file on which there is a summary of the release")
pflag.IntVar(&numberOfThreads, "threads", numberOfThreads, "Define the number of threads used to fetch data from GitHub's API. Lower this number if you hit request limit errors.")
pflag.Parse()

// The -version flag must be of a valid format.
Expand Down Expand Up @@ -547,26 +384,23 @@ func main() {
releaseNotes.KnownIssues = knownIssuesStr

// changelog with pull requests
prs, authorCommits, commits, err := loadMergedPRs(from, to)
if err != nil {
log.Fatal(err)
}
prInfos, authors, err := loadAllPRs(prs, authorCommits)
prs, authors, err := loadMergedPRsAndAuthors(versionName)
if err != nil {
log.Fatal(err)
}
releaseNotes.ChangeLog, err = groupAndStringifyPullRequest(prInfos)

releaseNotes.ChangeLog, err = groupAndStringifyPullRequest(prs)
if err != nil {
log.Fatal(err)
}

// changelog metrics
if commits > 0 && len(authors) > 0 {
if len(prs) > 0 && len(authors) > 0 {
releaseNotes.ChangeMetrics = fmt.Sprintf(`
The release includes %d commits (excluding merges)
The release includes %d merged Pull Requests.
Thanks to all our contributors: @%s
`, commits, strings.Join(authors, ", @"))
`, len(prs), strings.Join(authors, ", @"))
}

if err := releaseNotes.generate(nil, nil); err != nil {
Expand Down
Loading

0 comments on commit 0ff8986

Please sign in to comment.