From 0a1abead1b9710d6a2a4dcbbf54e28426a90197a Mon Sep 17 00:00:00 2001
From: JSOBot <69814847+JSOBot@users.noreply.github.com>
Date: Fri, 24 Sep 2021 13:57:08 -0400
Subject: [PATCH 1/5] adding/updating file: push_benchmarks.sh

---
 push_benchmarks.sh | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 push_benchmarks.sh

diff --git a/push_benchmarks.sh b/push_benchmarks.sh
new file mode 100644
index 0000000..b36a3e7
--- /dev/null
+++ b/push_benchmarks.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+julia benchmark/send_comment_to_pr.jl -o $org -r $repo -p $pullrequest -c '**Starting benchmarks!**'
+
+LOCAL_BRANCH_NAME="temp_bmark"
+git checkout $LOCAL_BRANCH_NAME -- || true
+
+julia benchmark/$1 $repo
+
+if [ "$?" -eq "0" ] ; then
+    julia benchmark/send_comment_to_pr.jl -o $org -r $repo -p $pullrequest -c "Benchmark results" -g "gist.json"
+else
+    ERROR_LOGS="/home/jenkins/benchmarks/$org/$repo/${pullrequest}_${BUILD_NUMBER}_bmark_error.log"
+    julia benchmark/send_comment_to_pr.jl -o $org -r $repo -p $pullrequest -c "**An error occured while running $1**" -g $ERROR_LOGS
+fi

From 0392e39cd5033582130ddf855c24a0346b767fc0 Mon Sep 17 00:00:00 2001
From: JSOBot <69814847+JSOBot@users.noreply.github.com>
Date: Fri, 24 Sep 2021 13:57:09 -0400
Subject: [PATCH 2/5] adding/updating file: benchmark/run_benchmarks.jl

---
 benchmark/run_benchmarks.jl | 84 +++++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 benchmark/run_benchmarks.jl

diff --git a/benchmark/run_benchmarks.jl b/benchmark/run_benchmarks.jl
new file mode 100644
index 0000000..de3d96c
--- /dev/null
+++ b/benchmark/run_benchmarks.jl
@@ -0,0 +1,84 @@
+using Pkg
+bmark_dir = @__DIR__
+println(@__DIR__)
+Pkg.activate(bmark_dir)
+Pkg.instantiate()
+repo_name = string(split(ARGS[1], ".")[1])
+bmarkname = lowercase(repo_name)
+
+# if we are running these benchmarks from the git repository
+# we want to develop the package instead of using the release
+if isdir(joinpath(bmark_dir, "..", ".git"))
+  Pkg.develop(PackageSpec(url = joinpath(bmark_dir, "..")))
+end
+
+using DataFrames
+using GitHub
+using JLD2
+using JSON
+using PkgBenchmark
+using Plots
+
+using SolverBenchmark
+
+# NB: benchmarkpkg will run benchmarks/benchmarks.jl by default
+commit = benchmarkpkg(repo_name)  # current state of repository
+main = benchmarkpkg(repo_name, "main")
+judgement = judge(commit, main)
+
+commit_stats = bmark_results_to_dataframes(commit)
+main_stats = bmark_results_to_dataframes(main)
+judgement_stats = judgement_results_to_dataframes(judgement)
+
+export_markdown("judgement_$(bmarkname).md", judgement)
+export_markdown("main.md", main)
+export_markdown("$(bmarkname).md", commit)
+
+function profile_solvers_from_pkgbmark(stats::Dict{Symbol, DataFrame})
+  # guard against zero gctimes
+  costs =
+    [df -> df[!, :time], df -> df[!, :memory], df -> df[!, :gctime] .+ 1, df -> df[!, :allocations]]
+  profile_solvers(stats, costs, ["time", "memory", "gctime+1", "allocations"])
+end
+
+# extract stats for each benchmark to plot profiles
+# files_dict will be part of json_dict below
+files_dict = Dict{String, Any}()
+file_num = 1
+for k ∈ keys(judgement_stats)
+  global file_num
+  k_stats = Dict{Symbol, DataFrame}(:commit => commit_stats[k], :main => main_stats[k])
+  save_stats(k_stats, "$(bmarkname)_vs_main_$(k).jld2", force = true)
+
+  k_profile = profile_solvers_from_pkgbmark(k_stats)
+  savefig("profiles_commit_vs_main_$(k).svg")
+  # read contents of svg file to add to gist
+  k_svgfile = open("profiles_commit_vs_main_$(k).svg", "r") do fd
+    readlines(fd)
+  end
+  # file_num makes sure svg files appear before md files (added below)
+  files_dict["$(file_num)_$(k).svg"] = Dict{String, Any}("content" => join(k_svgfile))
+  file_num += 1
+end
+
+for mdfile ∈ [:judgement, :main, :commit]
+  global file_num
+  files_dict["$(file_num)_$(mdfile).md"] =
+    Dict{String, Any}("content" => "$(sprint(export_markdown, eval(mdfile)))")
+  file_num += 1
+end
+
+jldopen("$(bmarkname)_vs_main_judgement.jld2", "w") do file
+  file["jstats"] = judgement_stats
+end
+
+# json description of gist
+json_dict = Dict{String, Any}(
+  "description" => "$(repo_name) repository benchmark",
+  "public" => true,
+  "files" => files_dict,
+)
+
+open("gist.json", "w") do f
+  JSON.print(f, json_dict)
+end

From 16cc30df0dc332b4992405b8c6b5ba47bf1d45e3 Mon Sep 17 00:00:00 2001
From: JSOBot <69814847+JSOBot@users.noreply.github.com>
Date: Fri, 24 Sep 2021 13:57:09 -0400
Subject: [PATCH 3/5] adding/updating file: benchmark/send_comment_to_pr.jl

---
 benchmark/send_comment_to_pr.jl | 132 ++++++++++++++++++++++++++++++++
 1 file changed, 132 insertions(+)
 create mode 100644 benchmark/send_comment_to_pr.jl

diff --git a/benchmark/send_comment_to_pr.jl b/benchmark/send_comment_to_pr.jl
new file mode 100644
index 0000000..05ab5ba
--- /dev/null
+++ b/benchmark/send_comment_to_pr.jl
@@ -0,0 +1,132 @@
+using Pkg
+Pkg.activate("./benchmark/")
+Pkg.instantiate()
+
+using ArgParse
+using Git
+using GitHub
+using JSON
+
+DEFAULT_GIST_FILE_PATH = "gist.json"
+
+function parse_commandline()
+  s = ArgParseSettings()
+  @add_arg_table! s begin
+    "--org", "-o"
+    help = "Name of GitHub Organization"
+    arg_type = String
+    default = "JuliaSmoothOptimizers"
+    "--repo", "-r"
+    help = "The name of the repository on GitHub"
+    arg_type = String
+    required = true
+    "--pullrequest", "-p"
+    help = "An integer that corresponds to the pull request"
+    required = true
+    arg_type = Int
+    "--gist", "-g"
+    help = "specify this argument if you want to send a gist to the pullrequest. If this option is not specified, you must specify a comment"
+    arg_type = String
+    required = false
+    "--comment", "-c"
+    help = "Comment to post on the pull request"
+    arg_type = String
+    required = true
+  end
+
+  return parse_args(s, as_symbols = true)
+end
+
+function create_gist_from_json_file(myauth)
+  gistfile = DEFAULT_GIST_FILE_PATH
+  gist = begin
+    open(gistfile, "r") do f
+      return JSON.parse(f)
+    end
+  end
+  posted_gist = create_gist(params = gist, auth = myauth)
+  return posted_gist
+end
+
+function create_gist_from_log_file(gist_file, pullrequest_id, myauth)
+  file_content = ""
+  file = open(gist_file, "r")
+
+  file_lines = readlines(file)
+  line_number = findfirst(x -> !isnothing(match(r"ERROR:", x)), file_lines)
+  lines = !isnothing(line_number) ? file_lines[line_number:end] : [""]
+  for line in lines
+      file_content *= line*'\n'
+  end
+  close(file)
+
+  file_dict = Dict("$(pullrequest_id)_bmark_error.log" => Dict("content" => file_content))
+  gist = Dict{String,Any}("description" => "Benchmark logs",
+                            "public" => true,
+                            "files" => file_dict)
+
+  posted_gist = GitHub.create_gist(params = gist, auth = myauth)
+
+  return posted_gist
+end
+
+function get_repo(api::GitHub.GitHubWebAPI, org::String, repo_name::String; kwargs...)
+  my_params = Dict(:visibility => "all")
+  # return GitHub.repo(api, repo; params = my_params, kwargs...)
+  return Repo(GitHub.gh_get_json(api, "/repos/$(org)/$(repo_name)"; params = my_params, kwargs...))
+end
+
+function get_pull_request(
+  api::GitHub.GitHubWebAPI,
+  org::String,
+  repo::Repo,
+  pullrequest_id;
+  kwargs...,
+)
+  my_params = Dict(:sort => "popularity", :direction => "desc")
+  pull_request = PullRequest(
+    GitHub.gh_get_json(
+      api,
+      "/repos/$(org)/$(repo.name)/pulls/$(pullrequest_id)";
+      params = my_params,
+      kwargs...,
+    ),
+  )
+  return pull_request
+end
+
+function post_comment_to_pr(
+  org::String,
+  repo_name::String,
+  pullrequest_id::Int,
+  comment::String;
+  kwargs...,
+)
+  api = GitHub.DEFAULT_API
+  repo = get_repo(api, org, repo_name; kwargs...)
+  pull_request = get_pull_request(api, org, repo, pullrequest_id; kwargs...)
+  GitHub.create_comment(api, repo, pull_request, comment; kwargs...)
+end
+
+function main()
+  # Need to add GITHUB_AUTH to your .bashrc
+  myauth = GitHub.authenticate(ENV["GITHUB_AUTH"])
+  # parse the arguments: 
+  parsed_args = parse_commandline()
+  org = parsed_args[:org]
+  repo_name = parsed_args[:repo]
+  pullrequest_id = parsed_args[:pullrequest]
+  gist_file = parsed_args[:gist]
+  comment = parsed_args[:comment]
+
+  if !isnothing(gist_file) 
+      if gist_file == DEFAULT_GIST_FILE_PATH
+        comment = "$(comment): $(create_gist_from_json_file(myauth).html_url)"
+      else
+        comment = "$(comment): $(create_gist_from_log_file(gist_file, pullrequest_id, myauth).html_url)"
+      end
+  end
+  post_comment_to_pr(org, repo_name, pullrequest_id, comment; auth = myauth)
+end
+
+main()

From 7a5510e1c2e08149e0f7d44fdde700b0c3c5a32d Mon Sep 17 00:00:00 2001
From: JSOBot <69814847+JSOBot@users.noreply.github.com>
Date: Fri, 24 Sep 2021 13:57:10 -0400
Subject: [PATCH 4/5] adding/updating file: Jenkinsfile

---
 Jenkinsfile | 117 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 117 insertions(+)
 create mode 100644 Jenkinsfile

diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 0000000..5c56810
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,117 @@
+def bmarkFile = 'run_benchmarks.jl'
+pipeline {
+  agent any
+  environment {
+    REPO_EXISTS = fileExists "$repo"
+  }
+  options {
+    skipDefaultCheckout true
+  }
+  triggers {
+    GenericTrigger(
+     genericVariables: [
+        [
+            key: 'action', 
+            value: '$.action',
+            expressionType: 'JSONPath', //Optional, defaults to JSONPath
+            regexpFilter: '[^(created)]', //Optional, defaults to empty string
+            defaultValue: '' //Optional, defaults to empty string
+        ],
+        [
+            key: 'comment',
+            value: '$.comment.body',
+            expressionType: 'JSONPath', //Optional, defaults to JSONPath
+            regexpFilter: '', //Optional, defaults to empty string
+            defaultValue: '' //Optional, defaults to empty string
+        ],
+        [
+            key: 'org',
+            value: '$.organization.login',
+            expressionType: 'JSONPath', //Optional, defaults to JSONPath
+            regexpFilter: '', //Optional, defaults to empty string
+            defaultValue: 'JuliaSmoothOptimizers' //Optional, defaults to empty string
+        ],
+        [
+            key: 'pullrequest',
+            value: '$.issue.number',
+            expressionType: 'JSONPath', //Optional, defaults to JSONPath
+            regexpFilter: '[^0-9]', //Optional, defaults to empty string
+            defaultValue: '' //Optional, defaults to empty string
+        ],
+        [
+            key: 'repo',
+            value: '$.repository.name',
+            expressionType: 'JSONPath', //Optional, defaults to JSONPath
+            regexpFilter: '', //Optional, defaults to empty string
+            defaultValue: '' //Optional, defaults to empty string
+        ]
+     ],
+
+     causeString: 'Triggered on $comment',
+
+     token: "",
+
+     printContributedVariables: true,
+     printPostContent: true,
+
+     silentResponse: false,
+
+     regexpFilterText: '$comment',
+     regexpFilterExpression: '@JSOBot runbenchmarks'
+    )
+  }
+  stages {
+    stage('clone repo') {
+      when {
+        expression { REPO_EXISTS == 'false' }
+      }
+      steps {
+        sh 'git clone https://${GITHUB_AUTH}@github.com/$org/$repo.git'
+      }
+    }
+    stage('checkout on new branch') {
+      steps {
+        dir(WORKSPACE + "/$repo") {
+          sh '''
+            git clean -fd
+            git checkout main
+            git pull origin main
+            git fetch origin
+            LOCAL_BRANCH_NAME="temp_bmark"
+            git branch -D $LOCAL_BRANCH_NAME || true
+            git fetch origin pull/$pullrequest/head:$LOCAL_BRANCH_NAME
+            git checkout $LOCAL_BRANCH_NAME --
+          '''
+        }
+      }
+    }
+    stage('run benchmarks') {
+      steps {
+        script {
+          def data = env.comment.tokenize(' ')
+          if (data.size() > 2) {
+            bmarkFile = data.get(2);
+          }
+        }
+        dir(WORKSPACE + "/$repo") {
+        sh "mkdir -p $HOME/benchmarks/${org}/${repo}"
+        sh "qsub -N ${repo}_${pullrequest} -V -cwd -o $HOME/benchmarks/${org}/${repo}/${pullrequest}_${BUILD_NUMBER}_bmark_output.log -e $HOME/benchmarks/${org}/${repo}/${pullrequest}_${BUILD_NUMBER}_bmark_error.log push_benchmarks.sh $bmarkFile"
+        }   
+      }
+    }
+  }
+  post {
+    success {
+      echo "SUCCESS!"  
+    }
+    cleanup {
+      dir(WORKSPACE + "/$repo") {
+      sh 'printenv'
+      sh '''
+      git clean -fd
+      git checkout main
+      '''
+      }
+    }
+  }
+}

From dd82188d8a7b238535440b9508305ee1b67d989f Mon Sep 17 00:00:00 2001
From: MonssafToukal <32516773+MonssafToukal@users.noreply.github.com>
Date: Fri, 24 Sep 2021 13:58:07 -0400
Subject: [PATCH 5/5] Add token to jenkinsfile

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5c56810..0b938ac 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -49,7 +49,7 @@ pipeline {
 
      causeString: 'Triggered on $comment',
 
-     token: "",
+     token: "NLPModelsIpopt",
 
      printContributedVariables: true,
      printPostContent: true,