Skip to content

Commit

Permalink
add 95 percentile support (#246)
Browse files Browse the repository at this point in the history 
Co-authored-by: p8z <pav.zabka@gmail.com>
  • Loading branch information
@a-st @p-8-z
a-st and p-8-z authored Nov 3, 2023
1 parent 0e7256b commit 5600ac6
Show file tree
Hide file tree
Showing 20 changed files with 546 additions and 376 deletions.
74 changes: 41 additions & 33 deletions src/main/java/hudson/plugins/performance/PerformancePublisher.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
package hudson.plugins.performance;

import java.io.*;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
Expand All @@ -13,14 +18,13 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import edu.umd.cs.findbugs.annotations.NonNull;

import org.jenkinsci.Symbol;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
import org.kohsuke.stapler.DataBoundConstructor;
import org.kohsuke.stapler.DataBoundSetter;

import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.EnvVars;
import hudson.Extension;
import hudson.FilePath;
Expand Down Expand Up @@ -202,8 +206,8 @@ public PerformancePublisher(String sourceDataFiles,
boolean modeThroughput,
boolean showTrendGraphs,
/**
* Deprecated. Now use for support previous pipeline jobs.
*/
* Deprecated. Now use for support previous pipeline jobs.
*/
List<PerformanceReportParser> parsers) {
this.parsers = parsers;
this.sourceDataFiles = sourceDataFiles;
Expand Down Expand Up @@ -280,10 +284,10 @@ protected static List<FilePath> locatePerformanceReports(FilePath workspace, Str
throws IOException, InterruptedException {

// First use ant-style pattern
/*
* try { FilePath[] ret = workspace.list(includes); if (ret.length > 0) {
* return Arrays.asList(ret); }
*/
/*
* try { FilePath[] ret = workspace.list(includes); if (ret.length > 0) {
* return Arrays.asList(ret); }
*/
// Agoley : Possible fix, if we specify more than one result file pattern
try {
String[] parts = includes.split("\\s*[;:,]+\\s*");
Expand Down Expand Up @@ -470,7 +474,7 @@ private Collection<PerformanceReport> locatePerformanceReports(Run<?, ?> run, Fi
return Collections.emptyList();
}

logger.println("Performance: " + parser.getReportName() + " copying reports to master, files '" + files+"'");
logger.println("Performance: " + parser.getReportName() + " copying reports to master, files '" + files + "'");
List<File> localReports = copyReportsToMaster(run, logger, files, parser.getDescriptor().getDisplayName());
logger.println("Performance: " + parser.getReportName() + " parsing local reports '" + localReports
+ "'");
Expand All @@ -491,7 +495,7 @@ private void prepareParsers(Collection<PerformanceReportParser> performanceRepor
}

protected List<UriReport> getBuildUriReports(Run<?, ?> build, FilePath workspace, TaskListener listener,
List<PerformanceReportParser> parsers, boolean locatePerformanceReports)
List<PerformanceReportParser> parsers, boolean locatePerformanceReports)
throws IOException, InterruptedException {

List<UriReport> uriReports = new ArrayList<>();
Expand Down Expand Up @@ -632,6 +636,7 @@ private String appendStandardResultsStatsToXml(Collection<PerformanceReport> rep
xmlSB.append("<min>").append(report.getMin()).append("</min>\n\t");
xmlSB.append("<median>").append(report.getMedian()).append("</median>\n\t");
xmlSB.append("<ninetieth>").append(report.get90Line()).append("</ninetieth>\n\t");
xmlSB.append("<ninetyFifth>").append(report.get95Line()).append("</ninetyFifth>\n\t");
xmlSB.append("<max>").append(report.getMax()).append("</max>\n\t");
xmlSB.append("<httpCode>").append(report.getHttpCode()).append("</httpCode>\n\t");
xmlSB.append("<errors>").append(report.errorPercent()).append("</errors>\n");
Expand All @@ -644,7 +649,7 @@ private String appendStandardResultsStatsToXml(Collection<PerformanceReport> rep
// write report in xml, when checked Error Threshold comparison
private void writeErrorThresholdReportInXML(Run<?, ?> run, PerformanceReport performanceReport) throws IOException {
File xmlDirectory = createArchiveDirectoryIfMissing(run);

String glob = performanceReport.getReportFileName();
String[] arr = glob.split("/");

Expand Down Expand Up @@ -685,6 +690,10 @@ private String appendStatsToXml(List<UriReport> reports) {
percentileBuffer.append("\t\t<currentBuild90Line>").append(uriReport.get90Line()).append("</currentBuild90Line>\n");
percentileBuffer.append("\t</").append(uriReport.getStaplerUri()).append(">\n");

percentileBuffer.append("\t<").append(uriReport.getStaplerUri()).append(">\n");
percentileBuffer.append("\t\t<currentBuild95Line>").append(uriReport.get95Line()).append("</currentBuild95Line>\n");
percentileBuffer.append("\t</").append(uriReport.getStaplerUri()).append(">\n");

}

averageBuffer.append("</average>\n");
Expand Down Expand Up @@ -745,7 +754,7 @@ public void compareWithRelativeThreshold(Run<?, ?> run, FilePath workspace, Task
maxUriColumnWidth = Math.max(report.getStaplerUri().length(), maxUriColumnWidth);
}

printInfoAboutCompareBasedOn(logger, "%-"+maxUriColumnWidth+"s%20s%20s%20s%20s");
printInfoAboutCompareBasedOn(logger, "%-" + maxUriColumnWidth + "s%20s%20s%20s%20s");

compareUriReports(run,
currentUriReports,
Expand All @@ -756,7 +765,7 @@ public void compareWithRelativeThreshold(Run<?, ?> run, FilePath workspace, Task
(averageBuffer = new StringBuilder("<average>\n")),
(medianBuffer = new StringBuilder("<median>\n")),
(percentileBuffer = new StringBuilder("<percentile>\n")),
"%1$-"+maxUriColumnWidth+"s%2$20d%3$20d%4$20.0f%5$19.2f%%"
"%1$-" + maxUriColumnWidth + "s%2$20d%3$20d%4$20.0f%5$19.2f%%"
);

// close xml tags
Expand Down Expand Up @@ -827,7 +836,7 @@ private void calculateBuildStatus(Run<?, ?> run, PrintStream logger, String stap
}

private double calculateDiffInPercents(double value1, double value2) {
return Math.round(SafeMaths.safeDivide(value1 * 100, value2) * 100) / 100d;
return Math.round(SafeMaths.safeDivide(value1 * 100, value2) * 100) / 100d;
}

private boolean calculateRelativeFailedThresholdNegative(double relativeDiffPercent) {
Expand Down Expand Up @@ -859,24 +868,24 @@ private double calculateRelativeDiffInPercent(UriReport currentReport, UriReport
relativeDiffPercent = calculateDiffInPercents(relativeDiff, reportForComparison.getAverage());

logger.println(String.format(logFormat, reportForComparison.getStaplerUri(),
reportForComparison.getAverage(), currentReport.getAverage(),
relativeDiff, relativeDiffPercent));
reportForComparison.getAverage(), currentReport.getAverage(),
relativeDiff, relativeDiffPercent));

} else if (configType.equalsIgnoreCase("MRT")) {
relativeDiff = (double) currentReport.getMedian() - reportForComparison.getMedian();
relativeDiffPercent = calculateDiffInPercents(relativeDiff, reportForComparison.getMedian());

logger.println(String.format(logFormat, reportForComparison.getStaplerUri(),
reportForComparison.getMedian(), currentReport.getMedian(),
relativeDiff, relativeDiffPercent));
reportForComparison.getMedian(), currentReport.getMedian(),
relativeDiff, relativeDiffPercent));

} else if (configType.equalsIgnoreCase("PRT")) {
relativeDiff = (double) currentReport.get90Line() - reportForComparison.get90Line();
relativeDiffPercent = calculateDiffInPercents(relativeDiff, reportForComparison.get90Line());

logger.println(String.format(logFormat, reportForComparison.getStaplerUri(),
reportForComparison.get90Line(), currentReport.get90Line(),
relativeDiff, relativeDiffPercent));
reportForComparison.get90Line(), currentReport.get90Line(),
relativeDiff, relativeDiffPercent));
}
return relativeDiffPercent;
}
Expand Down Expand Up @@ -928,7 +937,7 @@ private void writeRelativeThresholdReportInXML(Run<?, ?> run, StringBuilder aver
private File createArchiveDirectoryIfMissing(Run<?, ?> run) {
File xmlDirectory = new File(run.getRootDir(), ARCHIVE_DIRECTORY);
if (!xmlDirectory.exists() && !xmlDirectory.mkdirs()) {
throw new IllegalStateException("Could not create archive directory "+xmlDirectory.getAbsolutePath());
throw new IllegalStateException("Could not create archive directory " + xmlDirectory.getAbsolutePath());
}
return xmlDirectory;
}
Expand Down Expand Up @@ -1015,15 +1024,15 @@ private void printInfoAboutCompareBasedOn(PrintStream logger, String logFormat)
if (configType.equalsIgnoreCase("ART")) {
logger.println("Average response time\n\n");
logger.println(String.format(logFormat,
"URI", "PrevBuildURIAvg", "CurrentBuildURIAvg", "RelativeDiff", "RelativeDiff(%)"));
"URI", "PrevBuildURIAvg", "CurrentBuildURIAvg", "RelativeDiff", "RelativeDiff(%)"));
} else if (configType.equalsIgnoreCase("MRT")) {
logger.println("Median response time\n\n");
logger.println(String.format(logFormat,
"URI", "PrevBuildURIMed", "CurrentBuildURIMed", "RelativeDiff", "RelativeDiff(%)"));
"URI", "PrevBuildURIMed", "CurrentBuildURIMed", "RelativeDiff", "RelativeDiff(%)"));
} else if (configType.equalsIgnoreCase("PRT")) {
logger.println("90 Percentile response time\n\n");
logger.println(String.format(logFormat,
"URI", "PrevBuildURI90%", "CurrentBuildURI90%", "RelativeDiff", "RelativeDiff(%)"));
"URI", "PrevBuildURI90%", "CurrentBuildURI90%", "RelativeDiff", "RelativeDiff(%)"));
}
}

Expand All @@ -1047,16 +1056,16 @@ public void evaluateInExpertMode(Run<?, ?> run, FilePath workspace, TaskListener
*/
ConstraintReport cr = new ConstraintReport(ceList, run.getParent().getBuilds().iterator().next(), persistConstraintLog);
logger.print(cr.getLoggerMsg());
/*
* Determine build result
*/
/*
* Determine build result
*/
run.setResult(cr.getBuildResult());

/*
* Write JUnit report back to workspace, for other post-build actions to pick up
*/
/*
* Write JUnit report back to workspace, for other post-build actions to pick up
*/
if (junitOutput != null && !junitOutput.isEmpty()) {
listener.getLogger().println("Performance: Generating JUnit output: "+junitOutput);
listener.getLogger().println("Performance: Generating JUnit output: " + junitOutput);
FilePath output = new FilePath(workspace, junitOutput);
FilePath parent = output.getParent();
if (parent != null) {
Expand Down Expand Up @@ -1170,7 +1179,6 @@ public static File[] getPerformanceReportDirectory(Run<?, ?> build, String parse
* Gets the Build object entered in the text box "Compare with nth Build"
*
* @param build, listener
*
* @return build object
*/

Expand Down
73 changes: 40 additions & 33 deletions src/main/java/hudson/plugins/performance/actions/PerformanceProjectAction.java
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,20 @@
package hudson.plugins.performance.actions;

import hudson.FilePath;
import hudson.model.AbstractBuild;
import hudson.model.AbstractProject;
import hudson.model.Action;
import hudson.model.Job;
import hudson.model.Run;
import hudson.plugins.performance.Messages;
import hudson.plugins.performance.PerformancePublisher;
import hudson.plugins.performance.PerformanceReportMap;
import hudson.plugins.performance.data.PerformanceReportPosition;
import hudson.plugins.performance.data.ReportValueSelector;
import hudson.plugins.performance.details.GraphConfigurationDetail;
import hudson.plugins.performance.details.TestSuiteReportDetail;
import hudson.plugins.performance.details.TrendReportDetail;
import hudson.plugins.performance.reports.PerformanceReport;
import hudson.plugins.performance.reports.UriReport;
import hudson.plugins.performance.reports.ThroughputReport;
import hudson.util.ChartUtil;
import hudson.util.ChartUtil.NumberOnlyBuildLabel;
import hudson.util.ColorPalette;
import hudson.util.DataSetBuilder;
import hudson.util.Graph;
import hudson.util.ShiftedCategoryAxis;
import java.awt.BasicStroke;
import java.awt.Color;
import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.GregorianCalendar;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.jfree.chart.ChartFactory;
import org.jfree.chart.JFreeChart;
import org.jfree.chart.axis.CategoryAxis;
Expand All @@ -47,16 +39,29 @@
import org.kohsuke.stapler.StaplerResponse;

import edu.umd.cs.findbugs.annotations.NonNull;

import java.awt.*;
import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import hudson.FilePath;
import hudson.model.AbstractBuild;
import hudson.model.AbstractProject;
import hudson.model.Action;
import hudson.model.Job;
import hudson.model.Run;
import hudson.plugins.performance.Messages;
import hudson.plugins.performance.PerformancePublisher;
import hudson.plugins.performance.PerformanceReportMap;
import hudson.plugins.performance.data.PerformanceReportPosition;
import hudson.plugins.performance.data.ReportValueSelector;
import hudson.plugins.performance.details.GraphConfigurationDetail;
import hudson.plugins.performance.details.TestSuiteReportDetail;
import hudson.plugins.performance.details.TrendReportDetail;
import hudson.plugins.performance.reports.PerformanceReport;
import hudson.plugins.performance.reports.ThroughputReport;
import hudson.plugins.performance.reports.UriReport;
import hudson.util.ChartUtil;
import hudson.util.ChartUtil.NumberOnlyBuildLabel;
import hudson.util.ColorPalette;
import hudson.util.DataSetBuilder;
import hudson.util.Graph;
import hudson.util.ShiftedCategoryAxis;

public class PerformanceProjectAction implements Action {

Expand Down Expand Up @@ -535,6 +540,8 @@ public void doRespondingTimeGraph(StaplerRequest request, StaplerResponse respon
Messages.ProjectAction_Average(), label);
dataSetBuilderAverage.add(performanceReport.get90Line(),
Messages.ProjectAction_Line90(), label);
dataSetBuilderAverage.add(performanceReport.get95Line(),
Messages.ProjectAction_Line95(), label);
}
nbBuildsToAnalyze--;
}
Expand Down
6 changes: 5 additions & 1 deletion src/main/java/hudson/plugins/performance/constraints/AbstractConstraint.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ protected double checkMetredValueforUriReport(Metric meteredValue, UriReport ur)
return (double) ur.getAverage();
case LINE90:
return (double) ur.get90Line();
case LINE95:
return (double) ur.get95Line();
case MEDIAN:
return (double) ur.getMedian();
case MINIMUM:
Expand All @@ -149,6 +151,8 @@ protected double checkMetredValueforPerfReport(Metric meteredValue, PerformanceR
return (double) pr.getAverage();
case LINE90:
return (double) pr.get90Line();
case LINE95:
return (double) pr.get95Line();
case MEDIAN:
return (double) pr.getMedian();
case MINIMUM:
Expand Down Expand Up @@ -214,7 +218,7 @@ protected void checkForDefectiveParams(List<? extends Run<?, ?>> builds) throws
}

public enum Metric {
AVERAGE("Average", false), MEDIAN("Median", false), LINE90("90% Line", false), MAXIMUM("Maximum", false), MINIMUM("Minimum", false), ERRORPRC("Error %", false);
AVERAGE("Average", false), MEDIAN("Median", false), LINE90("90% Line", false), LINE95("95% Line", false), MAXIMUM("Maximum", false), MINIMUM("Minimum", false), ERRORPRC("Error %", false);

private final String text;
private final boolean isSelected;
Expand Down
9 changes: 9 additions & 0 deletions src/main/java/hudson/plugins/performance/data/TaurusFinalStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ public class TaurusFinalStats implements Serializable {
private double perc0;
private double perc50;
private double perc90;
private double perc95;
private double perc100;

private Double testDuration;
Expand Down Expand Up @@ -77,10 +78,18 @@ public double getPerc90() {
return perc90;
}

public double getPerc95() {
return perc95;
}

public void setPerc90(double perc90) {
this.perc90 = perc90;
}

public void setPerc95(double perc95) {
this.perc95 = perc95;
}

public double getPerc0() {
return perc0;
}
Expand Down
Loading

0 comments on commit 5600ac6

Please sign in to comment.