diff --git a/pom.xml b/pom.xml index b20c4e9c..bffb9a32 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ logstash hpi - 1.2.1-SNAPSHOT + 1.3.0-SNAPSHOT Logstash A Logstash agent to send Jenkins logs to a Logstash indexer. http://wiki.jenkins-ci.org/display/JENKINS/Logstash+Plugin @@ -108,6 +108,12 @@ junit 1.10 + + + org.jenkins-ci.plugins + structs + 1.2 + diff --git a/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java b/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java index 17fda0ff..2e4a0100 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java @@ -108,7 +108,7 @@ public DescriptorImpl getDescriptor() { // Method to encapsulate calls for unit-testing LogstashWriter getLogStashWriter(AbstractBuild build, OutputStream errorStream) { - return new LogstashWriter(build, errorStream); + return new LogstashWriter(build, errorStream, null); } /** diff --git a/src/main/java/jenkins/plugins/logstash/LogstashNotifier.java b/src/main/java/jenkins/plugins/logstash/LogstashNotifier.java index efa50fb0..722ecc93 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashNotifier.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashNotifier.java @@ -26,18 +26,25 @@ import hudson.Extension; import hudson.Launcher; +import hudson.FilePath; import hudson.model.BuildListener; +import hudson.model.TaskListener; import hudson.model.AbstractBuild; +import hudson.model.Run; import hudson.model.AbstractProject; +import hudson.model.Result; import hudson.tasks.BuildStepDescriptor; import hudson.tasks.BuildStepMonitor; import hudson.tasks.Notifier; import hudson.tasks.Publisher; +import jenkins.tasks.SimpleBuildStep; import java.io.OutputStream; import java.io.PrintStream; +import java.io.IOException; import org.kohsuke.stapler.DataBoundConstructor; +import org.jenkinsci.Symbol; /** * Post-build action to push build log to Logstash. @@ -45,7 +52,7 @@ * @author Rusty Gerard * @since 1.0.0 */ -public class LogstashNotifier extends Notifier { +public class LogstashNotifier extends Notifier implements SimpleBuildStep { public int maxLines; public boolean failBuild; @@ -58,16 +65,27 @@ public LogstashNotifier(int maxLines, boolean failBuild) { @Override public boolean perform(AbstractBuild build, Launcher launcher, BuildListener listener) { + return perform(build, listener); + } + + @Override + public void perform(Run run, FilePath workspace, Launcher launcher, + TaskListener listener) throws InterruptedException, IOException { + if (!perform(run, listener)) { + run.setResult(Result.FAILURE); + } + } + + private boolean perform(Run run, TaskListener listener) { PrintStream errorPrintStream = listener.getLogger(); - LogstashWriter logstash = getLogStashWriter(build, errorPrintStream); + LogstashWriter logstash = getLogStashWriter(run, errorPrintStream, listener); logstash.writeBuildLog(maxLines); - return !(failBuild && logstash.isConnectionBroken()); } // Method to encapsulate calls for unit-testing - LogstashWriter getLogStashWriter(AbstractBuild build, OutputStream errorStream) { - return new LogstashWriter(build, errorStream); + LogstashWriter getLogStashWriter(Run run, OutputStream errorStream, TaskListener listener) { + return new LogstashWriter(run, errorStream, listener); } public BuildStepMonitor getRequiredMonitorService() { @@ -80,7 +98,7 @@ public Descriptor getDescriptor() { return (Descriptor) super.getDescriptor(); } - @Extension + @Extension @Symbol("logstashSend") public static class Descriptor extends BuildStepDescriptor { @Override diff --git a/src/main/java/jenkins/plugins/logstash/LogstashWriter.java b/src/main/java/jenkins/plugins/logstash/LogstashWriter.java index 06bbc984..fbfe66ba 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashWriter.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashWriter.java @@ -26,6 +26,8 @@ import hudson.model.AbstractBuild; +import hudson.model.TaskListener; +import hudson.model.Run; import jenkins.model.Jenkins; import jenkins.plugins.logstash.persistence.BuildData; import jenkins.plugins.logstash.persistence.IndexerDaoFactory; @@ -52,15 +54,17 @@ public class LogstashWriter { final OutputStream errorStream; - final AbstractBuild build; + final Run build; + final TaskListener listener; final BuildData buildData; final String jenkinsUrl; final LogstashIndexerDao dao; private boolean connectionBroken; - public LogstashWriter(AbstractBuild build, OutputStream error) { + public LogstashWriter(Run run, OutputStream error, TaskListener listener) { this.errorStream = error != null ? error : System.err; - this.build = build; + this.build = run; + this.listener = listener; this.dao = this.getDaoOrNull(); if (this.dao == null) { this.jenkinsUrl = ""; @@ -69,7 +73,6 @@ public LogstashWriter(AbstractBuild build, OutputStream error) { this.jenkinsUrl = getJenkinsUrl(); this.buildData = getBuildData(); } - } /** @@ -131,7 +134,11 @@ LogstashIndexerDao getDao() throws InstantiationException { } BuildData getBuildData() { - return new BuildData(build, new Date()); + if (build instanceof AbstractBuild) { + return new BuildData((AbstractBuild) build, new Date()); + } else { + return new BuildData(build, new Date(), listener); + } } String getJenkinsUrl() { diff --git a/src/main/java/jenkins/plugins/logstash/persistence/BuildData.java b/src/main/java/jenkins/plugins/logstash/persistence/BuildData.java index 1db81bf4..fa09c375 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/BuildData.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/BuildData.java @@ -28,7 +28,10 @@ import hudson.model.Environment; import hudson.model.Result; import hudson.model.AbstractBuild; +import hudson.model.TaskListener; +import hudson.model.Run; import hudson.model.Node; +import hudson.model.Executor; import hudson.tasks.test.AbstractTestResultAction; import hudson.tasks.test.TestResult; @@ -42,6 +45,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.io.IOException; import net.sf.json.JSONObject; @@ -112,19 +116,9 @@ public TestData(Action action) { BuildData() {} + // Freestyle project build public BuildData(AbstractBuild build, Date currentTime) { - result = build.getResult() == null ? null : build.getResult().toString(); - id = build.getId(); - projectName = build.getProject().getName(); - displayName = build.getDisplayName(); - fullDisplayName = build.getFullDisplayName(); - description = build.getDescription(); - url = build.getUrl(); - - Action testResultAction = build.getAction(AbstractTestResultAction.class); - if (testResultAction != null) { - testResults = new TestData(testResultAction); - } + initData(build, currentTime); Node node = build.getBuiltOn(); if (node == null) { @@ -135,10 +129,7 @@ public BuildData(AbstractBuild build, Date currentTime) { buildLabel = StringUtils.isBlank(node.getLabelString()) ? "master" : node.getLabelString(); } - buildNum = build.getNumber(); // build.getDuration() is always 0 in Notifiers - buildDuration = currentTime.getTime() - build.getStartTimeInMillis(); - timestamp = DATE_FORMATTER.format(build.getTimestamp().getTime()); rootProjectName = build.getRootBuild().getProject().getName(); rootProjectDisplayName = build.getRootBuild().getDisplayName(); rootBuildNum = build.getRootBuild().getNumber(); @@ -166,6 +157,48 @@ public BuildData(AbstractBuild build, Date currentTime) { } } + // Pipeline project build + public BuildData(Run build, Date currentTime, TaskListener listener) { + initData(build, currentTime); + + Executor executor = build.getExecutor(); + if (executor == null) { + buildHost = "master"; + } else { + buildHost = StringUtils.isBlank(executor.getDisplayName()) ? "master" : executor.getDisplayName(); + } + + rootProjectName = projectName; + rootProjectDisplayName = displayName; + rootBuildNum = buildNum; + + try { + // TODO: sensitive variables are not filtered, c.f. https://stackoverflow.com/questions/30916085 + buildVariables = build.getEnvironment(listener); + } catch (Exception e) { + buildVariables = new HashMap(); + } + } + + private void initData(Run build, Date currentTime) { + result = build.getResult() == null ? null : build.getResult().toString(); + id = build.getId(); + projectName = build.getParent().getName(); + displayName = build.getDisplayName(); + fullDisplayName = build.getFullDisplayName(); + description = build.getDescription(); + url = build.getUrl(); + buildNum = build.getNumber(); + + Action testResultAction = build.getAction(AbstractTestResultAction.class); + if (testResultAction != null) { + testResults = new TestData(testResultAction); + } + + buildDuration = currentTime.getTime() - build.getStartTimeInMillis(); + timestamp = DATE_FORMATTER.format(build.getTimestamp().getTime()); + } + @Override public String toString() { Gson gson = new GsonBuilder().create(); diff --git a/src/main/java/jenkins/plugins/logstash/persistence/IndexerDaoFactory.java b/src/main/java/jenkins/plugins/logstash/persistence/IndexerDaoFactory.java index ffd79345..86786053 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/IndexerDaoFactory.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/IndexerDaoFactory.java @@ -52,6 +52,7 @@ public final class IndexerDaoFactory { indexerMap.put(IndexerType.RABBIT_MQ, RabbitMqDao.class); indexerMap.put(IndexerType.ELASTICSEARCH, ElasticSearchDao.class); indexerMap.put(IndexerType.SYSLOG, SyslogDao.class); + indexerMap.put(IndexerType.LOGSTASH, LogstashDao.class); INDEXER_MAP = Collections.unmodifiableMap(indexerMap); } diff --git a/src/main/java/jenkins/plugins/logstash/persistence/LogstashDao.java b/src/main/java/jenkins/plugins/logstash/persistence/LogstashDao.java new file mode 100644 index 00000000..153d94dd --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/persistence/LogstashDao.java @@ -0,0 +1,55 @@ +package jenkins.plugins.logstash.persistence; + +import com.cloudbees.syslog.Facility; +import com.cloudbees.syslog.MessageFormat; +import com.cloudbees.syslog.Severity; +import com.cloudbees.syslog.sender.UdpSyslogMessageSender; +import org.apache.http.impl.client.HttpClientBuilder; + +import java.io.*; +import java.net.Socket; + + +public class LogstashDao extends AbstractLogstashIndexerDao { + + final String logstashHost; + final int logstashPort; + Socket logstashClientSocket; + + public LogstashDao(String logstashHostString, int logstashPortInt, String indexKey, String username, String password) { + this(null, logstashHostString, logstashPortInt, indexKey, username, password); + } + + public LogstashDao(HttpClientBuilder factory, String logstashHostString, int logstashPortInt, String indexKey, String username, String password) { + super(logstashHostString, logstashPortInt, indexKey, username, password); + this.logstashHost = logstashHostString; + this.logstashPort = logstashPortInt; + } + + @Override + public void push(String data) throws IOException { + + try + { + logstashClientSocket = new Socket(logstashHost, logstashPort); + DataOutputStream outToServer = new DataOutputStream(logstashClientSocket.getOutputStream()); + BufferedReader inFromServer = new BufferedReader(new InputStreamReader(logstashClientSocket.getInputStream())); + outToServer.writeBytes(data); + logstashClientSocket.setSoTimeout(10000); + logstashClientSocket.close(); + outToServer.close(); + inFromServer.close(); + } + catch (Exception exc) + { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + exc.printStackTrace(pw); + throw new IOException(sw.toString()); + } + + } + + @Override + public IndexerType getIndexerType() { return IndexerType.LOGSTASH; } +} diff --git a/src/main/java/jenkins/plugins/logstash/persistence/LogstashIndexerDao.java b/src/main/java/jenkins/plugins/logstash/persistence/LogstashIndexerDao.java index 5bbe65d1..373b9c05 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/LogstashIndexerDao.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/LogstashIndexerDao.java @@ -40,7 +40,8 @@ static enum IndexerType { REDIS, RABBIT_MQ, ELASTICSEARCH, - SYSLOG + SYSLOG, + LOGSTASH } String getDescription(); diff --git a/src/test/java/jenkins/plugins/logstash/LogstashNotifierTest.java b/src/test/java/jenkins/plugins/logstash/LogstashNotifierTest.java index f3ed01fc..65d6a827 100644 --- a/src/test/java/jenkins/plugins/logstash/LogstashNotifierTest.java +++ b/src/test/java/jenkins/plugins/logstash/LogstashNotifierTest.java @@ -6,6 +6,10 @@ import hudson.Launcher; import hudson.model.BuildListener; import hudson.model.AbstractBuild; +import hudson.model.AbstractProject; +import hudson.model.TaskListener; +import hudson.model.Run; +import hudson.model.Result; import hudson.tasks.BuildStepMonitor; import hudson.tasks.Notifier; @@ -38,7 +42,7 @@ static class MockLogstashNotifier extends LogstashNotifier { } @Override - LogstashWriter getLogStashWriter(AbstractBuild build, OutputStream errorStream) { + LogstashWriter getLogStashWriter(Run run, OutputStream errorStream, TaskListener listener) { // Simulate bad Writer if(writer.isConnectionBroken()) { try { @@ -50,6 +54,24 @@ LogstashWriter getLogStashWriter(AbstractBuild build, OutputStream errorSt } } + static class MockRun

, R extends AbstractBuild> extends Run { + Result result; + + MockRun(P job) throws IOException { + super(job); + } + + @Override + public void setResult(Result r) { + result = r; + } + + @Override + public Result getResult() { + return result; + } + } + @Mock AbstractBuild mockBuild; @Mock LogstashWriter mockWriter; @Mock Launcher mockLauncher; @@ -58,7 +80,7 @@ LogstashWriter getLogStashWriter(AbstractBuild build, OutputStream errorSt ByteArrayOutputStream errorBuffer; PrintStream errorStream; LogstashNotifier notifier; - + MockRun mockRun; @Before public void before() throws Exception { @@ -66,6 +88,7 @@ public void before() throws Exception { errorStream = new PrintStream(errorBuffer, true); when(mockBuild.getLog(anyInt())).thenReturn(Arrays.asList("line 1", "line 2", "line 3")); + mockRun = new MockRun(mock(AbstractProject.class)); when(mockListener.getLogger()).thenReturn(errorStream); @@ -101,6 +124,21 @@ public void performSuccess() throws Exception { } + @Test + public void performStepSuccess() throws Exception { + // Unit under test + notifier.perform(mockRun, null, mockLauncher, mockListener); + + // Verify results + assertEquals("Result not null", null, mockRun.getResult()); + + verify(mockListener).getLogger(); + verify(mockWriter).writeBuildLog(3); + verify(mockWriter).isConnectionBroken(); + + assertEquals("Errors were written", "", errorBuffer.toString()); + } + @Test public void performBadWriterDoNotFailBuild() throws Exception { // Initialize mocks @@ -121,6 +159,26 @@ public void performBadWriterDoNotFailBuild() throws Exception { assertEquals("Error was not written", "Mocked Constructor failure", errorBuffer.toString()); } + @Test + public void performStepBadWriterDoNotFailBuild() throws Exception { + // Initialize mocks + when(mockWriter.isConnectionBroken()).thenReturn(true); + + notifier = new MockLogstashNotifier(3, false, mockWriter); + + // Unit under test + notifier.perform(mockRun, null, mockLauncher, mockListener); + + // Verify results + assertEquals("Result not null", null, mockRun.getResult()); + + verify(mockListener).getLogger(); + verify(mockWriter).writeBuildLog(3); + verify(mockWriter).isConnectionBroken(); + + assertEquals("Error was not written", "Mocked Constructor failure", errorBuffer.toString()); + } + @Test public void performBadWriterDoFailBuild() throws Exception { // Initialize mocks @@ -141,6 +199,25 @@ public void performBadWriterDoFailBuild() throws Exception { assertEquals("Error was not written", "Mocked Constructor failure", errorBuffer.toString()); } + @Test + public void performStepBadWriterDoFailBuild() throws Exception { + // Initialize mocks + when(mockWriter.isConnectionBroken()).thenReturn(true); + + notifier = new MockLogstashNotifier(3, true, mockWriter); + + // Unit under test + notifier.perform(mockRun, null, mockLauncher, mockListener); + + // Verify results + assertEquals("Result is not FAILURE", Result.FAILURE, mockRun.getResult()); + + verify(mockListener).getLogger(); + verify(mockWriter).writeBuildLog(3); + verify(mockWriter, times(2)).isConnectionBroken(); + assertEquals("Error was not written", "Mocked Constructor failure", errorBuffer.toString()); + } + @Test public void performWriteFailDoFailBuild() throws Exception { final String errorMsg = "[logstash-plugin]: Unable to serialize log data.\n" + diff --git a/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java b/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java index 456ae688..4f27e18b 100644 --- a/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java +++ b/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java @@ -35,7 +35,7 @@ static LogstashWriter createLogstashWriter(final AbstractBuild testBuild, final String url, final LogstashIndexerDao indexer, final BuildData data) { - return new LogstashWriter(testBuild, error) { + return new LogstashWriter(testBuild, error, null) { @Override LogstashIndexerDao getDao() throws InstantiationException { if (indexer == null) {