From f15ddd0857c06e22162acbf6dd09e2169e34c8be Mon Sep 17 00:00:00 2001 From: Martin Kosiba Date: Tue, 21 Jul 2015 05:22:59 -0700 Subject: [PATCH] Enable streaming BuckEvents to a remote entity. Summary: This adds an EventListener that serializes everything it sees to Json and uses a configurable backend to stream the data. The EventListener will give up after a couple of tries (assuming the network has gone down). Test Plan: Added integration test, checked manually: 1. Configure a project with remote_log_upload_uri = http://localhost:5555 2. nc -l 5555 3. run buck test --all 4. Objserve JSON in netcat output. --- docs/concept/buckconfig.soy | 14 + src/com/facebook/buck/cli/BUCK | 2 +- src/com/facebook/buck/cli/BuckConfig.java | 12 + src/com/facebook/buck/cli/Main.java | 20 ++ src/com/facebook/buck/event/BUCK | 1 + src/com/facebook/buck/event/listener/BUCK | 2 +- .../RemoteLogUploaderEventListener.java | 174 ++++++++++ src/com/facebook/buck/log/Logger.java | 15 +- src/com/facebook/buck/model/Pair.java | 1 - src/com/facebook/buck/rules/BUCK | 6 +- src/com/facebook/buck/util/BUCK | 19 +- .../facebook/buck/util/HttpScribeLogger.java | 113 ------- src/com/facebook/buck/util/ScribeLogger.java | 23 -- .../AbstractBuildEnvironmentDescription.java | 76 +++++ src/com/facebook/buck/util/environment/BUCK | 3 +- src/com/facebook/buck/util/network/BUCK | 22 ++ .../buck/util/network/BatchingLogger.java | 106 ++++++ .../{ => network}/BlockingHttpEndpoint.java | 2 +- .../buck/util/{ => network}/HttpEndpoint.java | 2 +- .../buck/util/network/HttpPutLogger.java | 68 ++++ .../buck/util/{ => network}/HttpResponse.java | 2 +- .../buck/util/network/RemoteLogger.java | 40 +++ .../util/network/RemoteLoggerFactory.java | 47 +++ test/com/facebook/buck/event/listener/BUCK | 4 +- .../RemoteLogUploaderEventListenerTest.java | 315 ++++++++++++++++++ .../buck/event/listener/integration/BUCK | 8 +- .../RemoteLogFactoryIntegrationTest.java | 128 +++++++ .../testdata/remote_log/.buckconfig | 2 + .../integration/testdata/remote_log/test/BUCK | 3 + test/com/facebook/buck/json/BUCK | 20 +- test/com/facebook/buck/json/HasJsonField.java | 79 +++++ test/com/facebook/buck/util/BUCK | 4 +- .../buck/util/HttpScribeLoggerTest.java | 55 --- test/com/facebook/buck/util/network/BUCK | 16 + .../buck/util/network/BatchingLoggerTest.java | 81 +++++ .../BlockingHttpEndpointTest.java | 2 +- .../buck/util/network/HttpPutLoggerTest.java | 60 ++++ third-party/java/jackson/BUCK | 1 + third-party/java/jetty/BUCK | 1 + 39 files changed, 1321 insertions(+), 228 deletions(-) create mode 100644 src/com/facebook/buck/event/listener/RemoteLogUploaderEventListener.java delete mode 100644 src/com/facebook/buck/util/HttpScribeLogger.java delete mode 100644 src/com/facebook/buck/util/ScribeLogger.java create mode 100644 src/com/facebook/buck/util/environment/AbstractBuildEnvironmentDescription.java create mode 100644 src/com/facebook/buck/util/network/BUCK create mode 100644 src/com/facebook/buck/util/network/BatchingLogger.java rename src/com/facebook/buck/util/{ => network}/BlockingHttpEndpoint.java (99%) rename src/com/facebook/buck/util/{ => network}/HttpEndpoint.java (94%) create mode 100644 src/com/facebook/buck/util/network/HttpPutLogger.java rename src/com/facebook/buck/util/{ => network}/HttpResponse.java (95%) create mode 100644 src/com/facebook/buck/util/network/RemoteLogger.java create mode 100644 src/com/facebook/buck/util/network/RemoteLoggerFactory.java create mode 100644 test/com/facebook/buck/event/listener/RemoteLogUploaderEventListenerTest.java create mode 100644 test/com/facebook/buck/event/listener/integration/RemoteLogFactoryIntegrationTest.java create mode 100644 test/com/facebook/buck/event/listener/integration/testdata/remote_log/.buckconfig create mode 100644 test/com/facebook/buck/event/listener/integration/testdata/remote_log/test/BUCK create mode 100644 test/com/facebook/buck/json/HasJsonField.java delete mode 100644 test/com/facebook/buck/util/HttpScribeLoggerTest.java create mode 100644 test/com/facebook/buck/util/network/BUCK create mode 100644 test/com/facebook/buck/util/network/BatchingLoggerTest.java rename test/com/facebook/buck/util/{ => network}/BlockingHttpEndpointTest.java (98%) create mode 100644 test/com/facebook/buck/util/network/HttpPutLoggerTest.java diff --git a/docs/concept/buckconfig.soy b/docs/concept/buckconfig.soy index 19736ef866b..c1302a6d0ed 100644 --- a/docs/concept/buckconfig.soy +++ b/docs/concept/buckconfig.soy @@ -403,6 +403,20 @@ inspection.

Will compress the traces with GZIP.

+{literal}
+[log]
+  remote_log_url = http://all.your.logs:500/are/belong/to/us
+
+
{/literal} + +

Will stream the build logs to the indicated address. Currently{sp} +http and https scheme URLs are supported. +The events are sent in batches using PUT requests. Each batch contains a +JSON array composing of the long entries.

+ +

The format of the individual log entries is not yet finalized and may +change without warning between Buck versions.

+ {call .section}{param title: 'ndk' /}{/call} This section defines properties to configure building native code against diff --git a/src/com/facebook/buck/cli/BUCK b/src/com/facebook/buck/cli/BUCK index fa071af9d49..1009af10095 100644 --- a/src/com/facebook/buck/cli/BUCK +++ b/src/com/facebook/buck/cli/BUCK @@ -117,7 +117,6 @@ java_immutables_library( '//src/com/facebook/buck/util:escaper', '//src/com/facebook/buck/util:exceptions', '//src/com/facebook/buck/util:io', - '//src/com/facebook/buck/util:network', '//src/com/facebook/buck/util:util', '//src/com/facebook/buck/util:watchman', '//src/com/facebook/buck/util/unit:unit', @@ -125,6 +124,7 @@ java_immutables_library( '//src/com/facebook/buck/util/environment:environment', '//src/com/facebook/buck/util/environment:env-filter', '//src/com/facebook/buck/util/environment:platform', + '//src/com/facebook/buck/util/network:network', '//src/com/facebook/buck/timing:timing', '//third-party/java/args4j:args4j', '//third-party/java/ddmlib:ddmlib', diff --git a/src/com/facebook/buck/cli/BuckConfig.java b/src/com/facebook/buck/cli/BuckConfig.java index cba25be1acf..4ad3664a596 100644 --- a/src/com/facebook/buck/cli/BuckConfig.java +++ b/src/com/facebook/buck/cli/BuckConfig.java @@ -69,6 +69,7 @@ import java.io.Reader; import java.net.InetAddress; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.net.UnknownHostException; import java.nio.file.Path; @@ -118,6 +119,13 @@ public class BuckConfig { private static final String DEFAULT_HTTP_CACHE_MODE = CacheMode.readwrite.name(); private static final String DEFAULT_HTTP_CACHE_TIMEOUT_SECONDS = "3"; + private static final Function TO_URI = new Function() { + @Override + public URI apply(String input) { + return URI.create(input); + } + }; + private final Config config; private final ImmutableMap aliasToBuildTargetMap; @@ -707,6 +715,10 @@ private boolean readCacheMode(String fieldName, String defaultValue) { return doStore; } + public Optional getRemoteLogUrl() { + return getValue("log", "remote_log_url").transform(TO_URI); + } + public Optional getValue(String sectionName, String propertyName) { return config.getValue(sectionName, propertyName); } diff --git a/src/com/facebook/buck/cli/Main.java b/src/com/facebook/buck/cli/Main.java index 25f4a436b5c..5cff8b6f561 100644 --- a/src/com/facebook/buck/cli/Main.java +++ b/src/com/facebook/buck/cli/Main.java @@ -29,6 +29,7 @@ import com.facebook.buck.event.listener.FileSerializationEventBusListener; import com.facebook.buck.event.listener.JavaUtilsLoggingBuildListener; import com.facebook.buck.event.listener.LoggingBuildListener; +import com.facebook.buck.event.listener.RemoteLogUploaderEventListener; import com.facebook.buck.event.listener.SimpleConsoleEventBusListener; import com.facebook.buck.event.listener.SuperConsoleEventBusListener; import com.facebook.buck.httpserver.WebServer; @@ -61,10 +62,12 @@ import com.facebook.buck.util.ProcessManager; import com.facebook.buck.util.ProjectFilesystemWatcher; import com.facebook.buck.util.PropertyFinder; +import com.facebook.buck.util.network.RemoteLoggerFactory; import com.facebook.buck.util.Verbosity; import com.facebook.buck.util.WatchmanWatcher; import com.facebook.buck.util.WatchmanWatcherException; import com.facebook.buck.util.concurrent.TimeSpan; +import com.facebook.buck.util.environment.BuildEnvironmentDescription; import com.facebook.buck.util.environment.DefaultExecutionEnvironment; import com.facebook.buck.util.environment.EnvironmentFilter; import com.facebook.buck.util.environment.ExecutionEnvironment; @@ -92,6 +95,7 @@ import java.io.OutputStream; import java.io.PrintStream; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.net.URLClassLoader; import java.nio.file.Path; @@ -585,6 +589,7 @@ public int runMainWithExitCode( rootRepository.getBuckConfig(), webServer, clock, + executionEnvironment, console, consoleListener, rootRepository.getKnownBuildRuleTypes(), @@ -889,6 +894,7 @@ private ImmutableList addEventListeners( BuckConfig config, Optional webServer, Clock clock, + ExecutionEnvironment executionEnvironment, Console console, AbstractConsoleEventBusListener consoleEventBusListener, KnownBuildRuleTypes knownBuildRuleTypes, @@ -913,6 +919,20 @@ private ImmutableList addEventListeners( loadListenersFromBuckConfig(eventListenersBuilder, projectFilesystem, config); + Optional remoteLogUrl = config.getRemoteLogUrl(); + ImmutableMap environmentExtraData = ImmutableMap.of(); + if (remoteLogUrl.isPresent()) { + eventListenersBuilder.add( + new RemoteLogUploaderEventListener( + objectMapper, + RemoteLoggerFactory.create(remoteLogUrl.get(), objectMapper), + BuildEnvironmentDescription.of( + executionEnvironment, + config.getArtifactCacheModes(), + environmentExtraData) + )); + } + JavacOptions javacOptions = new JavaBuckConfig(config) .getDefaultJavacOptions(new ProcessExecutor(console)); diff --git a/src/com/facebook/buck/event/BUCK b/src/com/facebook/buck/event/BUCK index 2119d728501..4b8c54f2d2b 100644 --- a/src/com/facebook/buck/event/BUCK +++ b/src/com/facebook/buck/event/BUCK @@ -10,6 +10,7 @@ java_immutables_library( '//src/com/facebook/buck/model:model', '//src/com/facebook/buck/timing:timing', '//src/com/facebook/buck/util:exceptions', + '//src/com/facebook/buck/util:util', '//src/com/facebook/buck/util/concurrent:concurrent', ], provided_deps = [ diff --git a/src/com/facebook/buck/event/listener/BUCK b/src/com/facebook/buck/event/listener/BUCK index efd47499e0f..77f38737416 100644 --- a/src/com/facebook/buck/event/listener/BUCK +++ b/src/com/facebook/buck/event/listener/BUCK @@ -24,7 +24,7 @@ java_library( '//src/com/facebook/buck/util:constants', '//src/com/facebook/buck/util:exceptions', '//src/com/facebook/buck/util:io', - '//src/com/facebook/buck/util:network', + '//src/com/facebook/buck/util/network:network', '//src/com/facebook/buck/util:util', '//third-party/java/guava:guava', '//third-party/java/jackson:jackson-annotations', diff --git a/src/com/facebook/buck/event/listener/RemoteLogUploaderEventListener.java b/src/com/facebook/buck/event/listener/RemoteLogUploaderEventListener.java new file mode 100644 index 00000000000..642103e6eee --- /dev/null +++ b/src/com/facebook/buck/event/listener/RemoteLogUploaderEventListener.java @@ -0,0 +1,174 @@ +/* + * Copyright 2015-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.event.listener; + +import com.facebook.buck.cli.CommandEvent; +import com.facebook.buck.event.BuckEvent; +import com.facebook.buck.event.BuckEventListener; +import com.facebook.buck.log.Logger; +import com.facebook.buck.model.BuildId; +import com.facebook.buck.rules.BuildRuleEvent; +import com.facebook.buck.util.environment.BuildEnvironmentDescription; +import com.facebook.buck.util.network.BlockingHttpEndpoint; +import com.facebook.buck.util.network.RemoteLogger; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.eventbus.Subscribe; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Handles uploading events to a remote log service. + */ +public class RemoteLogUploaderEventListener implements BuckEventListener { + + private static final Logger LOG = Logger.get(RemoteLogUploaderEventListener.class); + @VisibleForTesting + static final int MAX_FAILURE_COUNT = 3; + + private final RemoteLogger remoteLogger; + private final ObjectMapper mapper; + private final BuildEnvironmentDescription buildEnvironmentDescription; + private final Set> pendingUploads; + private final AtomicInteger failureCount = new AtomicInteger(0); + private int sentEventsCount = 0; + + public RemoteLogUploaderEventListener( + ObjectMapper objectMapper, + RemoteLogger remoteLogger, + BuildEnvironmentDescription buildEnvironmentDescription) { + this.mapper = objectMapper; + this.buildEnvironmentDescription = buildEnvironmentDescription; + this.remoteLogger = remoteLogger; + this.pendingUploads = new HashSet<>(); + } + + @Subscribe + public void buckEvent(final BuckEvent event) { + putInSink(event.getBuildId(), event); + } + + private void putInSink(BuildId buildId, Object object) { + if (failureCount.get() > MAX_FAILURE_COUNT) { + return; + } + sentEventsCount++; + + ObjectNode jsonNode = mapper.valueToTree(object); + jsonNode.put("@class", object.getClass().getCanonicalName()); + jsonNode.put("buildId", buildId.toString()); + + Optional> upload = remoteLogger.log(jsonNode.toString()); + if (upload.isPresent()) { + registerPendingUpload(upload.get()); + } + } + + private void registerPendingUpload(final ListenableFuture upload) { + synchronized (pendingUploads) { + pendingUploads.add(upload); + } + Futures.addCallback( + upload, + new FutureCallback() { + @Override + public void onSuccess(Void result) { + failureCount.set(0); + onDone(); + } + + @Override + public void onFailure(Throwable t) { + onDone(); + LOG.info(t, "Failed uploading event to the remote log."); + if (failureCount.incrementAndGet() == MAX_FAILURE_COUNT) { + LOG.info("Maximum number of log upload failures reached, giving up."); + } + } + + private void onDone() { + synchronized (pendingUploads) { + pendingUploads.remove(upload); + } + } + }); + } + + @Subscribe + public void commandStarted(CommandEvent.Started started) { + putInSink( + started.getBuildId(), + ImmutableMap.of( + "type", "CommandStartedAux", + "environment", buildEnvironmentDescription + ) + ); + } + + @Subscribe + public void commandFinished(CommandEvent.Finished finished) { + putInSink( + finished.getBuildId(), + ImmutableMap.of( + "type", "CommandFinishedAux", + "environment", buildEnvironmentDescription, + "eventsCount", sentEventsCount + 1 + ) + ); + } + + @Subscribe + public void ruleStarted(BuildRuleEvent.Started started) { + putInSink( + started.getBuildId(), + ImmutableMap.of( + "type", "BuildRuleStartedAux", + "buildRule", started.getBuildRule(), + "deps", started.getBuildRule().getDeps() + )); + } + + @Override + public synchronized void outputTrace(BuildId buildId) throws InterruptedException { + ImmutableSet> uploads; + ListenableFuture drain = remoteLogger.close(); + synchronized (pendingUploads) { + pendingUploads.add(drain); + uploads = ImmutableSet.copyOf(pendingUploads); + pendingUploads.clear(); + } + try { + Futures.successfulAsList(uploads).get( + BlockingHttpEndpoint.DEFAULT_COMMON_TIMEOUT_MS * 100, + TimeUnit.MILLISECONDS); + } catch (ExecutionException|TimeoutException e) { + LOG.info(e, "Failed uploading remaining log data to remote server"); + } + } +} diff --git a/src/com/facebook/buck/log/Logger.java b/src/com/facebook/buck/log/Logger.java index 04580da9f39..5c69be46ee1 100644 --- a/src/com/facebook/buck/log/Logger.java +++ b/src/com/facebook/buck/log/Logger.java @@ -255,10 +255,11 @@ public void info(String message) * If the format string is invalid or the arguments are insufficient, an error will be logged and execution * will continue. * + * @param exception an exception associated with the warning being logged * @param format a format string compatible with String.format() * @param args arguments for the format string */ - public void info(String format, Object... args) + public void info(@Nullable Throwable exception, String format, Object... args) { if (logger.isLoggable(INFO)) { String message; @@ -269,10 +270,20 @@ public void info(String format, Object... args) logger.log(SEVERE, illegalFormatMessageFor("INFO", format, args), e); message = rawMessageFor(format, args); } - logger.info(message); + logger.log(INFO, message, exception); } } + /** + * Logs a message at INFO level. + * + * @param format a format string compatible with String.format() + * @param args arguments for the format string + */ + public void info(String format, Object... args) { + info(null, format, args); + } + /** * Logs a message at WARN level. * diff --git a/src/com/facebook/buck/model/Pair.java b/src/com/facebook/buck/model/Pair.java index 048d81c7563..89c4b35ab40 100644 --- a/src/com/facebook/buck/model/Pair.java +++ b/src/com/facebook/buck/model/Pair.java @@ -79,5 +79,4 @@ public int compare(Pair o1, Pair o2) { } }; } - } diff --git a/src/com/facebook/buck/rules/BUCK b/src/com/facebook/buck/rules/BUCK index 315dcc9a5f6..1cbb226a274 100644 --- a/src/com/facebook/buck/rules/BUCK +++ b/src/com/facebook/buck/rules/BUCK @@ -220,13 +220,13 @@ java_immutables_library( '//src/com/facebook/buck/test/selectors:selectors', '//src/com/facebook/buck/timing:timing', '//src/com/facebook/buck/util:constants', - '//src/com/facebook/buck/util/collect:collect', '//src/com/facebook/buck/util:exceptions', - '//src/com/facebook/buck/util/hash:hash', - '//src/com/facebook/buck/util:network', '//src/com/facebook/buck/util:util', + '//src/com/facebook/buck/util/collect:collect', + '//src/com/facebook/buck/util/hash:hash', '//src/com/facebook/buck/util/concurrent:concurrent', '//src/com/facebook/buck/util/environment:environment', + '//src/com/facebook/buck/util/network:network', '//src/com/facebook/buck/zip:steps', '//src/com/facebook/buck/zip:unzip', '//third-party/java/commons-compress:commons-compress', diff --git a/src/com/facebook/buck/util/BUCK b/src/com/facebook/buck/util/BUCK index 4bd0ad253e7..e8f8a613897 100644 --- a/src/com/facebook/buck/util/BUCK +++ b/src/com/facebook/buck/util/BUCK @@ -62,23 +62,6 @@ java_immutables_library( visibility = [ 'PUBLIC' ], ) -NETWORK_SRCS = [ - 'BlockingHttpEndpoint.java', - 'HttpResponse.java', - 'HttpEndpoint.java', -] -java_library( - name = 'network', - srcs = NETWORK_SRCS, - deps = [ - ':exceptions', - '//third-party/java/guava:guava', - '//third-party/java/jsr:jsr305', - '//src/com/facebook/buck/log:api', - ], - visibility = [ 'PUBLIC' ], -) - ESCAPER_SRCS = [ 'Escaper.java', 'WindowsCreateProcessEscape.java', @@ -124,7 +107,7 @@ java_immutables_library( name = 'util', srcs = glob( includes = ['*.java'], - excludes = CONSTANT_SRCS + ESCAPER_SRCS + EXCEPTION_SRCS + IO_SRCS + NETWORK_SRCS + + excludes = CONSTANT_SRCS + ESCAPER_SRCS + EXCEPTION_SRCS + IO_SRCS + WATCHMAN_WATCHER_SRCS), deps = [ ':constants', diff --git a/src/com/facebook/buck/util/HttpScribeLogger.java b/src/com/facebook/buck/util/HttpScribeLogger.java deleted file mode 100644 index fb54e3f4a7d..00000000000 --- a/src/com/facebook/buck/util/HttpScribeLogger.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2015-present Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.facebook.buck.util; - -import com.google.common.base.Function; -import com.google.common.base.Throwables; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; - -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.util.List; - -import javax.annotation.Nullable; - -public class HttpScribeLogger implements ScribeLogger { - - public static final String ENDPOINT = "https://interngraph.intern.facebook.com/scribe/log"; - public static final int MAX_REQUEST_SIZE_BYTES = 1024 * 1024; - public static final int MAX_PARALLEL_REQUESTS = 5; - - private static final String APP_ID = "480696962007533"; - private static final String TOKEN = "AeNzhrutjcZ0ew5LBlo"; - - private final HttpEndpoint httpEndpoint; - private final int maxRequestSizeBytes; - - public HttpScribeLogger( - HttpEndpoint httpEndpoint, - int maxRequestSizeBytes) { - this.httpEndpoint = httpEndpoint; - this.maxRequestSizeBytes = maxRequestSizeBytes; - } - - private ListenableFuture toVoid(ListenableFuture future) { - return Futures.transform( - future, - new Function() { - @Nullable - @Override - public Void apply(T input) { - return null; - } - }); - } - - private ListenableFuture submit(String category, StringBuilder lines) { - String header = - String.format( - "app=%s&token=%s&category=%s&loglines=", - APP_ID, - TOKEN, - category); - return toVoid(httpEndpoint.post(header + lines)); - } - - @Override - public ListenableFuture log(String category, Iterable lines) { - List> responses = Lists.newArrayList(); - - StringBuilder concatenatedLines = new StringBuilder(); - - for (String line : lines) { - StringBuilder toAdd = new StringBuilder(); - - // If we've already added a line to the request, then add a line terminator before we - // add this one. - if (concatenatedLines.length() > 0) { - toAdd.append('\n'); - } - - // Add the URL encoded line. - try { - toAdd.append(URLEncoder.encode(line, "UTF-8")); - } catch (UnsupportedEncodingException e) { - return Futures.immediateFailedFuture(Throwables.propagate(e)); - } - - // If adding this line would push us over the request size limit, dispatch the current - // request, and start a new one. - if (concatenatedLines.length() + toAdd.length() >= maxRequestSizeBytes) { - responses.add(submit(category, concatenatedLines)); - concatenatedLines = new StringBuilder(); - } - - concatenatedLines.append(toAdd); - } - - // If there is anything remaining in the current request, dispatch it. - if (concatenatedLines.length() > 0) { - responses.add(submit(category, concatenatedLines)); - } - - // Flatten the list of void futures to single void future. - return toVoid(Futures.allAsList(responses)); - } - -} diff --git a/src/com/facebook/buck/util/ScribeLogger.java b/src/com/facebook/buck/util/ScribeLogger.java deleted file mode 100644 index 850854de3bd..00000000000 --- a/src/com/facebook/buck/util/ScribeLogger.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2015-present Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.facebook.buck.util; - -import com.google.common.util.concurrent.ListenableFuture; - -public interface ScribeLogger { - ListenableFuture log(String category, Iterable lines); -} diff --git a/src/com/facebook/buck/util/environment/AbstractBuildEnvironmentDescription.java b/src/com/facebook/buck/util/environment/AbstractBuildEnvironmentDescription.java new file mode 100644 index 00000000000..966bc11dac6 --- /dev/null +++ b/src/com/facebook/buck/util/environment/AbstractBuildEnvironmentDescription.java @@ -0,0 +1,76 @@ +/* + * Copyright 2015-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.util.environment; + +import com.facebook.buck.util.TriState; +import com.facebook.buck.util.immutables.BuckStyleImmutable; +import com.google.common.base.StandardSystemProperty; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import org.immutables.value.Value; + +@Value.Immutable +@BuckStyleImmutable +abstract class AbstractBuildEnvironmentDescription { + public static final int PROTOCOL_VERSION = 1; + + public abstract String getUser(); + public abstract String getHostname(); + public abstract String getOs(); + public abstract int getAvailableCores(); + public abstract long getSystemMemoryMb(); + public abstract TriState getBuckDirty(); + public abstract String getBuckCommit(); + public abstract String getJavaVersion(); + public abstract ImmutableList getCacheModes(); + public abstract ImmutableMap getExtraData(); + @Value.Default + public int getJsonProtocolVersion() { + return PROTOCOL_VERSION; + } + + public static BuildEnvironmentDescription of( + ExecutionEnvironment executionEnvironment, + ImmutableList cacheModes, + ImmutableMap extraData) { + TriState buckDirty; + String dirty = executionEnvironment.getProperty("buck.git_dirty", "unknown"); + if (dirty.equals("1")) { + buckDirty = TriState.TRUE; + } else if (dirty.equals("0")) { + buckDirty = TriState.FALSE; + } else { + buckDirty = TriState.UNSPECIFIED; + } + + return BuildEnvironmentDescription.builder() + .setUser(executionEnvironment.getUsername()) + .setHostname(executionEnvironment.getHostname()) + .setOs( + executionEnvironment + .getPlatform().getPrintableName().toLowerCase().replace(' ', '_')) + .setAvailableCores(executionEnvironment.getAvailableCores()) + .setSystemMemoryMb(executionEnvironment.getTotalMemoryInMb()) + .setBuckDirty(buckDirty) + .setBuckCommit(executionEnvironment.getProperty("buck.git_commit", "unknown")) + .setJavaVersion(StandardSystemProperty.JAVA_VM_VERSION.value()) + .setCacheModes(cacheModes) + .setExtraData(extraData) + .build(); + } +} diff --git a/src/com/facebook/buck/util/environment/BUCK b/src/com/facebook/buck/util/environment/BUCK index c936b71f7b5..28e35bb413a 100644 --- a/src/com/facebook/buck/util/environment/BUCK +++ b/src/com/facebook/buck/util/environment/BUCK @@ -26,7 +26,7 @@ java_library( ], ) -java_library( +java_immutables_library( name = 'environment', srcs = glob(['*.java'], excludes = FILTER_SRCS + PLATFORM_SRCS), deps = [ @@ -34,6 +34,7 @@ java_library( ':platform', '//third-party/java/guava:guava', '//src/com/facebook/buck/util:io', + '//src/com/facebook/buck/util:util', ], visibility = [ 'PUBLIC', diff --git a/src/com/facebook/buck/util/network/BUCK b/src/com/facebook/buck/util/network/BUCK new file mode 100644 index 00000000000..bdcd8a39fc7 --- /dev/null +++ b/src/com/facebook/buck/util/network/BUCK @@ -0,0 +1,22 @@ +java_library( + name = 'network', + srcs = glob(['*.java']), + deps = [ + '//third-party/java/guava:guava', + '//third-party/java/jackson:jackson-core', + '//third-party/java/jackson:jackson-databind', + '//third-party/java/jsr:jsr305', + '//src/com/facebook/buck/log:api', + '//src/com/facebook/buck/util:exceptions', + '//src/com/facebook/buck/util:util', + ], + visibility = [ + '//src/com/facebook/buck/cli:cli', + '//src/com/facebook/buck/event/listener:listener', + '//src/com/facebook/buck/rules:rules', + '//src/com/facebook/buck/util:util', + '//test/com/facebook/buck/util:util', + '//test/com/facebook/buck/event/listener:listener', + '//test/com/facebook/buck/util/network:network', + ], +) diff --git a/src/com/facebook/buck/util/network/BatchingLogger.java b/src/com/facebook/buck/util/network/BatchingLogger.java new file mode 100644 index 00000000000..0795fa4e6d7 --- /dev/null +++ b/src/com/facebook/buck/util/network/BatchingLogger.java @@ -0,0 +1,106 @@ +/* + * Copyright 2015-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.util.network; + +import com.google.common.base.Objects; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; + +/** + * Common functionality for uploading log entries in batches. + */ +public abstract class BatchingLogger implements RemoteLogger { + + public static final int DEFAULT_MIN_BATCH_SIZE = 1024 * 128; // This is pretty arbitrary. + + protected static class BatchEntry { + private final String line; + + public BatchEntry(String line) { + this.line = line; + } + + public String getLine() { + return line; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof BatchEntry)) { + return false; + } + + BatchEntry that = (BatchEntry) other; + return line.equals(that.line); + } + + @Override + public int hashCode() { + return Objects.hashCode(line); + } + + @Override + public String toString() { + return String.format("BatchEntry(%s)", line); + } + } + + private ImmutableList.Builder batch; + private int currentBatchSize; + private final int minBatchSize; + + public BatchingLogger(int minBatchSize) { + this.batch = ImmutableList.builder(); + this.currentBatchSize = 0; + this.minBatchSize = minBatchSize; + } + + public BatchingLogger() { + this(DEFAULT_MIN_BATCH_SIZE); + } + + @Override + public final Optional> log(String jsonBlob) { + batch.add(new BatchEntry(jsonBlob)); + currentBatchSize += jsonBlob.length(); + if (currentBatchSize >= minBatchSize) { + return Optional.of(sendBatch()); + } + return Optional.absent(); + } + + @Override + public final ListenableFuture close() { + return sendBatch(); + } + + private ListenableFuture sendBatch() { + ImmutableList toSend = batch.build(); + batch = ImmutableList.builder(); + currentBatchSize = 0; + if (toSend.isEmpty()) { + return Futures.immediateFuture(null); + } else { + return logMultiple(toSend); + } + } + + protected abstract ListenableFuture logMultiple(ImmutableCollection data); +} diff --git a/src/com/facebook/buck/util/BlockingHttpEndpoint.java b/src/com/facebook/buck/util/network/BlockingHttpEndpoint.java similarity index 99% rename from src/com/facebook/buck/util/BlockingHttpEndpoint.java rename to src/com/facebook/buck/util/network/BlockingHttpEndpoint.java index 785442c73c6..1ef2e81670c 100644 --- a/src/com/facebook/buck/util/BlockingHttpEndpoint.java +++ b/src/com/facebook/buck/util/network/BlockingHttpEndpoint.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations * under the License. */ -package com.facebook.buck.util; +package com.facebook.buck.util.network; import com.facebook.buck.log.Logger; import com.google.common.annotations.VisibleForTesting; diff --git a/src/com/facebook/buck/util/HttpEndpoint.java b/src/com/facebook/buck/util/network/HttpEndpoint.java similarity index 94% rename from src/com/facebook/buck/util/HttpEndpoint.java rename to src/com/facebook/buck/util/network/HttpEndpoint.java index ee12c32f996..bf8674d7706 100644 --- a/src/com/facebook/buck/util/HttpEndpoint.java +++ b/src/com/facebook/buck/util/network/HttpEndpoint.java @@ -14,7 +14,7 @@ * under the License. */ -package com.facebook.buck.util; +package com.facebook.buck.util.network; import com.google.common.util.concurrent.ListenableFuture; diff --git a/src/com/facebook/buck/util/network/HttpPutLogger.java b/src/com/facebook/buck/util/network/HttpPutLogger.java new file mode 100644 index 00000000000..0608c67d2d0 --- /dev/null +++ b/src/com/facebook/buck/util/network/HttpPutLogger.java @@ -0,0 +1,68 @@ +/* + * Copyright 2015-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.util.network; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.google.common.collect.ImmutableCollection; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; + +import java.io.IOException; +import java.io.StringWriter; + +import javax.annotation.Nullable; + +/** + * Uploads log entries in batches as newline-separated JSON strings in a HTTP PUT request. + */ +public class HttpPutLogger extends BatchingLogger { + + private final HttpEndpoint endpoint; + private final ObjectMapper objectMapper; + + public HttpPutLogger(HttpEndpoint endpoint, ObjectMapper objectMapper) { + this.endpoint = endpoint; + this.objectMapper = objectMapper; + } + + @Override + protected ListenableFuture logMultiple(ImmutableCollection data) { + StringWriter stringWriter = new StringWriter(); + try { + JsonGenerator jsonGenerator = objectMapper.getJsonFactory().createJsonGenerator(stringWriter); + jsonGenerator.writeStartArray(); + for (BatchEntry entry : data) { + jsonGenerator.writeRawValue(entry.getLine()); + } + jsonGenerator.writeEndArray(); + jsonGenerator.close(); + return Futures.transform( + endpoint.post(stringWriter.toString()), + new Function() { + @Nullable + @Override + public Void apply(HttpResponse input) { + return null; + } + }); + } catch (IOException e) { + return Futures.immediateFailedFuture(e); + } + } +} diff --git a/src/com/facebook/buck/util/HttpResponse.java b/src/com/facebook/buck/util/network/HttpResponse.java similarity index 95% rename from src/com/facebook/buck/util/HttpResponse.java rename to src/com/facebook/buck/util/network/HttpResponse.java index 4e66cfa14ee..6c88b952835 100644 --- a/src/com/facebook/buck/util/HttpResponse.java +++ b/src/com/facebook/buck/util/network/HttpResponse.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations * under the License. */ -package com.facebook.buck.util; +package com.facebook.buck.util.network; public class HttpResponse { diff --git a/src/com/facebook/buck/util/network/RemoteLogger.java b/src/com/facebook/buck/util/network/RemoteLogger.java new file mode 100644 index 00000000000..a6a7a0ef2d3 --- /dev/null +++ b/src/com/facebook/buck/util/network/RemoteLogger.java @@ -0,0 +1,40 @@ +/* + * Copyright 2015-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.util.network; + +import com.google.common.base.Optional; +import com.google.common.util.concurrent.ListenableFuture; + +/** + * Implemented by classes providing the functionality to upload log data to remote entities. + */ +public interface RemoteLogger { + /** + * @param jsonBlob data to upload. + * @return {@link Optional#absent()} if the data has merely been buffered, a + * {@link ListenableFuture} representing the upload otherwise. + */ + Optional> log(String jsonBlob); + + /** + * If the underlying logger employs buffering this signals it to upload whatever remaining + * information it had stored. + * + * @return future representing the forced upload. + */ + ListenableFuture close(); +} diff --git a/src/com/facebook/buck/util/network/RemoteLoggerFactory.java b/src/com/facebook/buck/util/network/RemoteLoggerFactory.java new file mode 100644 index 00000000000..0a5488d4378 --- /dev/null +++ b/src/com/facebook/buck/util/network/RemoteLoggerFactory.java @@ -0,0 +1,47 @@ +/* + * Copyright 2015-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.util.network; + +import com.facebook.buck.util.HumanReadableException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.net.MalformedURLException; +import java.net.URI; + +public abstract class RemoteLoggerFactory { + + public static final int MAX_PARALLEL_REQUESTS = 5; + + /** + * @param uri URI to create the logger for. + * @return The {@link RemoteLogger} instance matching the given scheme. + */ + public static RemoteLogger create(URI uri, ObjectMapper objectMapper) { + + try { + return new HttpPutLogger( + new BlockingHttpEndpoint( + uri.toString(), + MAX_PARALLEL_REQUESTS, + BlockingHttpEndpoint.DEFAULT_COMMON_TIMEOUT_MS), + objectMapper); + } catch (MalformedURLException e) { + throw new HumanReadableException(e, "Don't know how to upload logs to %s", uri); + } + } + +} diff --git a/test/com/facebook/buck/event/listener/BUCK b/test/com/facebook/buck/event/listener/BUCK index b568d9ec2c5..a3f91959396 100644 --- a/test/com/facebook/buck/event/listener/BUCK +++ b/test/com/facebook/buck/event/listener/BUCK @@ -28,11 +28,13 @@ java_test( '//src/com/facebook/buck/util:constants', '//src/com/facebook/buck/util:io', '//src/com/facebook/buck/util:exceptions', + '//src/com/facebook/buck/util:util', '//src/com/facebook/buck/util/environment:environment', '//src/com/facebook/buck/util/environment:platform', - '//src/com/facebook/buck/util:network', + '//src/com/facebook/buck/util/network:network', '//test/com/facebook/buck/event:testutil', '//test/com/facebook/buck/java:testutil', + '//test/com/facebook/buck/json:testutil', '//test/com/facebook/buck/model:BuildTargetFactory', '//test/com/facebook/buck/rules:testutil', '//test/com/facebook/buck/step:testutil', diff --git a/test/com/facebook/buck/event/listener/RemoteLogUploaderEventListenerTest.java b/test/com/facebook/buck/event/listener/RemoteLogUploaderEventListenerTest.java new file mode 100644 index 00000000000..063be9501c4 --- /dev/null +++ b/test/com/facebook/buck/event/listener/RemoteLogUploaderEventListenerTest.java @@ -0,0 +1,315 @@ +/* + * Copyright 2015-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.event.listener; + +import static com.facebook.buck.event.TestEventConfigerator.configureTestEventAtTime; +import static org.hamcrest.junit.MatcherAssert.assertThat; + +import com.facebook.buck.cli.CommandEvent; +import com.facebook.buck.event.AbstractBuckEvent; +import com.facebook.buck.event.BuckEventBusFactory; +import com.facebook.buck.json.HasJsonField; +import com.facebook.buck.model.BuildTargetFactory; +import com.facebook.buck.rules.BuildEvent; +import com.facebook.buck.rules.BuildRule; +import com.facebook.buck.rules.BuildRuleEvent; +import com.facebook.buck.rules.BuildRuleResolver; +import com.facebook.buck.rules.FakeBuildRule; +import com.facebook.buck.rules.RuleKey; +import com.facebook.buck.rules.SourcePathResolver; +import com.facebook.buck.util.TriState; +import com.facebook.buck.util.environment.BuildEnvironmentDescription; +import com.facebook.buck.util.network.RemoteLogger; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.eventbus.EventBus; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; + +import org.hamcrest.Matcher; +import org.hamcrest.Matchers; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class RemoteLogUploaderEventListenerTest { + + private static class TestRemoteLogger implements RemoteLogger { + List loggedEntries = new ArrayList<>(); + + @Override + public Optional> log(String jsonBlob) { + loggedEntries.add(jsonBlob); + return Optional.absent(); + } + + @Override + public ListenableFuture close() { + return Futures.immediateFuture(null); + } + } + + private ObjectMapper objectMapper = new ObjectMapper(); + + private static final BuildEnvironmentDescription BUILD_ENVIRONMENT_DESCRIPTION = + BuildEnvironmentDescription.builder() + .setUser("user") + .setHostname("hostname") + .setOs("os") + .setAvailableCores(1) + .setSystemMemoryMb(1024) + .setBuckDirty(TriState.UNSPECIFIED) + .setBuckCommit("unknown") + .setJavaVersion("1.7") + .build(); + + @Test + @SuppressWarnings("unchecked") + public void testAuxEventSentInAdditionToOriginal() throws Exception { + FluentIterable nodes = getJsonNodes( + CommandEvent.started( + "cook", + ImmutableList.of("--dinner"), + true)); + + assertThat( + nodes, + Matchers.containsInAnyOrder( + hasJsonField("type", "CommandStarted"), + hasJsonField("type", "CommandStartedAux") + )); + } + + @Test + public void testGivesUpAfterConsecutiveFailures() throws Exception { + final AtomicInteger logCount = new AtomicInteger(0); + RemoteLogger consistentlyFailingLogger = new RemoteLogger() { + + @Override + public Optional> log(String jsonBlob) { + logCount.incrementAndGet(); + return Optional.of(Futures.immediateFailedFuture(new Throwable())); + } + + @Override + public ListenableFuture close() { + return Futures.immediateFuture(null); + } + }; + RemoteLogUploaderEventListener eventListener = + new RemoteLogUploaderEventListener( + objectMapper, + consistentlyFailingLogger, + BUILD_ENVIRONMENT_DESCRIPTION); + EventBus eventBus = new EventBus(); + eventBus.register(eventListener); + + final int maxRepetitions = 2 * RemoteLogUploaderEventListener.MAX_FAILURE_COUNT; + for (int i = 0; i < maxRepetitions; ++i) { + eventBus.post(configureTestEvent(BuildEvent.started(ImmutableSet.of("")))); + } + + assertThat(logCount.get(), Matchers.lessThan(maxRepetitions)); + } + + @Test + public void testDoesNotGiveUpAfterSparseFailures() throws Exception { + final AtomicInteger logCount = new AtomicInteger(0); + RemoteLogger failingEveryOtherTimeLogger = new RemoteLogger() { + + @Override + public Optional> log(String jsonBlob) { + System.out.println(jsonBlob); + int i = logCount.incrementAndGet(); + if (i % 2 == 0) { + return Optional.of(Futures.immediateFailedFuture(new Throwable())); + } else { + return Optional.of(Futures.immediateFuture(null)); + } + } + + @Override + public ListenableFuture close() { + return Futures.immediateFuture(null); + } + }; + RemoteLogUploaderEventListener eventListener = + new RemoteLogUploaderEventListener( + objectMapper, + failingEveryOtherTimeLogger, + BUILD_ENVIRONMENT_DESCRIPTION); + EventBus eventBus = new EventBus(); + eventBus.register(eventListener); + + final int maxRepetitions = 2 * RemoteLogUploaderEventListener.MAX_FAILURE_COUNT; + for (int i = 0; i < maxRepetitions; ++i) { + eventBus.post(configureTestEvent(BuildEvent.started(ImmutableSet.of("")))); + } + + assertThat(logCount.get(), Matchers.equalTo(maxRepetitions)); + } + + private Matcher hasJsonField( + String fieldName, + Matcher matcher) { + return new HasJsonField(objectMapper, fieldName, matcher); + } + + private Matcher hasJsonField( + String fieldName, + T value) { + return new HasJsonField( + objectMapper, + fieldName, + Matchers.equalTo(objectMapper.valueToTree(value))); + } + + private Matcher hasJsonField(String fieldName) { + return hasJsonField(fieldName, Matchers.anything()); + } + + private FluentIterable getJsonNodes(AbstractBuckEvent event) { + TestRemoteLogger logger = new TestRemoteLogger(); + RemoteLogUploaderEventListener eventListener = + new RemoteLogUploaderEventListener( + objectMapper, + logger, + BUILD_ENVIRONMENT_DESCRIPTION); + EventBus eventBus = new EventBus(); + eventBus.register(eventListener); + + AbstractBuckEvent configuredEvent = configureTestEvent(event); + eventBus.post(configuredEvent); + + return FluentIterable.from(logger.loggedEntries) + .transform( + new Function() { + @Override + public JsonNode apply(String input) { + try { + return objectMapper.readTree(input); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + } + + private void singleSchemaTest(AbstractBuckEvent event, Matcher eventMatcher) { + FluentIterable loggedNodes = getJsonNodes(event); + Matcher nodeMatcher = Matchers.allOf( + hasJsonField("buildId", BuckEventBusFactory.BUILD_ID_FOR_TEST.toString()), + hasJsonField("type"), + eventMatcher + ); + assertThat(loggedNodes, Matchers.hasItem(nodeMatcher)); + } + + private final Matcher buildRuleShapeMatcher = Matchers.allOf( + hasJsonField("type"), + hasJsonField("name") + ); + + private final Matcher eventShapeMatcher = Matchers.allOf( + hasJsonField("timestamp"), + hasJsonField("threadId") + ); + + /** + * The purpose of this is to document the set of properties that are stable-ish, if you need + * to remove or rename any of these you'll want to give the community and the team a heads up + * and increment the protocol version in + * {@link com.facebook.buck.util.environment.AbstractBuildEnvironmentDescription}. + */ + @Test + public void testSchema() throws Exception { + singleSchemaTest( + CommandEvent.started("cook", ImmutableList.of("--dinner"), true), + Matchers.allOf( + hasJsonField("type", "CommandStarted"), + eventShapeMatcher, + hasJsonField("args", Arrays.asList("--dinner")), + hasJsonField("commandName", "cook"), + hasJsonField("daemon", true) + )); + + SourcePathResolver resolver = new SourcePathResolver(new BuildRuleResolver()); + FakeBuildRule buildRule = createFakeBuildRule("//build:rule1", resolver, new RuleKey("aaaa")); + FakeBuildRule buildRule2 = createFakeBuildRule( + "//build:rule2", + resolver, + ImmutableSortedSet.of(buildRule), + new RuleKey("aaaa")); + + singleSchemaTest( + BuildRuleEvent.started(buildRule), + Matchers.allOf( + hasJsonField("type", "BuildRuleStarted"), + eventShapeMatcher, + hasJsonField( + "buildRule", + Matchers.allOf( + hasJsonField("type", "fake_build_rule"), + hasJsonField("name", "//build:rule1") + )))); + + singleSchemaTest( + BuildRuleEvent.started(buildRule2), + Matchers.allOf( + hasJsonField("type", "BuildRuleStartedAux"), + hasJsonField("buildRule", buildRuleShapeMatcher), + hasJsonField("buildRule", hasJsonField("name", "//build:rule2")), + hasJsonField("deps", Matchers.contains(buildRuleShapeMatcher)), + hasJsonField("deps", Matchers.contains(hasJsonField("name", "//build:rule1"))) + )); + } + + private static T configureTestEvent(T event) { + return configureTestEventAtTime(event, 0L, TimeUnit.SECONDS, 1L); + } + + private FakeBuildRule createFakeBuildRule( + String target, + SourcePathResolver resolver, + RuleKey ruleKey) { + return createFakeBuildRule(target, resolver, ImmutableSortedSet.of(), ruleKey); + } + + private FakeBuildRule createFakeBuildRule( + String target, + SourcePathResolver resolver, + ImmutableSortedSet deps, + RuleKey ruleKey) { + FakeBuildRule fakeBuildRule = new FakeBuildRule( + BuildTargetFactory.newInstance(target), + resolver, + deps); + fakeBuildRule.setRuleKey(ruleKey); + return fakeBuildRule; + } +} diff --git a/test/com/facebook/buck/event/listener/integration/BUCK b/test/com/facebook/buck/event/listener/integration/BUCK index dafa9080229..eedec52ade4 100644 --- a/test/com/facebook/buck/event/listener/integration/BUCK +++ b/test/com/facebook/buck/event/listener/integration/BUCK @@ -5,9 +5,13 @@ java_test( '//src/com/facebook/buck/event/listener:listener', ], deps = [ - '//third-party/java/hamcrest:hamcrest', - '//third-party/java/junit:junit', '//src/com/facebook/buck/event/listener:listener', + '//test/com/facebook/buck/json:testutil', '//test/com/facebook/buck/testutil/integration:integration', + '//third-party/java/guava:guava', + '//third-party/java/hamcrest:hamcrest', + '//third-party/java/jackson:jackson', + '//third-party/java/jetty:jetty', + '//third-party/java/junit:junit', ], ) diff --git a/test/com/facebook/buck/event/listener/integration/RemoteLogFactoryIntegrationTest.java b/test/com/facebook/buck/event/listener/integration/RemoteLogFactoryIntegrationTest.java new file mode 100644 index 00000000000..b583caee8a0 --- /dev/null +++ b/test/com/facebook/buck/event/listener/integration/RemoteLogFactoryIntegrationTest.java @@ -0,0 +1,128 @@ +/* + * Copyright 2015-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.event.listener.integration; + +import static org.junit.Assert.assertThat; + +import com.facebook.buck.json.HasJsonField; +import com.facebook.buck.testutil.integration.DebuggableTemporaryFolder; +import com.facebook.buck.testutil.integration.HttpdForTests; +import com.facebook.buck.testutil.integration.ProjectWorkspace; +import com.facebook.buck.testutil.integration.TestDataHelper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ListMultimap; +import com.google.common.io.CharStreams; + +import org.eclipse.jetty.http.HttpMethod; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.AbstractHandler; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.List; +import java.util.Set; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +public class RemoteLogFactoryIntegrationTest { + + private static class PostRequestsHandler extends AbstractHandler { + + private final ListMultimap responsePathToContentsMap = + ArrayListMultimap.create(); + + @Override + public void handle( + String target, + Request request, + HttpServletRequest httpServletRequest, + HttpServletResponse httpServletResponse) throws IOException, ServletException { + if (!HttpMethod.POST.is(request.getMethod())) { + return; + } + String contents = CharStreams.toString(new InputStreamReader(request.getInputStream())); + responsePathToContentsMap.put(request.getUri().toString(), contents); + request.setHandled(true); + } + + public Set getPutRequestsPaths() { + return responsePathToContentsMap.keySet(); + } + + public List getContents(String path) throws IllegalArgumentException { + return responsePathToContentsMap.get(path); + } + } + @Rule + public DebuggableTemporaryFolder temporaryFolder = new DebuggableTemporaryFolder(); + + private ObjectMapper objectMapper = new ObjectMapper(); + private HttpdForTests httpd; + private PostRequestsHandler putRequestsHandler; + + @Before + public void startHttpd() throws Exception { + httpd = new HttpdForTests(); + putRequestsHandler = new PostRequestsHandler(); + httpd.addHandler(putRequestsHandler); + httpd.start(); + } + + @After + public void shutdownHttpd() throws Exception { + httpd.close(); + } + + @Test + public void testResultsUpload() throws Exception { + ProjectWorkspace workspace = TestDataHelper.createProjectWorkspaceForScenario( + this, "remote_log", temporaryFolder); + // We won't verify the workspace as there are no "expected" files. + workspace.setUp(); + workspace.replaceFileContents(".buckconfig", "", httpd.getUri("").toString()); + + ProjectWorkspace.ProcessResult result = workspace.runBuckCommand("build", "//test:test"); + result.assertSuccess("buck project should exit cleanly"); + + assertThat(putRequestsHandler.getPutRequestsPaths(), Matchers.contains("/")); + + List contents = putRequestsHandler.getContents("/"); + ImmutableList.Builder eventsBuilder = ImmutableList.builder(); + for (String batch : contents) { + JsonNode jsonNode = objectMapper.readTree(batch); + assertThat(jsonNode, Matchers.hasProperty("array", Matchers.is(true))); + eventsBuilder.addAll(jsonNode); + } + ImmutableList events = eventsBuilder.build(); + assertThat(events, + Matchers.hasItem( + new HasJsonField( + objectMapper, + "type", + Matchers.equalTo(objectMapper.valueToTree("CommandStarted"))))); + } +} diff --git a/test/com/facebook/buck/event/listener/integration/testdata/remote_log/.buckconfig b/test/com/facebook/buck/event/listener/integration/testdata/remote_log/.buckconfig new file mode 100644 index 00000000000..5a0cf8542ef --- /dev/null +++ b/test/com/facebook/buck/event/listener/integration/testdata/remote_log/.buckconfig @@ -0,0 +1,2 @@ +[log] + remote_log_url = diff --git a/test/com/facebook/buck/event/listener/integration/testdata/remote_log/test/BUCK b/test/com/facebook/buck/event/listener/integration/testdata/remote_log/test/BUCK new file mode 100644 index 00000000000..29584fe3a1b --- /dev/null +++ b/test/com/facebook/buck/event/listener/integration/testdata/remote_log/test/BUCK @@ -0,0 +1,3 @@ +java_library( + name = 'test', +) diff --git a/test/com/facebook/buck/json/BUCK b/test/com/facebook/buck/json/BUCK index 499a1a4317d..e11a03cae8e 100644 --- a/test/com/facebook/buck/json/BUCK +++ b/test/com/facebook/buck/json/BUCK @@ -1,6 +1,24 @@ +STANDARD_TEST_SRCS = [ + '*Test.java', +] + +java_library( + name = 'testutil', + srcs = glob(['*.java'], excludes = STANDARD_TEST_SRCS), + deps = [ + '//third-party/java/guava:guava', + '//third-party/java/hamcrest:hamcrest', + '//third-party/java/junit:junit', + '//third-party/java/jackson:jackson', + ], + visibility = [ + 'PUBLIC', + ], +) + java_test( name = 'json', - srcs = glob(['*.java']), + srcs = glob(STANDARD_TEST_SRCS), source_under_test = [ '//src/com/facebook/buck/json:json', '//src/com/facebook/buck/json:raw_parser', diff --git a/test/com/facebook/buck/json/HasJsonField.java b/test/com/facebook/buck/json/HasJsonField.java new file mode 100644 index 00000000000..c1b736002e5 --- /dev/null +++ b/test/com/facebook/buck/json/HasJsonField.java @@ -0,0 +1,79 @@ +/* + * Copyright 2015-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.hamcrest.Matcher; + +import java.io.IOException; + +/** + * Matches an {@link JsonNode} which has the specified field. + */ +public class HasJsonField extends BaseMatcher { + private ObjectMapper objectMapper; + private String fieldName; + private Matcher valueMatcher; + + public HasJsonField( + ObjectMapper objectMapper, + String fieldName, + Matcher valueMatcher) { + this.objectMapper = objectMapper; + this.fieldName = fieldName; + this.valueMatcher = valueMatcher; + } + + @Override + public boolean matches(Object o) { + if (o instanceof JsonNode) { + JsonNode node = (JsonNode) o; + if (!node.has(fieldName)) { + return false; + } + return valueMatcher.matches(node.get(fieldName)); + } + return false; + } + + @Override + public void describeTo(Description description) { + description.appendText("JSON object with field [" + fieldName + "] "); + description.appendDescriptionOf(valueMatcher); + description.appendText("\n"); + } + + @Override + public void describeMismatch(Object item, Description description) { + if (item instanceof JsonNode) { + JsonNode node = (JsonNode) item; + try { + description.appendText("was ").appendText( + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString( + node)); + } catch (IOException e) { + super.describeMismatch(item, description); + } + } else { + super.describeMismatch(item, description); + } + } +} diff --git a/test/com/facebook/buck/util/BUCK b/test/com/facebook/buck/util/BUCK index 76b4e369d3e..c7185c5b8c4 100644 --- a/test/com/facebook/buck/util/BUCK +++ b/test/com/facebook/buck/util/BUCK @@ -24,8 +24,8 @@ java_test( '//src/com/facebook/buck/util:constants', '//src/com/facebook/buck/util:exceptions', '//src/com/facebook/buck/util:io', - '//src/com/facebook/buck/util:network', '//src/com/facebook/buck/util:util', + '//src/com/facebook/buck/util/network:network', ], deps = [ ':testutil', @@ -36,10 +36,10 @@ java_test( '//src/com/facebook/buck/util:escaper', '//src/com/facebook/buck/util:exceptions', '//src/com/facebook/buck/util:io', - '//src/com/facebook/buck/util:network', '//src/com/facebook/buck/util:util', '//src/com/facebook/buck/util:watchman', '//src/com/facebook/buck/util/environment:platform', + '//src/com/facebook/buck/util/network:network', '//test/com/facebook/buck/timing:testutil', '//test/com/facebook/buck/testutil:testutil', '//test/com/facebook/buck/testutil/integration:integration', diff --git a/test/com/facebook/buck/util/HttpScribeLoggerTest.java b/test/com/facebook/buck/util/HttpScribeLoggerTest.java deleted file mode 100644 index af4ca0bb790..00000000000 --- a/test/com/facebook/buck/util/HttpScribeLoggerTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2015-present Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.facebook.buck.util; - -import static org.junit.Assert.assertEquals; - -import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; - -import org.junit.Test; - -import java.util.concurrent.atomic.AtomicInteger; - -public class HttpScribeLoggerTest { - - @Test - public void chunking() { - final AtomicInteger requestCount = new AtomicInteger(0); - HttpEndpoint httpEndpoint = new HttpEndpoint() { - @Override - public ListenableFuture post(String content) { - requestCount.incrementAndGet(); - return Futures.immediateFuture(new HttpResponse("")); - } - }; - - HttpScribeLogger logger = new HttpScribeLogger(httpEndpoint, 25); - logger.log( - "blah", - ImmutableList.of( - "hello world", - "hello world", - "hello world")); - - // Test that two of the log messages got batched together, while the third required an - // additional request. - assertEquals(2, requestCount.get()); - } - -} diff --git a/test/com/facebook/buck/util/network/BUCK b/test/com/facebook/buck/util/network/BUCK new file mode 100644 index 00000000000..0093c56de13 --- /dev/null +++ b/test/com/facebook/buck/util/network/BUCK @@ -0,0 +1,16 @@ +java_test( + name = 'network', + srcs = glob(['*.java']), + source_under_test = [ + '//src/com/facebook/buck/util/network:network', + ], + deps = [ + '//src/com/facebook/buck/util:util', + '//src/com/facebook/buck/util/network:network', + '//third-party/java/guava:guava', + '//third-party/java/hamcrest:hamcrest', + '//third-party/java/jackson:jackson-annotations', + '//third-party/java/jackson:jackson-databind', + '//third-party/java/junit:junit', + ], +) diff --git a/test/com/facebook/buck/util/network/BatchingLoggerTest.java b/test/com/facebook/buck/util/network/BatchingLoggerTest.java new file mode 100644 index 00000000000..4bda7d974a0 --- /dev/null +++ b/test/com/facebook/buck/util/network/BatchingLoggerTest.java @@ -0,0 +1,81 @@ +/* + * Copyright 2015-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.util.network; + +import static org.hamcrest.junit.MatcherAssert.assertThat; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; + +import org.hamcrest.Matchers; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class BatchingLoggerTest { + + private static class TestBatchingLogger extends BatchingLogger { + private List> uploadedBatches = new ArrayList<>(); + + public TestBatchingLogger(int minBatchSize) { + super(minBatchSize); + } + + public List> getUploadedBatches() { + return uploadedBatches; + } + + @Override + protected ListenableFuture logMultiple(ImmutableCollection data) { + uploadedBatches.add(data); + return Futures.immediateFuture(null); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testBatchingLogger() { + String shortData = "data"; + String longData = "datdatdatdatdatdatdataaaaaaadata"; + BatchingLogger.BatchEntry shortDataBatch = new BatchingLogger.BatchEntry(shortData); + BatchingLogger.BatchEntry longDataBatch = new BatchingLogger.BatchEntry(longData); + TestBatchingLogger testBatchingLogger = new TestBatchingLogger(longData.length() + 1); + + testBatchingLogger.log(longData); + // Data should still be buffered at this point. + assertThat(testBatchingLogger.getUploadedBatches(), Matchers.hasSize(0)); + + // This one should tip it over. + testBatchingLogger.log(shortData); + assertThat(testBatchingLogger.getUploadedBatches(), Matchers.hasSize(1)); + + testBatchingLogger.log(shortData); + testBatchingLogger.log(shortData); + assertThat(testBatchingLogger.getUploadedBatches(), Matchers.hasSize(1)); + testBatchingLogger.close(); + assertThat(testBatchingLogger.getUploadedBatches(), Matchers.hasSize(2)); + + assertThat( + testBatchingLogger.getUploadedBatches(), + Matchers.contains( + Matchers.contains(longDataBatch, shortDataBatch), + Matchers.contains(shortDataBatch, shortDataBatch) + )); + } +} diff --git a/test/com/facebook/buck/util/BlockingHttpEndpointTest.java b/test/com/facebook/buck/util/network/BlockingHttpEndpointTest.java similarity index 98% rename from test/com/facebook/buck/util/BlockingHttpEndpointTest.java rename to test/com/facebook/buck/util/network/BlockingHttpEndpointTest.java index e9e4e81fd60..d144c0fe0d0 100644 --- a/test/com/facebook/buck/util/BlockingHttpEndpointTest.java +++ b/test/com/facebook/buck/util/network/BlockingHttpEndpointTest.java @@ -14,7 +14,7 @@ * under the License. */ -package com.facebook.buck.util; +package com.facebook.buck.util.network; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.junit.Assert.assertThat; diff --git a/test/com/facebook/buck/util/network/HttpPutLoggerTest.java b/test/com/facebook/buck/util/network/HttpPutLoggerTest.java new file mode 100644 index 00000000000..27386b16afb --- /dev/null +++ b/test/com/facebook/buck/util/network/HttpPutLoggerTest.java @@ -0,0 +1,60 @@ +/* + * Copyright 2015-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.util.network; + +import static org.hamcrest.junit.MatcherAssert.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; + +import org.hamcrest.Matchers; +import org.junit.Test; + +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; + +public class HttpPutLoggerTest { + + @Test + public void testHttpLogger() throws Exception { + final ConcurrentLinkedQueue uploadedData = new ConcurrentLinkedQueue<>(); + HttpEndpoint testEndpoint = new HttpEndpoint() { + @Override + public ListenableFuture post(String content) { + uploadedData.add(content); + return Futures.immediateFuture(new HttpResponse("")); + } + }; + HttpPutLogger httpPutLogger = + new HttpPutLogger(testEndpoint, new ObjectMapper()); + + String entry1 = "{\"e\":1}"; + String entry2 = "{\"e\":2}"; + + httpPutLogger.log(entry1); + httpPutLogger.log(entry2); + httpPutLogger.close().get(0, TimeUnit.SECONDS); + + assertThat( + uploadedData, + Matchers.contains( + "[" + entry1 + "," + entry2 + "]" + ) + ); + } +} diff --git a/third-party/java/jackson/BUCK b/third-party/java/jackson/BUCK index e12485a2a90..61c9f4726ea 100644 --- a/third-party/java/jackson/BUCK +++ b/third-party/java/jackson/BUCK @@ -60,6 +60,7 @@ prebuilt_jar( '//src/com/facebook/buck/java/intellij:intellij', '//src/com/facebook/buck/model:model', '//src/com/facebook/buck/rules:rules', + '//src/com/facebook/buck/util/network:network', '//src/com/facebook/buck/util:watchman', '//src/com/facebook/buck/shell:rules', '//src/com/facebook/buck/shell:steps', diff --git a/third-party/java/jetty/BUCK b/third-party/java/jetty/BUCK index 6bc42d47269..cbf69cdabaa 100644 --- a/third-party/java/jetty/BUCK +++ b/third-party/java/jetty/BUCK @@ -25,6 +25,7 @@ java_library( '//test/com/facebook/buck/cli:cli', '//test/com/facebook/buck/httpserver:', '//test/com/facebook/buck/maven:maven', + '//test/com/facebook/buck/event/listener/integration:integration', '//test/com/facebook/buck/testutil/integration:integration', ], )