diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4ad90b9..9e66066 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,12 +1,6 @@ name: run-tests -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] jobs: test: diff --git a/phpunit.xml b/phpunit.xml index 189adf6..f53aaf3 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -17,8 +17,8 @@ - - + + diff --git a/src/AppkeepProvider.php b/src/AppkeepProvider.php index 904c88c..9578e57 100644 --- a/src/AppkeepProvider.php +++ b/src/AppkeepProvider.php @@ -2,9 +2,9 @@ namespace Appkeep\Laravel; +use Appkeep\Laravel\Facades\Appkeep; use Illuminate\Support\ServiceProvider; use Appkeep\Laravel\Commands\RunCommand; -use Illuminate\Console\Scheduling\Event; use Appkeep\Laravel\Commands\InitCommand; use Appkeep\Laravel\Commands\ListCommand; use Appkeep\Laravel\Commands\LoginCommand; @@ -27,6 +27,10 @@ public function register() return new AppkeepService(); }); + $this->app->singleton(EventCollector::class, function () { + return new EventCollector(); + }); + $this->app->bind(HttpClient::class, function () { return new HttpClient(config('appkeep.key')); }); @@ -43,13 +47,29 @@ public function boot() { $this->loadRoutesFrom(__DIR__ . '/../routes/web.php'); - if ($this->app->runningInConsole()) { - $this->bootForConsole(); - } + $this->bootForConsole(); + + $this->app->booted(function () { + if ($this->app->runningInConsole()) { + $this->scheduleRunCommand(); + } + + // Watch slow queries, scheduled tasks, etc. + Appkeep::watch($this->app); + }); + + $this->app->terminating(function () { + // Write in-memory events to cache. + $this->app->make(EventCollector::class)->persist(); + }); } public function bootForConsole() { + if (!$this->app->runningInConsole()) { + return; + } + $this->publishes([ __DIR__ . '/../config/appkeep.php' => config_path('appkeep.php'), ], 'config'); @@ -65,45 +85,19 @@ public function bootForConsole() LoginCommand::class, PostDeployCommand::class, ]); - - $this->app->booted(function () { - $schedule = $this->app->make(Schedule::class); - - $schedule->command('appkeep:run') - ->everyMinute() - ->runInBackground() - ->evenInMaintenanceMode(); - - $this->watchScheduledTasks($schedule); - }); } - protected function watchScheduledTasks(Schedule $schedule) + protected function scheduleRunCommand() { - /** - * @var AppkeepService - */ - $appkeep = app('appkeep'); + $schedule = $this->app->make(Schedule::class); - if (!$appkeep->scheduledTaskMonitoringEnabled) { - return; - } + $schedule->command('appkeep:run') + ->everyMinute() + ->runInBackground() + ->evenInMaintenanceMode(); - collect($schedule->events()) - ->filter(function ($event) { - // Don't monitor the Appkeep scheduled task itself. - return $event->command && !str_contains($event->command, 'appkeep:run'); - }) - ->each(function (Event $event) use ($appkeep) { - $event->before(fn () => $appkeep->scheduledTaskStarted($event)); - - $event->onSuccessWithOutput( - fn () => $appkeep->scheduledTaskCompleted($event) - ); - - $event->onFailureWithOutput( - fn () => $appkeep->scheduledTaskFailed($event) - ); - }); + $this->app->singleton('command.appkeep.run', function ($app) { + return new RunCommand($app['appkeep']); + }); } } diff --git a/src/AppkeepService.php b/src/AppkeepService.php index 81b9c41..782d6f4 100644 --- a/src/AppkeepService.php +++ b/src/AppkeepService.php @@ -3,17 +3,20 @@ namespace Appkeep\Laravel; use InvalidArgumentException; -use Appkeep\Laravel\Concerns\ReportsScheduledTaskOutputs; +use Appkeep\Laravel\Concerns\WatchesSlowQueries; +use Illuminate\Contracts\Foundation\Application; +use Appkeep\Laravel\Concerns\WatchesScheduledTasks; class AppkeepService { - use ReportsScheduledTaskOutputs; + use WatchesSlowQueries; + use WatchesScheduledTasks; public $checks = []; public function version() { - return '0.6.1'; + return '0.7.0'; } public function client() @@ -31,6 +34,13 @@ public function forgetDefaultChecks() return $this; } + public function watch(Application $app) + { + $this->watchScheduledTasks($app); + + $this->watchSlowQueries($app); + } + public function checks(array $checks = [], $replace = false) { if (! app()->runningInConsole()) { diff --git a/src/Checks/DatabaseConnectionCountCheck.php b/src/Checks/DatabaseConnectionCountCheck.php new file mode 100644 index 0000000..3b44d5a --- /dev/null +++ b/src/Checks/DatabaseConnectionCountCheck.php @@ -0,0 +1,119 @@ + MySqlInspector::class, + 'pgsql' => PostgresInspector::class, + ]; + + /** + * Check a different connection + */ + public function connection($connection) + { + $this->connection = $connection; + + return $this; + } + + public function warnIfConnectionCountIsAbove($count) + { + $this->warnAt = (int) $count; + + return $this; + } + + public function failIfConnectionCountIsAbove($count) + { + $this->failAt = (int) $count; + + return $this; + } + + /** + * @var Result + */ + public function run() + { + $connection = $this->connection ?? config('database.default'); + $meta = ['connection' => $connection]; + + $db = DB::connection($connection); + $driver = $db->getDriverName(); + + // If there's no inspector for this connection, we can't check it. + // In that case, we'll just skip this check. + if (! isset($this->inspectors[$driver])) { + return Result::ok()->meta($meta); + } + + /** + * @var DatabaseIn + */ + $inspector = app($this->inspectors[$driver], [ + 'connection' => $db, + ]); + + $cacheKey = 'appkeep.db.' . $connection . '.connection_count'; + + $maxConnections = Cache::remember( + $cacheKey, + now()->addHours(2), + function () use ($inspector) { + try { + return $inspector->maximumConnectionCount(); + } catch (Exception $e) { + // Failed to get the maximum connection count. + report($e); + + // Let's just assume the default. + return 100; + } + } + ); + + $currentConnections = $inspector->currentConnectionCount(); + + if (is_null($this->warnAt)) { + $this->warnAt = floor($maxConnections * 0.8); + } + + if (is_null($this->failAt)) { + $this->failAt = $maxConnections - 1; + } + + if ($currentConnections >= $this->failAt) { + return Result::fail("Too many active database connections.") + ->summary($currentConnections) + ->meta($meta); + } + + if ($currentConnections >= $this->warnAt) { + return Result::warn("Approaching the maximum number of database connections.") + ->summary($currentConnections) + ->meta($meta); + } + + return Result::ok() + ->summary($currentConnections) + ->meta($meta); + } +} diff --git a/src/Commands/RunCommand.php b/src/Commands/RunCommand.php index 61b0de4..d5cf127 100644 --- a/src/Commands/RunCommand.php +++ b/src/Commands/RunCommand.php @@ -5,6 +5,7 @@ use Appkeep\Laravel\Result; use Illuminate\Console\Command; use Appkeep\Laravel\Enums\Status; +use Appkeep\Laravel\EventCollector; use Appkeep\Laravel\Facades\Appkeep; use Appkeep\Laravel\Events\ChecksEvent; @@ -14,6 +15,35 @@ class RunCommand extends Command protected $description = 'Run all Appkeep checks'; public function handle() + { + $this->sendCollectedEvents(); + + $this->runChecks(); + } + + private function sendCollectedEvents() + { + $collector = resolve(EventCollector::class); + + $events = $collector->pull(); + + if (empty($events)) { + $this->line('No collected events to send.'); + + return; + } + + $this->info(sprintf('Sending %d collected events to Appkeep...', count($events))); + + try { + Appkeep::client()->sendBatchEvents(array_values($events))->throw(); + } catch (\Exception $e) { + $this->warn('Failed to post collected events to Appkeep.'); + $this->line($e->getMessage()); + } + } + + private function runChecks() { $checks = Appkeep::checks(); diff --git a/src/Concerns/RegistersDefaultChecks.php b/src/Concerns/RegistersDefaultChecks.php index 8a51fb3..b08945c 100644 --- a/src/Concerns/RegistersDefaultChecks.php +++ b/src/Concerns/RegistersDefaultChecks.php @@ -11,6 +11,7 @@ use Appkeep\Laravel\Checks\SystemLoadCheck; use Appkeep\Laravel\Checks\OptimizationCheck; use Appkeep\Laravel\Checks\ProductionModeCheck; +use Appkeep\Laravel\Checks\DatabaseConnectionCountCheck; use Laravel\Horizon\Contracts\MasterSupervisorRepository; trait RegistersDefaultChecks @@ -24,6 +25,8 @@ protected function registerDefaultChecks() DatabaseCheck::make(), + DatabaseConnectionCountCheck::make(), + CacheCheck::make(), DiskUsageCheck::make(), diff --git a/src/Concerns/ReportsScheduledTaskOutputs.php b/src/Concerns/WatchesScheduledTasks.php similarity index 59% rename from src/Concerns/ReportsScheduledTaskOutputs.php rename to src/Concerns/WatchesScheduledTasks.php index b539520..ecdbd6b 100644 --- a/src/Concerns/ReportsScheduledTaskOutputs.php +++ b/src/Concerns/WatchesScheduledTasks.php @@ -4,24 +4,53 @@ use Appkeep\Laravel\ScheduledTaskOutput; use Illuminate\Console\Scheduling\Event; +use Illuminate\Console\Scheduling\Schedule; +use Illuminate\Contracts\Foundation\Application; -trait ReportsScheduledTaskOutputs +trait WatchesScheduledTasks { - public $scheduledTaskMonitoringEnabled = true; + public $watchScheduledTasks = true; private $scheduledTaskStartMs; + private $scheduledTaskStartedAt; /** - * Disables scheduled task monitoring. + * Disable monitoring of scheduled tasks. */ - public function dontMonitorScheduledTasks() + public function dontWatchScheduledTasks() { - $this->scheduledTaskMonitoringEnabled = false; + $this->watchScheduledTasks = false; return $this; } + protected function watchScheduledTasks(Application $app) + { + if (!$this->watchScheduledTasks) { + return; + } + + if (!$app->runningInConsole()) { + return; + } + + $schedule = $app->make(Schedule::class); + + collect($schedule->events()) + ->filter(function ($event) { + // Don't monitor the Appkeep scheduled task itself. + return $event->command && !str_contains($event->command, 'appkeep:run'); + }) + ->each(function (Event $event) { + $event->before(fn () => $this->scheduledTaskStarted($event)); + + $event->onSuccessWithOutput(fn () => $this->scheduledTaskCompleted($event)); + + $event->onFailureWithOutput(fn () => $this->scheduledTaskFailed($event)); + }); + } + public function scheduledTaskStarted(Event $task) { $this->scheduledTaskStartMs = hrtime(true); diff --git a/src/Concerns/WatchesSlowQueries.php b/src/Concerns/WatchesSlowQueries.php new file mode 100644 index 0000000..e71b994 --- /dev/null +++ b/src/Concerns/WatchesSlowQueries.php @@ -0,0 +1,54 @@ +watchSlowQueries = false; + + return $this; + } + + /** + * Change the threshold for slow queries in milliseconds. + * By default, Appkeep will report queries that take longer than 500ms. + */ + public function reportQueriesSlowerThan($threshold) + { + static::$slowQueryThreshold = $threshold; + + return $this; + } + + public function watchSlowQueries(Application $app) + { + if (! $this->watchSlowQueries) { + return; + } + + Event::listen(QueryExecuted::class, function (QueryExecuted $event) use ($app) { + if ($event->time < static::$slowQueryThreshold) { + return; + } + + $app->make(EventCollector::class)->push( + new SlowQueryEvent($event) + ); + }); + } +} diff --git a/src/Contexts/DatabaseContext.php b/src/Contexts/DatabaseContext.php new file mode 100644 index 0000000..380106a --- /dev/null +++ b/src/Contexts/DatabaseContext.php @@ -0,0 +1,24 @@ +connection = $connection; + } + + public function toArray() + { + return [ + 'name' => $this->connection->getName(), + 'driver' => $this->connection->getDriverName(), + ]; + } +} diff --git a/src/Contexts/RequestContext.php b/src/Contexts/RequestContext.php new file mode 100644 index 0000000..8cbc1a9 --- /dev/null +++ b/src/Contexts/RequestContext.php @@ -0,0 +1,24 @@ +runningUnitTests() && app()->runningInConsole()) { + return [ + 'command' => implode(' ', $_SERVER['argv']), + ]; + } + + $request = request(); + + return [ + 'path' => '/' . ltrim($request->path(), '/'), + 'method' => $request->method(), + ]; + } +} diff --git a/src/Database/DatabaseInspector.php b/src/Database/DatabaseInspector.php new file mode 100644 index 0000000..8500fa2 --- /dev/null +++ b/src/Database/DatabaseInspector.php @@ -0,0 +1,17 @@ +connection = $connection; + } + + public function maximumConnectionCount(): int + { + return (int) $this->connection->selectOne('SHOW VARIABLES LIKE "max_connections"')->Value; + } + + public function currentConnectionCount(): int + { + return (int) $this->connection->selectOne('SHOW STATUS LIKE "threads_connected"')->Value; + } +} diff --git a/src/Database/PostgresInspector.php b/src/Database/PostgresInspector.php new file mode 100644 index 0000000..6c64987 --- /dev/null +++ b/src/Database/PostgresInspector.php @@ -0,0 +1,25 @@ +connection = $connection; + } + + public function maximumConnectionCount(): int + { + return (int) $this->connection->selectOne('SHOW max_connections;')->max_connections; + } + + public function currentConnectionCount(): int + { + return (int) $this->connection->selectOne('select count(*) as connections from pg_stat_activity')->connections; + } +} diff --git a/src/EventCollector.php b/src/EventCollector.php new file mode 100644 index 0000000..349bac0 --- /dev/null +++ b/src/EventCollector.php @@ -0,0 +1,83 @@ +toArray(); + $hash = $event->dedupeHash(); + + // For now, we just store it in the memory + if (isset($this->events[$hash])) { + return; + } + + if (count($this->events) >= $this->maxItemsInMemory) { + logger()->debug('Appkeep: Reached maxItemsInMemory. Dropping event.'); + + return; + } + + $this->events[$hash] = $data; + } + + /** + * Persist events in the memory to cache. + */ + public function persist() + { + $storedEvents = Cache::get($this->cacheKey, []); + $numStoredEvents = count($storedEvents); + + foreach ($this->events as $hash => $data) { + // Item already exists in cache. + if (isset($storedEvents[$hash])) { + continue; + } + + if ($numStoredEvents >= $this->maxItemsInCache) { + logger()->warning( + 'Appkeep: Event cache is full. Dropping event.' + . ' Run appkeep:run to send cached events.' + ); + + break; + } + + $storedEvents[$hash] = $data; + $numStoredEvents++; + } + + // Put the updated events back to cache. + Cache::forever($this->cacheKey, $storedEvents); + } + + /** + * Pull all events from cache and clear the cache. + */ + public function pull() + { + $events = Cache::pull($this->cacheKey, []); + + return $events; + } +} diff --git a/src/Events/Contracts/CollectableEvent.php b/src/Events/Contracts/CollectableEvent.php new file mode 100644 index 0000000..35aef32 --- /dev/null +++ b/src/Events/Contracts/CollectableEvent.php @@ -0,0 +1,10 @@ +queryExecutedEvent = $event; + + $this->setContext('database', new DatabaseContext($event->connection)); + $this->setContext('request', new RequestContext()); + } + + /** + * This hash will help us group the same query across multiple requests on Appkeep side. + */ + public function dedupeHash(): string + { + return md5($this->queryExecutedEvent->sql); + } + + public function toArray() + { + return array_merge( + parent::toArray(), + [ + 'query' => [ + 'hash' => $this->dedupeHash(), + 'sql' => $this->queryExecutedEvent->sql, + ], + 'time' => $this->queryExecutedEvent->time, + ] + ); + } +} diff --git a/src/HttpClient.php b/src/HttpClient.php index 7b6c80b..737d75c 100644 --- a/src/HttpClient.php +++ b/src/HttpClient.php @@ -38,6 +38,19 @@ public function sendScheduledTaskOutput(ScheduledTaskOutput $output) return $this->sendEvent(new ScheduledTaskEvent($output))->throw(); } + /** + * Send batch events to Appkeep + */ + public function sendBatchEvents(array $events) + { + return Http::withHeaders($this->defaultHeaders())->post( + config('appkeep.endpoint'), + [ + 'batch' => $events, + ] + ); + } + protected function defaultHeaders() { return [ diff --git a/tests/Feature/Checks/DatabaseConnectionCountCheckTest.php b/tests/Feature/Checks/DatabaseConnectionCountCheckTest.php new file mode 100644 index 0000000..f9df551 --- /dev/null +++ b/tests/Feature/Checks/DatabaseConnectionCountCheckTest.php @@ -0,0 +1,72 @@ +mockDatabaseConnection(155, 155); + + $result = DatabaseConnectionCountCheck::make()->connection('mysql')->warnIfConnectionCountIsAbove(35)->run(); + $this->assertEquals(Status::FAIL, $result->status); + } + + /** + * @test + */ + public function warns_if_connection_count_is_exceeded() + { + $this->mockDatabaseConnection(50, 155); + + $result = DatabaseConnectionCountCheck::make()->connection('mysql')->warnIfConnectionCountIsAbove(35)->run(); + $this->assertEquals(Status::WARN, $result->status); + } + + /** + * @test + */ + public function succeed_if_connection_count_is_below_warn_limit() + { + $this->mockDatabaseConnection(8, 155); + + $result = DatabaseConnectionCountCheck::make()->connection('mysql')->warnIfConnectionCountIsAbove(35)->run(); + $this->assertEquals(Status::OK, $result->status); + } + + private function mockDatabaseConnection($currentConnections, $maxConnections) + { + // Set DB Mock + DB::shouldReceive("connection") + ->once() + ->with('mysql') + ->andReturn( + Mockery::mock('Illuminate\Database\MysqlConnection', function ($mock) use ($currentConnections, $maxConnections) { + $mock->shouldReceive('getDriverName')->andReturn('mysql'); + + $mock->shouldReceive("selectOne") + ->once() + ->with('SHOW STATUS LIKE "threads_connected"') + ->andReturn((object) [ + 'Value' => $currentConnections, + ]); + + $mock->shouldReceive("selectOne") + ->once() + ->with('SHOW VARIABLES LIKE "max_connections"') + ->andReturn((object) [ + 'Value' => $maxConnections, + ]); + }) + ); + } +} diff --git a/tests/Feature/RunCommandTest.php b/tests/Feature/RunCommandTest.php index 61aabab..83235bd 100644 --- a/tests/Feature/RunCommandTest.php +++ b/tests/Feature/RunCommandTest.php @@ -6,8 +6,12 @@ use Tests\TestCase; use Tests\TestCheck; use Appkeep\Laravel\Result; +use Illuminate\Support\Facades\DB; +use Appkeep\Laravel\AppkeepService; +use Appkeep\Laravel\EventCollector; use Appkeep\Laravel\Facades\Appkeep; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Facades\Cache; class RunCommandTest extends TestCase { @@ -51,15 +55,19 @@ public function it_returns_a_table_output() */ public function it_runs_only_due_checks() { + Appkeep::forgetDefaultChecks()->checks([ + TestCheck::make('test-check')->result( + Result::ok()->summary('50%') + ), + TestCheck::make('test-check-2')->result( + Result::ok()->summary('50%') + )->everyFifteenMinutes(), + ]); + Http::fake(); Carbon::setTestNow(Carbon::now()->setMinute(3)->setSecond(0)); - Appkeep::forgetDefaultChecks()->checks([ - TestCheck::make('test-check-1')->everyMinute(), - TestCheck::make('test-check-15')->everyFifteenMinutes(), - ]); - // Prevent hitting Appkeep server. $this->artisan('appkeep:run')->assertExitCode(0); @@ -68,7 +76,7 @@ public function it_runs_only_due_checks() $data = $request->data(); return count($data['checks']) === 1 - && $data['checks'][0]['check'] === 'test-check-1'; + && $data['checks'][0]['check'] === 'test-check'; }); Carbon::setTestNow(Carbon::now()->setMinute(15)->setSecond(0)); @@ -82,4 +90,38 @@ public function it_runs_only_due_checks() return count($data['checks']) === 2; }); } + + /** + * @test + */ + public function it_sends_batched_events() + { + Http::fake(); + Cache::flush(); + AppkeepService::$slowQueryThreshold = 0; + + // Don't execute checks + Appkeep::forgetDefaultChecks(); + + $this->artisan('appkeep:run')->assertExitCode(0); + Http::assertNothingSent(); + + // Now run a query... This should trigger a slow query event. + DB::statement('SELECT RANDOM() AS random_number1;'); + DB::statement('SELECT RANDOM() AS random_number2;'); + DB::statement('SELECT RANDOM() AS random_number2;'); + DB::statement('SELECT RANDOM() AS random_number3;'); + + $this->app->make(EventCollector::class)->persist(); + + $this->artisan('appkeep:run')->assertExitCode(0); + + Http::assertSent(function ($request) { + $data = $request->data(); + + $eventCount = count($data['batch']); + + return $eventCount == 3; + }); + } } diff --git a/tests/Feature/SlowQueryWatcherTest.php b/tests/Feature/SlowQueryWatcherTest.php new file mode 100644 index 0000000..fd36ff5 --- /dev/null +++ b/tests/Feature/SlowQueryWatcherTest.php @@ -0,0 +1,107 @@ +app->make(EventCollector::class)->persist(); + + $events = $this->app->make(EventCollector::class)->pull(); + + $this->assertCount(1, $events); + + $event = array_values($events)[0]; + + $this->assertEquals('slow-query', Arr::get($event, 'name')); + $this->assertEquals('sqlite', Arr::get($event, 'context.database.driver')); + + $this->assertEquals('SELECT RANDOM() AS random_number;', Arr::get($event, 'query.sql')); + + // TODO: Assert we can see the + } + + /** + * @test + */ + public function it_listens_to_slow_queries_from_http() + { + Route::get('/slow-query', function () { + DB::statement('SELECT RANDOM() AS random_number;'); + + return 'ok'; + }); + + $this->get('/slow-query') + ->assertStatus(200) + ->assertSee('ok'); + + $events = $this->app->make(EventCollector::class)->pull(); + $this->assertCount(1, $events); + + $event = array_values($events)[0]; + + $this->assertEquals('slow-query', Arr::get($event, 'name')); + $this->assertEquals('SELECT RANDOM() AS random_number;', Arr::get($event, 'query.sql')); + + $this->assertEquals('/slow-query', Arr::get($event, 'context.request.path')); + } + + /** + * @test + */ + public function it_deduplicates_the_same_query() + { + Route::get('/slow-query/{number}', function ($number) { + DB::statement("SELECT $number AS random_number_$number;"); + + return 'ok'; + }); + + $this->get('/slow-query/1') + ->assertStatus(200) + ->assertSee('ok'); + + $this->get('/slow-query/1') + ->assertStatus(200) + ->assertSee('ok'); + + $this->get('/slow-query/2') + ->assertStatus(200) + ->assertSee('ok'); + + + $events = $this->app->make(EventCollector::class)->pull(); + $this->assertCount(2, $events); + } +}