From 8032f7791b98d13e29b5d6830065e20258f39143 Mon Sep 17 00:00:00 2001 From: Timur Shagiakhmetov Date: Tue, 28 Jan 2020 13:11:26 +0000 Subject: [PATCH] speed up method usage page --- CHANGELOG.md | 7 +++++++ .../DB/Validators/Functions.php | 1 + .../Interfaces/MethodDataInterface.php | 3 ++- .../Interfaces/SnapshotInterface.php | 1 + .../DataProviders/MethodData.php | 8 ++++++-- .../LiveProfilerUI/DataProviders/Snapshot.php | 19 +++++++++++++++++++ .../LiveProfilerUI/Pages/MethodUsagePage.php | 18 ++++++++++++++++-- src/config/services.yaml | 6 ++++-- src/templates/method_usage.php | 2 -- .../Pages/MethodUsagePageTest.php | 6 ++++-- 10 files changed, 60 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f487ca6..58ed8d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ There are next changes: +## 1.2.11 + +There are next changes: + +- optimized a method usage page for huge tables +- added a calls_count param to display calls count recorded during specified day + ## 1.2.10 There are next changes: diff --git a/src/Badoo/LiveProfilerUI/DB/Validators/Functions.php b/src/Badoo/LiveProfilerUI/DB/Validators/Functions.php index 5d67bbe..1c42d2d 100644 --- a/src/Badoo/LiveProfilerUI/DB/Validators/Functions.php +++ b/src/Badoo/LiveProfilerUI/DB/Validators/Functions.php @@ -15,6 +15,7 @@ class Functions 'sum', 'date', 'max', + 'min', ]; public static function validate(string $function) : bool diff --git a/src/Badoo/LiveProfilerUI/DataProviders/Interfaces/MethodDataInterface.php b/src/Badoo/LiveProfilerUI/DataProviders/Interfaces/MethodDataInterface.php index 70e8370..b195691 100644 --- a/src/Badoo/LiveProfilerUI/DataProviders/Interfaces/MethodDataInterface.php +++ b/src/Badoo/LiveProfilerUI/DataProviders/Interfaces/MethodDataInterface.php @@ -20,9 +20,10 @@ public function getDataBySnapshotId(int $snapshot_id) : array; * @param int[] $snapshot_ids * @param int[] $method_ids * @param int $limit + * @param int $start_snapshot_id * @return MethodData[] */ - public function getDataByMethodIdsAndSnapshotIds(array $snapshot_ids, array $method_ids, int $limit = 0) : array; + public function getDataByMethodIdsAndSnapshotIds(array $snapshot_ids, array $method_ids, int $limit = 0, int $start_snapshot_id = 0) : array; public function getOneParamDataBySnapshotIds(array $snapshot_ids, string $param, int $threshold = 1000) : array; public function deleteBySnapshotId(int $snapshot_id) : bool; public function insertMany(array $inserts) : bool; diff --git a/src/Badoo/LiveProfilerUI/DataProviders/Interfaces/SnapshotInterface.php b/src/Badoo/LiveProfilerUI/DataProviders/Interfaces/SnapshotInterface.php index fe1b5d0..e1f39a1 100644 --- a/src/Badoo/LiveProfilerUI/DataProviders/Interfaces/SnapshotInterface.php +++ b/src/Badoo/LiveProfilerUI/DataProviders/Interfaces/SnapshotInterface.php @@ -22,6 +22,7 @@ public function getOneByAppAndLabel(string $app, string$label) : Snapshot; public function getOneByAppAndLabelAndDate(string $app, string $label, string $date) : Snapshot; public function getSnapshotsByDates(array $dates, string $param = '') : array; public function getSnapshotIdsByDates(array $dates, string $app, string $label) : array; + public function getMinSnapshotIdByDates(array $dates) : int; public function getOldSnapshots(int $keep_days = 200, int $limit = 2000) : array; public function getDatesByAppAndLabel(string $app, string $label) : array; public function getMaxCallsCntByAppAndLabel(string $app, string $label) : int; diff --git a/src/Badoo/LiveProfilerUI/DataProviders/MethodData.php b/src/Badoo/LiveProfilerUI/DataProviders/MethodData.php index 69d52ea..da3ef3f 100644 --- a/src/Badoo/LiveProfilerUI/DataProviders/MethodData.php +++ b/src/Badoo/LiveProfilerUI/DataProviders/MethodData.php @@ -40,7 +40,8 @@ public function getDataBySnapshotId(int $snapshot_id) : array public function getDataByMethodIdsAndSnapshotIds( array $snapshot_ids, array $method_ids, - int $limit = 0 + int $limit = 0, + int $start_snapshot_id = 0 ) : array { if (empty($snapshot_ids) && empty($method_ids)) { return []; @@ -53,6 +54,9 @@ public function getDataByMethodIdsAndSnapshotIds( if (!empty($method_ids)) { $filters[] = ['method_id', $method_ids]; } + if ($start_snapshot_id) { + $filters[] = ['snapshot_id', $start_snapshot_id, '>=']; + } $records = $this->AggregatorStorage->getAll( self::TABLE_NAME, @@ -60,7 +64,7 @@ public function getDataByMethodIdsAndSnapshotIds( [ 'filter' => $filters, 'order' => ['snapshot_id' => 'desc'], - 'limit' => $limit + 'limit' => $limit, ] ); diff --git a/src/Badoo/LiveProfilerUI/DataProviders/Snapshot.php b/src/Badoo/LiveProfilerUI/DataProviders/Snapshot.php index b1866b0..6b566f9 100644 --- a/src/Badoo/LiveProfilerUI/DataProviders/Snapshot.php +++ b/src/Badoo/LiveProfilerUI/DataProviders/Snapshot.php @@ -250,6 +250,25 @@ public function getSnapshotIdsByDates(array $dates, string $app, string $label) return $result; } + public function getMinSnapshotIdByDates(array $dates) : int + { + if (empty($dates)) { + return 0; + } + + $snapshot = $this->AggregatorStorage->getOne( + self::TABLE_NAME, + [['field' => 'id', 'function' => 'min']], + [ + 'filter' => [ + ['date', $dates], + ], + ] + ); + + return $snapshot ? (int)$snapshot['id'] : 0; + } + public function getOldSnapshots(int $keep_days = 200, int $limit = 2000) : array { $snapshots = $this->AggregatorStorage->getAll( diff --git a/src/Badoo/LiveProfilerUI/Pages/MethodUsagePage.php b/src/Badoo/LiveProfilerUI/Pages/MethodUsagePage.php index ec3e5c0..f8f358e 100644 --- a/src/Badoo/LiveProfilerUI/Pages/MethodUsagePage.php +++ b/src/Badoo/LiveProfilerUI/Pages/MethodUsagePage.php @@ -25,19 +25,23 @@ class MethodUsagePage extends BasePage protected $MethodData; /** @var FieldList */ protected $FieldList; + /** @var bool */ + protected $use_method_usage_optimisation = false; public function __construct( View $View, SnapshotInterface $Snapshot, MethodInterface $Method, MethodDataInterface $MethodData, - FieldList $FieldList + FieldList $FieldList, + bool $use_method_usage_optimisation = false ) { $this->View = $View; $this->Snapshot = $Snapshot; $this->Method = $Method; $this->MethodData = $MethodData; $this->FieldList = $FieldList; + $this->use_method_usage_optimisation = $use_method_usage_optimisation; } protected function cleanData() : bool @@ -73,10 +77,19 @@ public function getTemplateData() : array $results = []; if (!empty($methods)) { + $start_snapshot_id = 0; + if ($this->use_method_usage_optimisation) { + $last_two_days = \Badoo\LiveProfilerUI\DateGenerator::getDatesArray(date('Y-m-d'), 2, 2); + $start_snapshot_id = in_array(current($methods)['date'], $last_two_days, true) + ? $this->Snapshot->getMinSnapshotIdByDates($last_two_days) + : 0; + } + $method_data = $this->MethodData->getDataByMethodIdsAndSnapshotIds( [], array_keys($methods), - 100 + 200, + $start_snapshot_id ); $snapshot_ids = []; @@ -97,6 +110,7 @@ public function getTemplateData() : array foreach ($fields as $field) { $result['fields'][$field] = $values[$field]; } + $result['fields']['calls_count'] = $snapshots[$Row->getSnapshotId()]['calls_count']; $results[] = $result; } } diff --git a/src/config/services.yaml b/src/config/services.yaml index a89b538..4e5f925 100644 --- a/src/config/services.yaml +++ b/src/config/services.yaml @@ -8,9 +8,11 @@ parameters: aggregator.log_file: '' aggregator.minimum_profiles_count: 0 aggregator.aggregating_jobs_lock_file: /app/db_data/aggregator_processor.lock + aggregator.use_method_usage_optimisation: false # returns only last day usage if method run for last 2 days aggregator.fields_descriptions: - ct: Average calls count in this context during specified day (µs) + calls_count: Calls count recorded during specified day + ct: Average calls count in this context during specified day min_ct: Minimum calls count in this context during specified day (µs) max_ct: Maximim calls count in this context during specified day (µs) percent_ct: 90 percentile of calls count in this context during specified day (µs) @@ -93,7 +95,7 @@ services: method_usage_page: class: \Badoo\LiveProfilerUI\Pages\MethodUsagePage - arguments: ['@view', '@snapshot', '@method', '@method_data', '@fields'] + arguments: ['@view', '@snapshot', '@method', '@method_data', '@fields', '%aggregator.use_method_usage_optimisation%'] top_diff_page: class: \Badoo\LiveProfilerUI\Pages\TopDiffPage diff --git a/src/templates/method_usage.php b/src/templates/method_usage.php index 228d063..48a2971 100644 --- a/src/templates/method_usage.php +++ b/src/templates/method_usage.php @@ -32,7 +32,6 @@ # date - method label app $field_value) { ?> @@ -53,7 +52,6 @@ - diff --git a/tests/unit/Badoo/LiveProfilerUI/Pages/MethodUsagePageTest.php b/tests/unit/Badoo/LiveProfilerUI/Pages/MethodUsagePageTest.php index be51277..4544594 100644 --- a/tests/unit/Badoo/LiveProfilerUI/Pages/MethodUsagePageTest.php +++ b/tests/unit/Badoo/LiveProfilerUI/Pages/MethodUsagePageTest.php @@ -59,7 +59,8 @@ public function providerGetTemplateData() 'label' => 'label', 'fields' => [ 'ct' => 1, - 'wt' => 1 + 'wt' => 1, + 'calls_count' => 1 ] ] ], @@ -92,7 +93,8 @@ public function testGetTemplateData($method_name, $found_methods, $methods_data, 'id' => 1, 'app' => 'app', 'label' => 'label', - 'date' => 'date' + 'date' => 'date', + 'calls_count' => 1 ]; $SnapshotMock = $this->getMockBuilder(\Badoo\LiveProfilerUI\DataProviders\Snapshot::class) ->disableOriginalConstructor()