speed up method usage page

badoo · Jan 28, 2020 · 8032f77 · 8032f77
1 parent 4891cbd
commit 8032f77
Show file tree

Hide file tree

Showing 10 changed files with 60 additions and 11 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,13 @@
 
 There are next changes:
 
+## 1.2.11
+
+There are next changes:
+
+- optimized a method usage page for huge tables
+- added a calls_count param to display calls count recorded during specified day
+
 ## 1.2.10
 
 There are next changes:

diff --git a/src/Badoo/LiveProfilerUI/DB/Validators/Functions.php b/src/Badoo/LiveProfilerUI/DB/Validators/Functions.php
@@ -15,6 +15,7 @@ class Functions
         'sum',
         'date',
         'max',
+        'min',
     ];
 
     public static function validate(string $function) : bool

diff --git a/src/Badoo/LiveProfilerUI/DataProviders/Interfaces/MethodDataInterface.php b/src/Badoo/LiveProfilerUI/DataProviders/Interfaces/MethodDataInterface.php
@@ -20,9 +20,10 @@ public function getDataBySnapshotId(int $snapshot_id) : array;
      * @param int[] $snapshot_ids
      * @param int[] $method_ids
      * @param int $limit
+     * @param int $start_snapshot_id
      * @return MethodData[]
      */
-    public function getDataByMethodIdsAndSnapshotIds(array $snapshot_ids, array $method_ids, int $limit = 0) : array;
+    public function getDataByMethodIdsAndSnapshotIds(array $snapshot_ids, array $method_ids, int $limit = 0, int $start_snapshot_id = 0) : array;
     public function getOneParamDataBySnapshotIds(array $snapshot_ids, string $param, int $threshold = 1000) : array;
     public function deleteBySnapshotId(int $snapshot_id) : bool;
     public function insertMany(array $inserts) : bool;

diff --git a/src/Badoo/LiveProfilerUI/DataProviders/Interfaces/SnapshotInterface.php b/src/Badoo/LiveProfilerUI/DataProviders/Interfaces/SnapshotInterface.php
@@ -22,6 +22,7 @@ public function getOneByAppAndLabel(string $app, string$label) : Snapshot;
     public function getOneByAppAndLabelAndDate(string $app, string $label, string $date) : Snapshot;
     public function getSnapshotsByDates(array $dates, string $param = '') : array;
     public function getSnapshotIdsByDates(array $dates, string $app, string $label) : array;
+    public function getMinSnapshotIdByDates(array $dates) : int;
     public function getOldSnapshots(int $keep_days = 200, int $limit = 2000) : array;
     public function getDatesByAppAndLabel(string $app, string $label) : array;
     public function getMaxCallsCntByAppAndLabel(string $app, string $label) : int;

diff --git a/src/Badoo/LiveProfilerUI/DataProviders/MethodData.php b/src/Badoo/LiveProfilerUI/DataProviders/MethodData.php
@@ -40,7 +40,8 @@ public function getDataBySnapshotId(int $snapshot_id) : array
     public function getDataByMethodIdsAndSnapshotIds(
         array $snapshot_ids,
         array $method_ids,
-        int $limit = 0
+        int $limit = 0,
+        int $start_snapshot_id = 0
     ) : array {
         if (empty($snapshot_ids) && empty($method_ids)) {
             return [];
@@ -53,14 +54,17 @@ public function getDataByMethodIdsAndSnapshotIds(
         if (!empty($method_ids)) {
             $filters[] = ['method_id', $method_ids];
         }
+        if ($start_snapshot_id) {
+            $filters[] = ['snapshot_id', $start_snapshot_id, '>='];
+        }
 
         $records = $this->AggregatorStorage->getAll(
             self::TABLE_NAME,
             ['all'],
             [
                 'filter' => $filters,
                 'order' => ['snapshot_id' => 'desc'],
-                'limit' => $limit
+                'limit' => $limit,
             ]
         );
 

diff --git a/src/Badoo/LiveProfilerUI/DataProviders/Snapshot.php b/src/Badoo/LiveProfilerUI/DataProviders/Snapshot.php
@@ -250,6 +250,25 @@ public function getSnapshotIdsByDates(array $dates, string $app, string $label)
         return $result;
     }
 
+    public function getMinSnapshotIdByDates(array $dates) : int
+    {
+        if (empty($dates)) {
+            return 0;
+        }
+
+        $snapshot = $this->AggregatorStorage->getOne(
+            self::TABLE_NAME,
+            [['field' => 'id', 'function' => 'min']],
+            [
+                'filter' => [
+                    ['date', $dates],
+                ],
+            ]
+        );
+
+        return $snapshot ? (int)$snapshot['id'] : 0;
+    }
+
     public function getOldSnapshots(int $keep_days = 200, int $limit = 2000) : array
     {
         $snapshots = $this->AggregatorStorage->getAll(

diff --git a/src/Badoo/LiveProfilerUI/Pages/MethodUsagePage.php b/src/Badoo/LiveProfilerUI/Pages/MethodUsagePage.php
@@ -25,19 +25,23 @@ class MethodUsagePage extends BasePage
     protected $MethodData;
     /** @var FieldList */
     protected $FieldList;
+    /** @var bool */
+    protected $use_method_usage_optimisation = false;
 
     public function __construct(
         View $View,
         SnapshotInterface $Snapshot,
         MethodInterface $Method,
         MethodDataInterface $MethodData,
-        FieldList $FieldList
+        FieldList $FieldList,
+        bool $use_method_usage_optimisation = false
     ) {
         $this->View = $View;
         $this->Snapshot = $Snapshot;
         $this->Method = $Method;
         $this->MethodData = $MethodData;
         $this->FieldList = $FieldList;
+        $this->use_method_usage_optimisation = $use_method_usage_optimisation;
     }
 
     protected function cleanData() : bool
@@ -73,10 +77,19 @@ public function getTemplateData() : array
 
         $results = [];
         if (!empty($methods)) {
+            $start_snapshot_id = 0;
+            if ($this->use_method_usage_optimisation) {
+                $last_two_days = \Badoo\LiveProfilerUI\DateGenerator::getDatesArray(date('Y-m-d'), 2, 2);
+                $start_snapshot_id = in_array(current($methods)['date'], $last_two_days, true)
+                    ? $this->Snapshot->getMinSnapshotIdByDates($last_two_days)
+                    : 0;
+            }
+
             $method_data = $this->MethodData->getDataByMethodIdsAndSnapshotIds(
                 [],
                 array_keys($methods),
-                100
+                200,
+                $start_snapshot_id
             );
 
             $snapshot_ids = [];
@@ -97,6 +110,7 @@ public function getTemplateData() : array
                 foreach ($fields as $field) {
                     $result['fields'][$field] = $values[$field];
                 }
+                $result['fields']['calls_count'] = $snapshots[$Row->getSnapshotId()]['calls_count'];
                 $results[] = $result;
             }
         }

diff --git a/src/config/services.yaml b/src/config/services.yaml
@@ -8,9 +8,11 @@ parameters:
   aggregator.log_file: ''
   aggregator.minimum_profiles_count: 0
   aggregator.aggregating_jobs_lock_file: /app/db_data/aggregator_processor.lock
+  aggregator.use_method_usage_optimisation: false # returns only last day usage if method run for last 2 days
 
   aggregator.fields_descriptions:
-    ct: Average calls count in this context during specified day (µs)
+    calls_count: Calls count recorded during specified day
+    ct: Average calls count in this context during specified day
     min_ct: Minimum calls count in this context during specified day (µs)
     max_ct: Maximim calls count in this context during specified day (µs)
     percent_ct: 90 percentile of calls count in this context during specified day (µs)
@@ -93,7 +95,7 @@ services:
 
   method_usage_page:
     class: \Badoo\LiveProfilerUI\Pages\MethodUsagePage
-    arguments: ['@view', '@snapshot', '@method', '@method_data', '@fields']
+    arguments: ['@view', '@snapshot', '@method', '@method_data', '@fields', '%aggregator.use_method_usage_optimisation%']
 
   top_diff_page:
     class: \Badoo\LiveProfilerUI\Pages\TopDiffPage

diff --git a/src/templates/method_usage.php b/src/templates/method_usage.php
@@ -32,7 +32,6 @@
         <tr>
             <th class="sorter-false" style="width: 20px;">#</th>
             <th class="sorter-text">date</th>
-            <th>method</th>
             <th>label</th>
             <th>app</th>
             <?php foreach ($data['results'][0]['fields'] as $field_name => $field_value) { ?>
@@ -53,7 +52,6 @@
                 </a>
             </td>
             <td><?= $result['date'] ?></td>
-            <td><?= $result['method_name'] ?></td>
             <td><?= $result['label'] ?></td>
             <td><?= $result['app'] ?></td>
             <?php foreach ($result['fields'] as $field) { ?>

diff --git a/tests/unit/Badoo/LiveProfilerUI/Pages/MethodUsagePageTest.php b/tests/unit/Badoo/LiveProfilerUI/Pages/MethodUsagePageTest.php
@@ -59,7 +59,8 @@ public function providerGetTemplateData()
                             'label' => 'label',
                             'fields' => [
                                 'ct' => 1,
-                                'wt' => 1
+                                'wt' => 1,
+                                'calls_count' => 1
                             ]
                         ]
                     ],
@@ -92,7 +93,8 @@ public function testGetTemplateData($method_name, $found_methods, $methods_data,
             'id' => 1,
             'app' => 'app',
             'label' => 'label',
-            'date' => 'date'
+            'date' => 'date',
+            'calls_count' => 1
         ];
         $SnapshotMock = $this->getMockBuilder(\Badoo\LiveProfilerUI\DataProviders\Snapshot::class)
             ->disableOriginalConstructor()