Merge pull request #132 from stat-kwon/master

Add group_by option when loading Widget and Apply Cache in DataTable and Widget

src/spaceone/dashboard/error/data_table.py

@@ -9,6 +9,18 @@ class ERROR_QUERY_OPTION(ERROR_INVALID_ARGUMENT):
     _message = "Query option is invalid. (key = {key})"
 
 
+class ERROR_INVALID_SORT_OPTIONS(ERROR_INVALID_ARGUMENT):
+    _message = "All sort keys must be included in group_by. (group_by = {group_by}, sort = {sort})"
+
+
+class ERROR_EMPTY_DATA_FIELD(ERROR_INVALID_ARGUMENT):
+    _message = "Data field is empty. (fields = {fields})"
+
+
+class ERROR_QUERY_GROUP_BY_OPTION(ERROR_INVALID_ARGUMENT):
+    _message = "Query group by option is invalid. (key = {key}, fields={fields})"
+
+
 class ERROR_NOT_SUPPORTED_OPERATOR(ERROR_INVALID_ARGUMENT):
     _message = "Data table does not support operator. (operator = {operator})"
 

src/spaceone/dashboard/manager/data_table_manager/__init__.py

@@ -11,6 +11,9 @@
     ERROR_NO_FIELDS_TO_GLOBAL_VARIABLES,
     ERROR_NOT_GLOBAL_VARIABLE_KEY,
     ERROR_QUERY_OPTION,
+    ERROR_QUERY_GROUP_BY_OPTION,
+    ERROR_EMPTY_DATA_FIELD,
+    ERROR_INVALID_SORT_OPTIONS,
 )
 
 _LOGGER = logging.getLogger(__name__)
@@ -28,6 +31,7 @@ def __init__(self, *args, **kwargs):
         self.jinja_variables = None
         self.state = None
         self.error_message = None
+        self.cache_hash_key = None
 
     def get_data_and_labels_info(self) -> Tuple[dict, dict]:
         raise NotImplementedError()
@@ -47,37 +51,23 @@ def load_from_widget(
         granularity: str,
         start: str,
         end: str,
+        group_by: list = None,
         sort: list = None,
         page: dict = None,
         vars: dict = None,
         column_sum: bool = False,
     ) -> dict:
+        self.check_group_by_and_sort_options(group_by, sort)
 
-        user_id = self.transaction.get_meta(
-            "authorization.user_id"
-        ) or self.transaction.get_meta("authorization.app_id")
-        role_type = self.transaction.get_meta("authorization.role_type")
+        query_data = self._prepare_query_data(
+            data_table_id, granularity, start, end, group_by, sort
+        )
 
-        query_data = {
-            "granularity": granularity,
-            "start": start,
-            "end": end,
-            "sort": sort,
-            "data_table_id": data_table_id,
-            "widget_id": self.widget_id,
-            "domain_id": self.domain_id,
-        }
+        self.cache_hash_key = utils.dict_to_hash(query_data)
+        response = self._get_cached_response()
 
-        if role_type == "WORKSPACE_MEMBER":
-            query_data["user_id"] = user_id
-
-        query_hash = utils.dict_to_hash(query_data)
+        if not response:
 
-        response = {"results": []}
-        if cache_data := cache.get(f"dashboard:Widget:load:{query_hash}"):
-            response = cache_data
-
-        else:
             self.load(
                 granularity,
                 start,
@@ -86,10 +76,14 @@ def load_from_widget(
             )
 
             if self.df is not None:
+                self._apply_group_by(group_by)
+
                 response = {
                     "results": self.df.copy(deep=True).to_dict(orient="records")
                 }
-                cache.set(f"dashboard:Widget:load:{query_hash}", response, expire=600)
+                cache.set(
+                    f"dashboard:Widget:load:{self.cache_hash_key}", response, expire=600
+                )
 
         if column_sum:
             return self.response_sum_data_from_widget(response)
@@ -116,7 +110,7 @@ def response_data_from_widget(
             "total_count": total_count,
         }
 
-    def response_sum_data_from_widget(self, response) -> dict:
+    def response_sum_data_from_widget(self, response: dict) -> dict:
         data = response["results"]
         if self.data_keys:
             sum_data = {
@@ -295,3 +289,63 @@ def change_expression_data_type(expression: str, gv_type_map: dict) -> str:
                 expression = expression.replace(gv_value, f'"{gv_value}"')
 
         return expression
+
+    @staticmethod
+    def check_group_by_and_sort_options(group_by: list, sort: list) -> None:
+        sort_keys = [sort_option["key"] for sort_option in sort]
+        for sort_key in sort_keys:
+            if sort_key not in group_by:
+                raise ERROR_INVALID_SORT_OPTIONS(group_by=group_by, sort=sort_keys)
+
+    def _prepare_query_data(
+        self,
+        data_table_id: str,
+        granularity: str,
+        start: str,
+        end: str,
+        group_by: list,
+        sort: list,
+    ) -> dict:
+        user_id = self.transaction.get_meta(
+            "authorization.user_id"
+        ) or self.transaction.get_meta("authorization.app_id")
+        role_type = self.transaction.get_meta("authorization.role_type")
+
+        query_data = {
+            "granularity": granularity,
+            "start": start,
+            "end": end,
+            "group_by": group_by,
+            "sort": sort,
+            "data_table_id": data_table_id,
+            "widget_id": self.widget_id,
+            "domain_id": self.domain_id,
+        }
+
+        if role_type == "WORKSPACE_MEMBER":
+            query_data["user_id"] = user_id
+
+        return query_data
+
+    def _get_cached_response(self) -> dict:
+        cache_data = cache.get(f"dashboard:Widget:load:{self.cache_hash_key}")
+        return cache_data if cache_data else None
+
+    def _apply_group_by(self, group_by: list):
+        if group_by:
+            for key in group_by:
+                if key not in self.df.columns:
+                    raise ERROR_QUERY_GROUP_BY_OPTION(
+                        key="group_by", fields=list(self.df.columns)
+                    )
+
+            agg_funcs = {
+                column: "sum"
+                for column in self.df.columns
+                if pd.api.types.is_numeric_dtype(self.df[column])
+            }
+
+            if not agg_funcs:
+                raise ERROR_EMPTY_DATA_FIELD(fields=list(self.df.columns))
+
+            self.df = self.df.groupby(group_by).agg(agg_funcs).reset_index()

src/spaceone/dashboard/model/private_data_table/database.py

@@ -18,6 +18,7 @@ class PrivateDataTable(MongoModel):
     labels_info = DictField(default=None)
     data_info = DictField(default=None)
     sort_keys = ListField(default=None)
+    cache_key = StringField(max_length=255, default=None, null=True)
     dashboard_id = StringField(max_length=40)
     widget_id = StringField(max_length=40)
     resource_group = StringField(
@@ -38,6 +39,7 @@ class PrivateDataTable(MongoModel):
             "labels_info",
             "data_info",
             "sort_keys",
+            "cache_key",
         ],
         "minimal_fields": [
             "data_table_id",

src/spaceone/dashboard/model/private_data_table/response.py

@@ -18,6 +18,7 @@ class PrivateDataTableResponse(BaseModel):
     labels_info: Union[dict, None] = None
     data_info: Union[dict, None] = None
     sort_keys: Union[List[str], None] = None
+    cache_key: Union[str, None] = None
     error_message: Union[str, None] = None
     dashboard_id: Union[str, None] = None
     widget_id: Union[str, None] = None

src/spaceone/dashboard/model/private_widget/request.py

@@ -52,6 +52,7 @@ class PrivateWidgetLoadRequest(BaseModel):
     granularity: str
     start: str
     end: str
+    group_by: Union[list, None] = None
     sort: Union[list, None] = None
     page: Union[dict, None] = None
     vars: Union[dict, None] = None

src/spaceone/dashboard/model/public_data_table/database.py

@@ -18,6 +18,7 @@ class PublicDataTable(MongoModel):
     labels_info = DictField(default=None)
     data_info = DictField(default=None)
     sort_keys = ListField(default=None)
+    cache_key = StringField(max_length=255, default=None, null=True)
     dashboard_id = StringField(max_length=40)
     widget_id = StringField(max_length=40)
     resource_group = StringField(
@@ -39,6 +40,7 @@ class PublicDataTable(MongoModel):
             "labels_info",
             "data_info",
             "sort_keys",
+            "cache_key",
             "project_id",
             "workspace_id",
         ],

src/spaceone/dashboard/model/public_data_table/response.py

@@ -20,6 +20,7 @@ class PublicDataTableResponse(BaseModel):
     labels_info: Union[dict, None] = None
     data_info: Union[dict, None] = None
     sort_keys: Union[List[str], None] = None
+    cache_key: Union[str, None] = None
     error_message: Union[str, None] = None
     dashboard_id: Union[str, None] = None
     widget_id: Union[str, None] = None

src/spaceone/dashboard/model/public_widget/request.py

@@ -62,6 +62,7 @@ class PublicWidgetLoadRequest(BaseModel):
     granularity: str
     start: str
     end: str
+    group_by: Union[list, None] = None
     sort: Union[list, None] = None
     page: Union[dict, None] = None
     vars: Union[dict, None] = None

src/spaceone/dashboard/service/private_data_table_service.py

@@ -2,6 +2,7 @@
 import copy
 from typing import Union
 
+from spaceone.core import cache
 from spaceone.core.service import *
 from spaceone.core.error import *
 
@@ -312,6 +313,13 @@ def update(
             if raw_filter:
                 params_dict["options"]["filter"] = raw_filter
 
+        if pri_data_table_vo.cache_key:
+            cache_key = f"dashboard:Widget:load:{pri_data_table_vo.cache_key}"
+            if cache.get(cache_key):
+                cache.delete(cache_key)
+
+            params_dict["cache_key"] = None
+
         pri_data_table_vo = self.pri_data_table_mgr.update_private_data_table_by_vo(
             params_dict, pri_data_table_vo
         )
@@ -345,6 +353,11 @@ def delete(self, params: PrivateDataTableDeleteRequest) -> None:
             )
         )
 
+        if pri_data_table_vo.cache_key:
+            cache_key = f"dashboard:Widget:load:{pri_data_table_vo.cache_key}"
+            if cache.get(cache_key):
+                cache.delete(cache_key)
+
         self.pri_data_table_mgr.delete_private_data_table_by_vo(pri_data_table_vo)
 
     @transaction(

src/spaceone/dashboard/service/private_widget_service.py

@@ -345,6 +345,7 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict:
                 'granularity': 'str',           # required
                 'start': 'str',                 # required
                 'end': 'str',                   # required
+                'group_by': 'list',
                 'sort': 'list',
                 'page': 'dict',
                 'vars': 'dict',
@@ -382,15 +383,18 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict:
                 pri_data_table_vo.widget_id,
                 pri_data_table_vo.domain_id,
             )
-            return ds_mgr.load_from_widget(
+            response = ds_mgr.load_from_widget(
                 pri_data_table_vo.data_table_id,
                 params.granularity,
                 params.start,
                 params.end,
+                params.group_by,
                 params.sort,
                 params.page,
                 params.vars,
             )
+
+            cached_hash_key = ds_mgr.cache_hash_key
         else:
             operator = pri_data_table_vo.operator
             options = pri_data_table_vo.options.get(operator, {})
@@ -402,16 +406,25 @@ def load(self, params: PrivateWidgetLoadRequest) -> dict:
                 pri_data_table_vo.widget_id,
                 pri_data_table_vo.domain_id,
             )
-            return dt_mgr.load_from_widget(
+            response = dt_mgr.load_from_widget(
                 pri_data_table_vo.data_table_id,
                 params.granularity,
                 params.start,
                 params.end,
                 params.sort,
+                params.group_by,
                 params.page,
                 params.vars,
             )
 
+            cached_hash_key = dt_mgr.cache_hash_key
+        if cached_hash_key and pri_data_table_vo.cache_key != cached_hash_key:
+            pri_data_table_mgr.update_private_data_table_by_vo(
+                {"cache_key": cached_hash_key}, pri_data_table_vo
+            )
+
+        return response
+
     @transaction(
         permission="dashboard:PrivateWidget.read",
         role_types=["USER"],
@@ -463,14 +476,17 @@ def load_sum(self, params: PrivateWidgetLoadSumRequest) -> dict:
                 pri_data_table_vo.widget_id,
                 pri_data_table_vo.domain_id,
             )
-            return ds_mgr.load_from_widget(
+            response = ds_mgr.load_from_widget(
                 pri_data_table_vo.data_table_id,
                 params.granularity,
                 params.start,
                 params.end,
                 vars=params.vars,
                 column_sum=True,
             )
+
+            cached_hash_key = ds_mgr.cache_hash_key
+
         else:
             operator = pri_data_table_vo.operator
             options = pri_data_table_vo.options.get(operator, {})
@@ -482,7 +498,7 @@ def load_sum(self, params: PrivateWidgetLoadSumRequest) -> dict:
                 pri_data_table_vo.widget_id,
                 pri_data_table_vo.domain_id,
             )
-            return dt_mgr.load_from_widget(
+            response = dt_mgr.load_from_widget(
                 pri_data_table_vo.data_table_id,
                 params.granularity,
                 params.start,
@@ -491,6 +507,15 @@ def load_sum(self, params: PrivateWidgetLoadSumRequest) -> dict:
                 column_sum=True,
             )
 
+            cached_hash_key = dt_mgr.cache_hash_key
+
+        if cached_hash_key and pri_data_table_vo.cache_key != cached_hash_key:
+            pri_data_table_mgr.update_private_data_table_by_vo(
+                {"cache_key": cached_hash_key}, pri_data_table_vo
+            )
+
+        return response
+
     @transaction(
         permission="dashboard:PrivateWidget.read",
         role_types=["USER"],