Merge branch 'dash-api' of https://github.com/KlimaDAO/dash-apps into…

… dash-api

src/apps/api/app.py

@@ -29,6 +29,7 @@ def output_json(data, code, headers=None):
 api.add_resource(endpoints.CreditsProjectsAggregation, '/credits/agg/projects')
 api.add_resource(endpoints.CreditsMethodologiesAggregation, '/credits/agg/methodologies')
 api.add_resource(endpoints.CreditsVintageAggregation, '/credits/agg/vintage')
+api.add_resource(endpoints.CreditsPoolAggregation, '/credits/agg/pool')
 api.add_resource(endpoints.CreditsPoolVintageAggregation, '/credits/agg/pool/vintage')
 api.add_resource(endpoints.CreditsPoolMethodologyAggregation, '/credits/agg/pool/methodology')
 api.add_resource(endpoints.CreditsPoolDatesAggregation, '/credits/agg/pool/<string:freq>')

src/apps/api/endpoints/__init__.py

@@ -7,6 +7,7 @@
     CreditsProjectsAggregation,
     CreditsMethodologiesAggregation,
     CreditsVintageAggregation,
+    CreditsPoolAggregation,
     CreditsPoolVintageAggregation,
     CreditsPoolMethodologyAggregation,
     CreditsPoolDatesAggregation,

src/apps/api/endpoints/credits.py

@@ -76,6 +76,9 @@ def get_credits(self, bridge=None):
 
         return df
 
+    def get_pool_credits(self, bridge=None):
+        return self.get_credits(bridge).pool_analysis()
+
 
 class CreditsRaw(AbstractCredits):
     @layout_cache.cached(query_string=True)
@@ -177,7 +180,19 @@ class CreditsPoolVintageAggregation(AbstractCredits):
     )
     @helpers.with_output_formatter
     def get(self):
-        credits = self.get_credits().vintage_agg().pool_summary("vintage")
+        credits = self.get_pool_credits().vintage_agg().pool_summary("vintage")
+        return credits
+
+
+class CreditsPoolAggregation(AbstractCredits):
+    @layout_cache.cached(query_string=True)
+    @helpers.with_errors_handler
+    @helpers.with_help(
+        f"""{BASE_HELP}
+        """
+    )
+    def get(self):
+        credits = self.get_pool_credits().pool_summary().resolve().to_dict(orient='records')[0]
         return credits
 
 
@@ -191,7 +206,7 @@ class CreditsPoolMethodologyAggregation(AbstractCredits):
     )
     @helpers.with_output_formatter
     def get(self):
-        credits = self.get_credits().methodologies_agg().pool_summary("methodology")
+        credits = self.get_pool_credits().methodologies_agg().pool_summary("methodology")
         return credits
 
 
@@ -206,7 +221,7 @@ class CreditsPoolDatesAggregation(AbstractCredits):
     @helpers.with_output_formatter
     def get(self, freq):
         date_column = self.get_default_date_field()
-        credits = self.get_credits().date_agg(date_column, freq).pool_summary(date_column)
+        credits = self.get_pool_credits().date_agg(date_column, freq).pool_summary(date_column)
         return credits
 
 

src/apps/api/test.py

@@ -13,6 +13,7 @@
     "credits/agg/projects",
     "credits/agg/methodologies",
     "credits/agg/vintage",
+    "credits/agg/pool",
     "credits/agg/pool/vintage",
     "credits/agg/pool/methodology",
     "credits/agg/pool/daily",

src/apps/services/cache.py

@@ -188,7 +188,7 @@ def daily_agg(self, df, columns):
         date_column = columns[0]
         """Adds an aggregation by day"""
         df = self.date_manipulations(df, date_column, "daily")
-        df = df.groupby(columns)
+        df = df.groupby(columns, group_keys=False)
         return df
 
     def monthly_agg(self, df, columns):
@@ -197,7 +197,7 @@ def monthly_agg(self, df, columns):
             columns = [columns]
         date_column = columns[0]
         df = self.date_manipulations(df, date_column, "monthly")
-        df = df.groupby(columns)
+        df = df.groupby(columns, group_keys=False)
         return df
 
     @final_cached_command()
@@ -226,7 +226,7 @@ def sum(self, df, column):
     def sum_over_time(self, df, date_column, column, freq):
         df = self.date_manipulations(df, date_column, freq)
         df = df.sort_values(by=date_column, ascending=True)
-        df = df.groupby(date_column)[column].sum().to_frame().reset_index()
+        df = df.groupby(date_column, group_keys=False)[column].sum().to_frame().reset_index()
         df[column] = df[column].cumsum()
         return df
 
@@ -238,7 +238,7 @@ def cumsum(self, df, column):
     @chained_cached_command()
     def monthly_sample(self, df, date_column):
         """Samples daily data into monthly data"""
-        return df.groupby(pd.DatetimeIndex(df[date_column]).to_period('M')).nth(-1).reset_index(drop=True)
+        return df.groupby(pd.DatetimeIndex(df[date_column]).to_period('M'), group_keys=False).nth(-1).reset_index(drop=True)
 
     def date_manipulations(self, df, date_column, freq):
         if date_column not in df:

src/apps/services/credits.py

@@ -95,26 +95,27 @@ def filter(self, df, bridge, pool, status):
     @chained_cached_command()
     def vintage_agg(self, df):
         """Adds an aggregation on vintage"""
-        df = df.groupby("vintage")
+        df = df.groupby("vintage", group_keys=False)
         return df
 
     @chained_cached_command()
     def countries_agg(self, df):
-        df = df.groupby(["country", "country_code"])
+        df = df.groupby(["country", "country_code"], group_keys=False)
         return df
 
     @chained_cached_command()
     def projects_agg(self, df):
-        df = df.groupby("project_type")
+        df = df.groupby("project_type", group_keys=False)
         return df
 
     @chained_cached_command()
     def methodologies_agg(self, df):
-        df = df.groupby("methodology")
+        df = df.groupby("methodology", group_keys=False)
         return df
 
+
     @chained_cached_command()
-    def pool_summary(self, df, date_field):
+    def pool_summary(self, df, kept_fields=[]):
         columns = [
             "quantity",
             "total_quantity",
@@ -124,16 +125,27 @@ def pool_summary(self, df, date_field):
             "nbo_quantity",
             "mco2_quantity"
         ]
+        if isinstance(df, pd.DataFrame):
+            df = df.groupby(lambda x: True, group_keys=False)
+
+        if not isinstance(kept_fields, list):
+            kept_fields = [kept_fields]
 
         def summary(df):
             res_df = pd.DataFrame()
-            res_df[date_field] = [df[date_field].iloc[0]]
+            for kept_field in kept_fields:
+                res_df[kept_field] = [df[kept_field].iloc[0]]
 
+            not_pooled_quantity = df["total_quantity"].sum()
             for column in columns:
                 if column in df:
-                    res_df[column] = [df[column].sum()]
+                    column_quantity = df[column].sum()
+                    res_df[column] = [column_quantity]
+                    if column not in ["quantity", "total_quantity"]:
+                        not_pooled_quantity -= column_quantity
+            res_df["not_pooled_quantity"] = [not_pooled_quantity]
+
             return res_df
-
         df = df.apply(summary).reset_index(drop=True)
         return df
 
@@ -195,7 +207,9 @@ def filter_pool_quantity(self, df, quantity_column):
 
         return df
 
-    def drop_duplicates(self, df):
+    @chained_cached_command()
+    def pool_analysis(self, df):
+        """When analysing pools we need a subset of the data"""
         df = df.drop_duplicates(subset=["token_address"], keep="first")
         df = df.reset_index(drop=True)
         return df

src/apps/services/retirements.py

@@ -77,10 +77,10 @@ def summary(df):
 
     @chained_cached_command()
     def beneficiaries_agg(self, df):
-        df = df.groupby("beneficiary")
+        df = df.groupby("beneficiary", group_keys=False)
         return df
 
     @chained_cached_command()
     def tokens_agg(self, df):
-        df = df.groupby("token")
+        df = df.groupby("token", group_keys=False)
         return df