fix: optimised upload gstr1

india_compliance/gst_india/doctype/gst_return_log/generate_gstr_1.py

@@ -5,7 +5,7 @@
 
 import frappe
 from frappe import unscrub
-from frappe.utils import flt
+from frappe.utils import floor, flt
 
 from india_compliance.gst_india.api_classes.taxpayer_returns import GSTR1API
 from india_compliance.gst_india.utils.gstr_1 import GSTR1_SubCategory
@@ -31,6 +31,7 @@
     "ER": "Error",
     "IP": "In Progress",
 }
+MAXIMUM_UPLOAD_SIZE = 5200000
 
 
 class SummarizeGSTR1:
@@ -891,41 +892,96 @@ def file_gstr1(self, pan, otp=None):
         return response
 
 
-# TODO : optimse this
+def is_within_limit(result):
+    return len(json.dumps(result, indent=4)) < MAXIMUM_UPLOAD_SIZE
+
+
 def get_partitioned_data(json_data):
-    result = []
+    if is_within_limit(json_data):
+        yield json_data
+        return
+
     base_data = {"gstin": json_data["gstin"], "fp": json_data["fp"]}
+    result = {}
+    has_yielded = False
+
+    for category, category_data in json_data.items():
+        result[category] = category_data
+
+        if is_within_limit(result):
+            continue
+
+        # 2 case : 1st - large categories, 2nd - combined category data is large
+        del result[category]
+        if result.keys() in [
+            "b2b",
+            "cdnr",
+        ]:  # Ensure at least one category is present before yielding the result
+            yield result
+
+        result = base_data.copy()
+        result[category] = category_data
+
+        if is_within_limit(result):
+            continue
+
+        has_yielded = True
+        # Handle the case where individual objects within the category need to be partitioned
+        yield from partition_by_objects(result, category, category_data, base_data)
+
+    if not has_yielded:
+        yield result
 
-    partial_data = base_data.copy()
-    gst_categories = [
-        category for category in json_data.keys() if category not in ["gstin", "fp"]
-    ]
 
-    for category in gst_categories:
-        partial_data[category] = []
-        index = 0
+def partition_by_objects(result, category, category_data, base_data):
+    result[category] = []
+    has_yielded = False
+    for object in category_data:
+        result[category].append(object)
+
+        if is_within_limit(result):
+            continue
+
+        # 2 case: 1st object is so big, combine 2 object is big
+        object_data = result[category].pop()
+        if result[
+            category
+        ]:  # Ensure at least one object is present before yielding the result
+            yield result
+
+        result = base_data.copy()
+        result[category] = [object_data]
+
+        if is_within_limit(result):
+            continue
 
-        for obj in json_data.get(category, []):
-            partial_data[category].append({"ctin": obj["ctin"], "inv": []})
+        has_yielded = True
+        # Handle the case where invoices within the object need to be partitioned
+        yield from partition_by_invoices(result, category, object)
 
-            for invoice in obj.get("inv", []):
-                partial_data[category][index]["inv"].append(invoice)
+    if not has_yielded:
+        yield result
 
-                if len(json.dumps(partial_data)) > 5200000:
-                    invoice = partial_data[category][index]["inv"].pop()
-                    result.append(partial_data)
 
-                    partial_data = base_data.copy()
-                    partial_data[category] = [{"ctin": obj["ctin"], "inv": []}]
+def partition_by_invoices(result, category, object):
+    result[category] = [{"ctin": object.get("ctin"), "inv": []}]
 
-                    partial_data[category][0]["inv"].append(invoice)
+    result_size = len(json.dumps(result, indent=4))
+    invoice_size = 1200
+    invoice_to_add = floor((MAXIMUM_UPLOAD_SIZE - result_size) / invoice_size)
+    count = 0
 
-            index += 1
+    for invoice in object.get("inv"):
+        count += 1
+        result[category][-1]["inv"].append(invoice)
 
-    if partial_data[category][index - 1]["inv"]:
-        result.append(partial_data)
+        if count == invoice_to_add:
+            count = 0
+            yield result
+            result[category] = [{"ctin": object.get("ctin"), "inv": []}]
 
-    return result
+    if result[category][-1]["inv"]:
+        yield result
 
 
 def get_differing_categories(mapped_summary, gov_summary):

india_compliance/gst_india/doctype/gstr_1_beta/gstr_1_beta.js

@@ -320,6 +320,7 @@ async function file_gstr1_data(frm) {
 }
 
 function perform_gstr1_action(frm, action, additional_args = {}) {
+    frm.gstr1.tabs.error_tab.hide();
     const base_args = {
         month_or_quarter: frm.doc.month_or_quarter,
         year: frm.doc.year,