rethinking multiprocess logger

countess/core/logger.py

@@ -2,23 +2,24 @@
 
 import sys
 import traceback
-from typing import Optional
-
+from typing import Optional, Iterable, Tuple
+import multiprocessing
+import queue
 
 class Logger:
     """Logger Base Class"""
 
     def __init__(self):
         pass
 
-    def progress(self, message: str = "Running", percentage: Optional[int] = None):
-        """Record progress"""
-        return None
-
     def log(self, level: str, message: str, detail: Optional[str] = None):
         """Log a message"""
         return None
 
+    def progress(self, message: str = "Running", percentage: Optional[int] = None):
+        """Record progress"""
+        self.log("progress", message, str(percentage) if percentage else None)
+
     def info(self, message: str, detail: Optional[str] = None):
         """Log a message at level info"""
         self.log("info", message, detail)
@@ -49,17 +50,26 @@ def __init__(self, stdout=sys.stdout, stderr=sys.stderr, prefix: Optional[str] =
         self.stderr = stderr
         self.prefix = prefix
 
-    def progress(self, message: str = "Running", percentage: Optional[int] = None):
-        if self.prefix:
-            message = self.prefix + ": " + message
-        if percentage:
-            message += f" [{int(percentage):2d}%]"
-        self.stdout.write(f"{message}\n")
-
     def log(self, level: str, message: str, detail: Optional[str] = None):
         if self.prefix:
             message = self.prefix + ": " + message
         if detail:
             message += " " + repr(detail)
 
         self.stderr.write(message + "\n")
+
+
+class MultiprocessLogger(Logger):
+
+    def __init__(self):
+        self.queue = multiprocessing.Queue()
+
+    def log(self, level: str, message: str, detail: Optional[str] = None):
+        self.queue.put((level, message, detail))
+
+    def poll(self) -> Iterable[Tuple[str, str, str]]:
+        try:
+            while True:
+                yield self.queue.get_nowait()
+        except queue.Empty:
+            pass

countess/gui/logger.py

@@ -4,7 +4,7 @@
 from tkinter import ttk
 from typing import MutableMapping, Optional
 
-from countess.core.logger import Logger
+from countess.core.logger import MultiprocessLogger, Logger
 
 
 class LoggerTreeview(ttk.Treeview):
@@ -57,6 +57,7 @@ def __init__(self, tk_parent, *a, **k):
 
         self.treeview = LoggerTreeview(self)
         self.treeview.grid(row=0, column=0, sticky=tk.NSEW)
+        self.treeview.bind("<<TreeviewSelect>>", self.on_click)
 
         self.scrollbar_x = ttk.Scrollbar(self, orient=tk.HORIZONTAL, command=self.treeview.xview)
         self.scrollbar_x.grid(row=1, column=0, sticky=tk.EW)
@@ -70,8 +71,45 @@ def __init__(self, tk_parent, *a, **k):
         self.progress_frame.grid(row=2, columnspan=2, sticky=tk.EW)
         self.progress_frame.columnconfigure(0, weight=1)
 
+        self.logger = MultiprocessLogger()
+        self.count = 0
+        self.details = {}
+        self.progress_bars = {}
+
+        self.poll()
+
+    def poll(self):
+        datetime_now = datetime.datetime.now()
+        for level, message, detail in self.logger.poll():
+            if level == 'progress':
+                try:
+                    pbar = self.progress_bars[message]
+                except KeyError:
+                    pbar = self.progress_bars[message] = LabeledProgressbar(self.progress_frame, mode="determinate", value=0)
+                    pbar.update_label(message)
+                    pbar.grid(sticky=tk.EW)
+
+                if detail is not None:
+                    pbar.config(mode="determinate", value=int(detail))
+                    pbar.update_label(f"{message} {detail}%")
+                else:
+                    pbar.config(mode="indeterminate")
+                    pbar.step(5)
+                    pbar.update_label(f"{message}")
+            else:
+                self.count += 1
+                iid = self.treeview.insert("", "end", text=datetime_now.isoformat(), values=(level, message))
+                if detail:
+                    self.details[iid] = detail
+        self.after(100, self.poll)
+
     def get_logger(self, name: str):
-        return TreeviewLogger(self.treeview, self.progress_frame, name)
+        return self.logger
+
+    def on_click(self, _):
+        # XXX display detail more nicely
+        TreeviewDetailWindow(self.details[self.treeview.focus()])
+
 
 
 class TreeviewDetailWindow(tk.Toplevel):
@@ -88,70 +126,3 @@ def __init__(self, detail, *a, **k):
 
         button = tk.Button(self, text="CLOSE", command=self.destroy)
         button.grid(sticky=tk.EW)
-
-
-class TreeviewLogger(Logger):
-    def __init__(self, treeview: ttk.Treeview, progress_frame: tk.Frame, name: str):
-        self.treeview = treeview
-        self.progress_bar = LabeledProgressbar(progress_frame, mode="determinate", value=0)
-        self.progress_bar.update_label(name)
-        self.name = name
-        self.count = 0
-        self.treeview["height"] = 0
-        self.detail: MutableMapping[str, str] = {}
-
-        self.treeview.bind("<<TreeviewSelect>>", self.on_click)
-
-        self.queue: queue.Queue = queue.Queue()
-
-    def on_click(self, event):
-        # XXX display detail more nicely
-        TreeviewDetailWindow(self.detail[self.treeview.focus()])
-
-    def poll(self):
-        # XXX dask apply etc don't seem to be thread safe when updating Tk, so
-        # this adds in a queue to separate the two.
-        try:
-            while True:
-                level, message, detail = self.queue.get_nowait()
-                if level == "progress":
-                    self.progress_bar.grid(sticky=tk.EW)
-                    if detail is not None:
-                        self.progress_bar.config(mode="determinate", value=detail)
-                        self.progress_bar.update_label(f"{self.name}: {message} {detail}%")
-                    else:
-                        self.progress_bar.config(mode="indeterminate")
-                        self.progress_bar.step(5)
-                        self.progress_bar.update_label(f"{self.name}: {message}")
-                else:
-                    self.count += 1
-                    datetime_now = datetime.datetime.now()
-                    values = [self.name, message]
-                    iid = self.treeview.insert("", "end", text=datetime_now.isoformat(), values=values)
-                    if detail is not None:
-                        self.detail[iid] = detail
-                    self.treeview["height"] = min(self.count, 10)
-
-        except queue.Empty:
-            pass
-
-    def log(self, level: str, message: str, detail: Optional[str] = None):
-        print(level, message, detail)
-        self.queue.put((level, message, detail))
-
-    def progress(self, message: str = "Running", percentage: Optional[int] = None):
-        self.queue.put(("progress", message, percentage))
-
-    def progress_hide(self):
-        self.progress_bar.grid_forget()
-
-    def clear(self):
-        self.progress_bar.config(mode="determinate", value=0)
-        self.progress_bar.update_label("")
-        for row in self.treeview.get_children():
-            self.treeview.delete(row)
-        self.count = 0
-        self.detail = {}
-
-    def __del__(self):
-        self.progress_bar.after(5000, lambda pbar=self.progress_bar: pbar.destroy())