Merge pull request #11 from karlicoss/upd

core updates: better sanitizing/docs/naming

README.org

@@ -1,90 +1,136 @@
 # -*- org-confirm-babel-evaluate: nil; -*-
 
-Orger converts your data into Org-mode to allow for quick access and search.
+Orger converts your data into a hierarchical Org-mode representation to allow for quick access and search.
 
 I write in detail about usecases and motivation for it [[https://beepb00p.xyz/orger.html][here]], this readme is mostly the setup manual!
 
 * Installing
 
-[[https://pypi.org/project/orger][PyPi]]: ~pip3 install --user orger~
+- simplest: install from [[https://pypi.org/project/orger][PyPi]]: ~pip3 install --user orger~
+
+  After that you should be able to run orger modules via =python3 -m=:
+
+  : python3 -m orger.modules.instapaper --help
+
+- [[https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs][editable]] install
+
+  This will allow you to quickly prototype and debug, the local changes to the code will be reflected immiedately.
+
+  - clone: =https://github.com/karlicoss/orger /path/to/orger=
+  - =cd /path/to/orger=
+  - =pip3 install --user .=
+  - after that you can use =python3 -m orger.modules.modulename=, same way as the previous section, or run =modules/modulename.py= directly
 
 * Usage and examples
-There are several examples [[https://beepb00p.xyz/orger.html#examples][here]], including a [[https://www.youtube.com/watch?v=ib_PDJpTh-Q][screencast]].
-
-See [[./modules/pocket_demo.py][pocket_demo]] for a documented literate demo and [[./modules][modules]] for some of modules I'm using.
-
-Here's a quick minimalistic demo.
-
-#+BEGIN_SRC python
-  from orger import StaticView
-  from orger.inorganic import node, link
-  from orger.common import dt_heading
-
-  import my.coding.github as github_data
-
-  class Github(StaticView):
-    def get_items(self):
-      for event in github_data.get_events():
-        yield node(dt_heading(event.dt, event.summary))
-
-  Github.main()
-#+END_SRC
-
-That ten line program results in a file =Github.org=:
-
-#+BEGIN_SRC org
-  # AUTOGENERATED BY /code/orger/github.py
-
-  ,* [2016-10-30 Sun 10:29] opened PR Add __enter__ and __exit__ to Pool stub
-  ,* [2016-11-10 Thu 09:29] opened PR Update gradle to 2.14.1 and gradle plugin to 2.1.1
-  ,* [2016-11-16 Wed 20:20] commented on issue Linker error makes it impossible to use a stack-provided ghc
-  ,* [2016-12-30 Fri 11:57] commented on issue Fix performance in the rare case of hashCode evaluating to zero 
-  ,* [2019-09-21 Sat 16:51] commented on issue Tags containing letters outside of a-zA-Z
-  ....
-#+END_SRC
-
-I run it automatically through Cron every night.
-
-The trick to accessing data is in ~import my.coding.github~.
-You can learn about setting it up and using [[https://github.com/karlicoss/HPI][here]].
-
-
-# TODO Use :session t???
-
-
-** types of views
- - 
-    #+BEGIN_SRC python :exports results :results raw
-      import sys
-      sys.path.insert(0, 'src')
-      import orger
-      return orger.org_view.StaticView.__doc__
-    #+END_SRC
-
-   You can run static module as:
-
-   #+BEGIN_SRC bash
-     ./orger_module.py --to /path/to/output.org
-   #+END_SRC
-
-
- - 
-    #+BEGIN_SRC python :exports results :results raw
-      import sys
-      sys.path.insert(0, 'src')
-      import orger
-      return orger.org_view.InteractiveView.__doc__
-    #+END_SRC
+I usually run Orger modules overnight via cron.
+
+- see [[./modules][modules]] for *all available modules*
+- Most modules are using [[https://github.com/karlicoss/HPI][HPI]] package for accessing the data.
+  You can learn about setting it up and using [[https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#orger][here]].
+- several examples [[https://beepb00p.xyz/orger.html#examples][here]]
+- [[https://beepb00p.xyz/myinfra-roam.html#orger][demonstration]] of Roam Research module, including a [[https://www.youtube.com/watch?v=ib_PDJpTh-Q][screencast]]
+- [[./modules/pocket_demo.py][pocket_demo]]: documented literate demo
+- and a short short demo:
+
+  #+BEGIN_SRC python
+    from orger import Mirror
+    from orger.inorganic import node, link
+    from orger.common import dt_heading
+
+    import my.coding.github as github_data
+
+    class Github(Mirror):
+      def get_items(self):
+        for event in github_data.get_events():
+          yield node(dt_heading(event.dt, event.summary))
+
+    Github.main()
+  #+END_SRC
+
+  That ten line program, when run (=./modules/github.py=), results in a file =Github.org=:
+
+  #+BEGIN_SRC org
+    # AUTOGENERATED BY /code/orger/github.py
+
+    ,* [2016-10-30 Sun 10:29] opened PR Add __enter__ and __exit__ to Pool stub
+    ,* [2016-11-10 Thu 09:29] opened PR Update gradle to 2.14.1 and gradle plugin to 2.1.1
+    ,* [2016-11-16 Wed 20:20] commented on issue Linker error makes it impossible to use a stack-provided ghc
+    ,* [2016-12-30 Fri 11:57] commented on issue Fix performance in the rare case of hashCode evaluating to zero
+    ,* [2019-09-21 Sat 16:51] commented on issue Tags containing letters outside of a-zA-Z
+    ....
+  #+END_SRC
+
+* types of modules
+- Mirror
+
+  #+begin_src python :dir src :exports results :results drawer output
+import orger
+print(orger.Mirror.__doc__)
+  #+end_src
+
+  #+RESULTS:
+  :results:
+
+      *Mirror* (old name =StaticView=): mirrors *all data* from a source, and generated from scratch every time, hence *read only*.
+
+  :end:
+
+     You can run such module with
+
+     : ./orger_module.py --to /path/to/output.org
+
+- Queue
+
+  #+BEGIN_SRC python :dir src :exports results :results drawer output
+import orger
+print(orger.Queue.__doc__)
+  #+END_SRC
+
+  #+RESULTS:
+  :results:
+
+      *Queue* (old name =InteractiveView=): works as a queue, *only previously unseen items* from the data source are appended to the output org-mode file.
+
+      To keep track of old/new items, it's using a separate JSON =state= file.
+
+      A typical usecase is a todo list, or a content processing queue.
+      You can use such a module as you use any other org-mode file: schedule/refile/comment/set priorities, etc.
+
+  :end:
+
    Typically you'd want to use these as a source of tasks for your todo list. See [[./modules/ip2org.py][ip2org]] as an example.
 
-   You can run interactive module as:
+   You can run such a module as:
+
+   :  # initialize the state file first to avoid surprises (you only need to do it once)
+   :  ./orger_module.py --to /path/to/output.org --state /path/to/state.json --init
+   :  # after that you can just run it:
+   :  ./orger_module.py --to /path/to/output.org --state /path/to/state.json
+
+* FAQ
+- Why are the files output by some modules read only?
+
+  =Mirror= type modules output read only files, so you don't modify them by accident, they are overwritten every time.
+
+  If you want to temporary lift this restriction (e.g. to experiment with the format), you can use =chmod +w=, or =M-x toggle-read-only= in Emacs.
+
+- How is it different from [[https://github.com/novoid/Memacs][Memacs]]?
+
+  The main reason Orger exists is because I discovered Memacs after I wrote Orger!
+  One day we might merge them, or at least [[https://github.com/karlicoss/orger/issues/5][reuse org-mode formatting routines]].
+
+  That said there are some differences at the moment:
+
+  - Memacs is more of a lifelogging utility, generating a linear output with the intent to be used with your org agenda
+  - Orger's =Mirror= modules are meant to be more of a full local reflection of a data source, preserving the hierarchy as much as possible
+  - Orger's =Queue= module: I believe they don't have Memacs analogue (but please correct me if I'm wrong)
+  - Orger modules are slim and relying on [[https://github.com/karlicoss/HPI][HPI]] to encapsulate data access. But you can also use HPI with Memacs, please ping me if you set up such an integration!
+
+- I want active timestamps for org-agenda integration
+
+  Pass the =--timestamp= argument to the module, for example:
 
-   #+BEGIN_SRC bash
-     # initialise the state file first to avoid surprises (you only need to do it once)
-     ./orger_module.py --to /path/to/output.org --state /path/to/state.json --init
-     # after that you can just run it:
-     ./orger_module.py --to /path/to/output.org --state /path/to/state.json
-   #+END_SRC
+  : modules/polar.py --timestamps active
 
 * Similar projects
 - [[https://github.com/novoid/Memacs][Memacs by novoid]]

modules/instapaper.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python3
-from orger import StaticView
+from orger import Mirror
 from orger.inorganic import node, link
 from orger.common import dt_heading
 
 from my.instapaper import pages
 
 
-class IpView(StaticView):
+class Instapaper(Mirror):
     def get_items(self):
         for page in pages():
             yield node(
@@ -28,11 +28,11 @@ def get_items(self):
         # TODO better error handling, cooperate with org_tools
 
 # TODO move tests to separate files, otherwise they would annoy other people
-test = IpView.make_test(
+test = Instapaper.make_test(
     heading='Life Extension Methods',
     contains='sleep a lot',
 )
 
 
 if __name__ == '__main__':
-    IpView.main()
+    Instapaper.main()

modules/materialistic.py

@@ -4,13 +4,13 @@
 https://play.google.com/store/apps/details?id=io.github.hidroh.materialistic
 """
 
-from orger import InteractiveView
+from orger import Queue
 from orger.inorganic import node, link
 from orger.common import dt_heading
 
 from my.materialistic import saves
 
-class MaterialisticView(InteractiveView):
+class Materialistic(Queue):
     def get_items(self):
         for s in saves():
             yield s.uid, node(
@@ -23,4 +23,4 @@ def get_items(self):
 
 
 if __name__ == '__main__':
-    MaterialisticView.main()
+    Materialistic.main()

modules/pocket_demo.py

@@ -74,15 +74,15 @@ def get_articles(json_path: Path) -> Sequence[Article]:
 """
 Ok, now we can get to implementing the adapter.
 """
-from orger import StaticView
+from orger import Mirror
 """
-StaticView means it's meant to be read-only view onto data (as opposed to InteractiveView).
+Mirror means it's meant to be read-only view onto data (as opposed to Queue).
 """
 from orger.inorganic import node, link
 from orger.common import dt_heading
 
 
-class PocketView(StaticView):
+class Pocket(Mirror):
     def get_items(self):
         """
         get_items returns a sequence/iterator of nodes
@@ -113,7 +113,7 @@ def setup_parser(p):
     """
     Usage example: ./pocket.py --file /backups/pocket/last-backup.json --to /data/orger/pocket.org
     """
-    PocketView.main(setup_parser=setup_parser)
+    Pocket.main(setup_parser=setup_parser)
 
 """
 Example pocket.org output:

mypy.ini

@@ -0,0 +1,10 @@
+[mypy]
+pretty = True
+show_error_context = True
+show_error_codes = True
+check_untyped_defs = True
+namespace_packages = True
+
+# an example of suppressing
+# [mypy-my.config.repos.pdfannots.pdfannots]
+# ignore_errors = True

pytest.ini

@@ -3,6 +3,7 @@
 python_files = *.py
 addopts =
   --verbose
+  --ignore=src/orger/modules
 
   # otherwise it won't discover doctests
   --doctest-modules

setup.py

@@ -17,8 +17,13 @@ def main():
 
         zip_safe=False,
 
-        packages=[pkg],
-        package_dir={'': 'src'},
+        packages=[pkg, 'orger.modules'],
+        # TODO ugh, it's weird. If it's editable install, python3 -m 'orger.modules.module' doesn't work. But it does if installed without editable??
+        # for not solved with a symlink...
+        package_dir={
+            '': 'src',
+            'orger.modules': 'modules',
+        },
         package_data={pkg: ['py.typed']},
 
         ## ^^^ this should be mostly automatic and not requiring any changes
@@ -31,7 +36,7 @@ def main():
         install_requires=['atomicwrites'],
         extras_require={
             'testing': ['pytest'],
-            'linting': ['pytest', 'mypy', 'pylint'],
+            'linting': ['pytest', 'mypy'],
         },
     )
 

src/orger/__init__.py

@@ -1,4 +1,4 @@
 from .org_view import StaticView, InteractiveView, OrgWithKey
-# TODO just rename org_view to init?
-
+from .org_view import Mirror, Queue
 
+# TODO just rename org_view to init?

src/orger/common.py

@@ -1,7 +1,11 @@
 from datetime import datetime
 from typing import Optional
 
-from .inorganic import OrgNode, timestamp
+from .inorganic import OrgNode, timestamp, timestamp_with_style, TimestampStyle
+
+
+class settings:
+    DEFAULT_TIMESTAMP_STYLE = TimestampStyle.INACTIVE
 
 
 def dt_heading(dt: Optional[datetime], heading: str):
@@ -12,7 +16,7 @@ def dt_heading(dt: Optional[datetime], heading: str):
     if dt is None:
         return heading
     else:
-        return timestamp(dt, inactive=True) + ' ' + heading
+        return timestamp_with_style(dt=dt, style=settings.DEFAULT_TIMESTAMP_STYLE) + ' ' + heading
 
 
 def error(e: Exception) -> OrgNode:
@@ -36,4 +40,5 @@ def todo(dt: datetime, **kwargs):
     )
 
 
+# todo use klogging2?
 from .klogging import LazyLogger, setup_logger