Allow reporters to be specified multiple times

This allows different jobs to be selected for reporting in different
ways, for example, allowing different changes to be emailed to different
addresses.

Closes #790

Signed-off-by: James Hewitt <james.hewitt@uk.ibm.com>

CHANGELOG.md

@@ -12,6 +12,7 @@ The format mostly follows [Keep a Changelog](http://keepachangelog.com/en/1.0.0/
 - New option `ignore_incomplete_reads` (Requested in #725 by wschoot, contributed in #787 by wfrisch)
 - New option `wait_for` in browser jobs (Requested in #763 by yuis-ice, contributed in #810 by jamstah)
 - Added tags to jobs and the ability to select them at the command line (#789 by jamstah)
+- Allow reporters to be specified multiple times (#822 by jamstah)
 
 ### Changed
 

docs/source/reporters.rst

@@ -51,6 +51,35 @@ If the notification does not work, check your configuration and/or add
 the ``--verbose`` command-line option to show detailed debug logs.
 
 
+Common options
+--------------
+
+You can use a list of configurations under a reporter type to report
+different jobs with different configurations.
+
+You can enable or disable a reporter by using the ``enabled`` option.
+
+You can select jobs to report by each reporter using tags.
+
+For example:
+
+.. code:: yaml
+
+   telegram:
+   - bot_token: '999999999:3tOhy2CuZE0pTaCtszRfKpnagOG8IQbP5gf' # your bot api token
+     chat_id:
+       - '11111111'
+       - '22222222'
+     enabled: true
+     tags: [chat1]
+   - bot_token: '999999999:90jf403vnc09m0vi4s09t409jc09fj09sdc' # your bot api token
+     chat_id:
+       - '33333333'
+       - '44444444'
+     tags: [chat2]
+     enabled: true
+
+
 Built-in reporters
 ------------------
 

lib/urlwatch/command.py

@@ -360,7 +360,10 @@ def set_error(job_state, message):
                                    'Same Old, Same Old\n'))
         report.error(set_error(build_job('Error Reporting', 'http://example.com/error', '', ''), 'Oh Noes!'))
 
-        report.finish_one(name)
+        reported = report.finish_one(name)
+
+        if not reported:
+            raise ValueError(f'Reporter not enabled: {name}')
 
         sys.exit(0)
 

lib/urlwatch/handler.py

@@ -57,6 +57,7 @@ def __init__(self, cache_storage, job):
         self.timestamp = None
         self.current_timestamp = None
         self.exception = None
+        self.reported_count = 0
         self.traceback = None
         self.tries = 0
         self.etag = None
@@ -214,10 +215,10 @@ def finish(self):
         end = datetime.datetime.now()
         duration = (end - self.start)
 
-        ReporterBase.submit_all(self, self.job_states, duration)
+        return ReporterBase.submit_all(self, self.job_states, duration)
 
     def finish_one(self, name):
         end = datetime.datetime.now()
         duration = (end - self.start)
 
-        ReporterBase.submit_one(name, self, self.job_states, duration)
+        return ReporterBase.submit_one(name, self, self.job_states, duration)

lib/urlwatch/jobs.py

@@ -199,8 +199,8 @@ class Job(JobBase):
     __required__ = ()
     __optional__ = ('name', 'filter', 'max_tries', 'diff_tool', 'compared_versions', 'diff_filter', 'enabled', 'treat_new_as_changed', 'user_visible_url', 'tags')
 
-    def matching_tags(self, tags: Set[str]) -> Set[str]:
-        return self.tags & tags
+    def matching_tags(self, tags: Iterable[str]) -> Set[str]:
+        return self.tags.intersection(tags)
 
     # determine if hyperlink "a" tag is used in HtmlReporter
     def location_is_url(self):

lib/urlwatch/main.py

@@ -109,5 +109,13 @@ def run_jobs(self):
         run_jobs(self)
 
     def close(self):
-        self.report.finish()
+        reported = self.report.finish()
+
+        if not reported:
+            logger.warning('No reporters enabled.')
+
+        for job_state in self.report.job_states:
+            if not job_state.reported_count:
+                logger.warning(f'Job {job_state.job.pretty_name()} was not reported on')
+
         self.cache_storage.close()

lib/urlwatch/reporters.py

@@ -28,6 +28,7 @@
 
 
 import asyncio
+from collections.abc import Mapping
 import difflib
 import re
 import email.utils
@@ -80,13 +81,18 @@
 WDIFF_REMOVED_RE = r'[\[][-].*?[-][]]'
 
 
+def filter_by_tags(job_states, tags):
+    return job_states if not tags else [job_state for job_state in job_states if job_state.job.matching_tags(tags)]
+
+
 class ReporterBase(object, metaclass=TrackSubClasses):
     __subclasses__ = {}
 
-    def __init__(self, report, config, job_states, duration):
+    def __init__(self, report, config, job_states, job_count_total, duration):
         self.report = report
         self.config = config
         self.job_states = job_states
+        self.job_count_total = job_count_total
         self.duration = duration
 
     def get_signature(self):
@@ -96,8 +102,10 @@ def get_signature(self):
                                                       copyright=urlwatch.__copyright__),
             'Website: {url}'.format(url=urlwatch.__url__),
             'Support urlwatch development: https://github.com/sponsors/thp',
-            'watched {count} URLs in {duration} seconds'.format(count=len(self.job_states),
-                                                                duration=self.duration.seconds),
+            'reported {count} of {total} watched URLs in {duration} seconds'.format(
+                count=len(self.job_states),
+                total=self.job_count_total,
+                duration=self.duration.seconds),
         )
 
     def convert(self, othercls):
@@ -123,35 +131,36 @@ def reporter_documentation(cls):
 
     @classmethod
     def submit_one(cls, name, report, job_states, duration):
+        any_enabled = False
         subclass = cls.__subclasses__[name]
-        cfg = report.config['report'].get(name, {'enabled': False})
-        if cfg['enabled']:
-            base_config = subclass.get_base_config(report)
-            if base_config.get('separate', False):
-                for job_state in job_states:
-                    subclass(report, cfg, [job_state], duration).submit()
-            else:
-                subclass(report, cfg, job_states, duration).submit()
-        else:
-            raise ValueError('Reporter not enabled: {name}'.format(name=name))
+        cfgs = report.config['report'].get(name, {'enabled': False})
+        if isinstance(cfgs, Mapping):
+            cfgs = [cfgs]
 
-    @classmethod
-    def submit_all(cls, report, job_states, duration):
-        any_enabled = False
-        for name, subclass in cls.__subclasses__.items():
-            cfg = report.config['report'].get(name, {})
+        for cfg in cfgs:
             if cfg.get('enabled', False):
                 any_enabled = True
                 logger.info('Submitting with %s (%r)', name, subclass)
                 base_config = subclass.get_base_config(report)
+                matching_job_states = filter_by_tags(job_states, cfg.get("tags", []))
                 if base_config.get('separate', False):
-                    for job_state in job_states:
-                        subclass(report, cfg, [job_state], duration).submit()
+                    for job_state in matching_job_states:
+                        subclass(report, cfg, [job_state], len(job_states), duration).submit()
+                        job_state.reported_count = job_state.reported_count + 1
                 else:
-                    subclass(report, cfg, job_states, duration).submit()
+                    subclass(report, cfg, matching_job_states, len(job_states), duration).submit()
+                    for job_state in matching_job_states:
+                        job_state.reported_count = job_state.reported_count + 1
+
+        return any_enabled
+
+    @classmethod
+    def submit_all(cls, report, job_states, duration):
+        any_enabled = False
+        for name in cls.__subclasses__.keys():
+            any_enabled = any_enabled | ReporterBase.submit_one(name, report, job_states, duration)
 
-        if not any_enabled:
-            logger.warning('No reporters enabled.')
+        return any_enabled
 
     def submit(self):
         raise NotImplementedError()