Merge pull request #14 from jscottbranson/develop

Improved Console Output

README.md

@@ -51,6 +51,9 @@ As written, this code will produce errors with asyncio in Python 3.9. The code i
 6. (optional) Save Twilio notification credentials as env variables.
 7. `python3 main.py`
 
+## Generating Configuration Files
+The `utils_for_humans` directory contains a `dict_convert.py` script that can be used to assist in generating a configuration file with monitoring output that is consistent across all servers.
+
 ## Updating
 1. `cd rippled_monitor`
 2. ` git pull`

process_responses/console_output.py

@@ -4,9 +4,10 @@
 import logging
 import time
 import asyncio
+import textwrap
 from copy import deepcopy
 
-from prettytable import PrettyTable
+from prettytable import PrettyTable, ALL
 
 from .common import decode_version
 from .common import copy_stock
@@ -37,11 +38,13 @@ async def format_table_validation(table):
             validator['forked'] = green + str(validator['forked']) + color_reset
         else:
             validator['forked'] = red + str(validator['forked']) + color_reset
+            validator['server_name'] = red + validator['server_name'] + color_reset
         # Green if validations are full
         if validator['full']:
             validator['full'] = green + str(validator['full']) + color_reset
         else:
             validator['full'] = red + str(validator['full']) + color_reset
+            validator['server_name'] = red + validator['server_name'] + color_reset
         # Calculate server version
         if isinstance(validator['server_version'], str):
             if validator['server_version'][0:].isdigit():
@@ -79,6 +82,7 @@ async def print_table_validation(table):
             validator['time_updated'],
         ])
 
+    pretty_table.sortby = "Validator Name"
     print(pretty_table)
     logging.info("Successfully printed updated validations table.")
 
@@ -124,11 +128,13 @@ async def format_table_server(table):
                 server['server_status'] = green + server['server_status'] + color_reset
             else:
                 server['server_status'] = red + server['server_status'] + color_reset
+                server['server_name'] = red + str(server['server_name']) + color_reset
         # Forked Servers in Red
         if server['forked'] is False:
             server['forked'] = green + str(server['forked']) + color_reset
         else:
             server['forked'] = red + str(server['forked']) + color_reset
+            server['server_name'] = red + str(server['server_name']) + color_reset
         # Base load factor in green
         if isinstance(server['load_factor'], int) and isinstance(server['load_base'], int):
             if server['load_factor'] == server['load_base']:
@@ -170,6 +176,7 @@ async def print_table_server(table):
             server['forked'],
             server['time_updated'],
             ])
+    pretty_table.sortby = "Server Name"
     print(pretty_table)
     logging.info("Successfully printed updated server table.")
 
@@ -189,6 +196,23 @@ async def sort_amendments(table_validator, amendments):
 
     return amendments
 
+async def format_amendment(amendment, table_validator):
+    '''
+    Color and other fun things!
+    '''
+    color_reset = "\033[0;0m"
+    green = "\033[0;32m"
+
+    amendment['supporters'].sort(key=str.lower)
+
+    support_percent = round(len(amendment['supporters']) / len(table_validator) * 100, 1)
+    if support_percent > 80:
+        amendment['name'] = green + str(amendment['name']) + color_reset
+        amendment['support_percent'] = green + str(support_percent) + color_reset
+    else:
+        amendment['support_percent'] = str(support_percent)
+    return amendment
+
 async def print_table_amendments(table_validator, amendments):
     '''
     Print information on amendment voting.
@@ -208,15 +232,19 @@ async def print_table_amendments(table_validator, amendments):
 
     amendment_votes = await sort_amendments(table_validator, deepcopy(amendments))
     for amendment in amendment_votes:
+        amendment = await format_amendment(amendment, table_validator)
         supporters = ''
         for supporter in amendment['supporters']:
             supporters = supporters + supporter + ', '
         pretty_table.add_row([
             amendment['name'],
             len(amendment['supporters']),
             len(table_validator) - len(amendment['supporters']),
-            str(round(len(amendment['supporters']) / len(table_validator) * 100, 1)) + '%',
-            supporters,
+            amendment['support_percent'],
+            textwrap.fill(supporters, width=180),
         ])
+    pretty_table.hrules=ALL
+    pretty_table.sortby = "Yea Votes"
+    pretty_table.reversesort = True
     print(pretty_table)
     logging.info("Successfully printed the amendments table.")

process_responses/process_output.py

@@ -44,7 +44,8 @@ async def process_console_output(self):
             await console_output.print_table_server(self.table_stock)
             if self.table_validator:
                 await console_output.print_table_validation(self.table_validator)
-                await console_output.print_table_amendments(self.table_validator, self.settings.AMENDMENTS)
+                if self.settings.PRINT_AMENDMENTS:
+                    await console_output.print_table_amendments(self.table_validator, self.settings.AMENDMENTS)
             self.time_last_output = time.time()
 
     async def evaluate_forks(self):

process_responses/process_validation_output.py

@@ -48,6 +48,20 @@ async def clean_validations(settings, val_keys, table, processed_validations):
             val_keys, table = await del_dup_validators(table)
     return val_keys, table, processed_validations
 
+async def reset_potentially_omitted_values(validator):
+    '''
+    Don't assume that potentially omitted values persist.
+    For example, an amendment may be supported by a validator
+    then the validator operator drops support, resulting in a flag ledger
+    with the 'amendment' key/value missing. Without this function, it would appear
+    that the operator still supports the amendment, despite dropping support.
+
+    :param dict validator: An individual validator's dictionary
+    '''
+    message_keys = ['amendments', 'base_fee', 'load_fee', 'reserve_base', 'reserve_inc', 'server_version']
+    for i in message_keys:
+        validator[i] = None
+
 async def update_table_validator(table, message):
     '''
     Update the table based on a received validation message.
@@ -57,11 +71,14 @@ async def update_table_validator(table, message):
     '''
     message = message['data']
 
-    # Consider notifying if the ephemeral key changes for a server
+    # Consider notifying if the ephemeral/master key or cookie changes for a server
 
     for validator in table:
         if message.get('master_key') and message.get('master_key') == validator['master_key'] \
            or message.get('validation_public_key') == validator['validation_public_key']:
+            # Check if this is a flag ledger.
+            if (int(message['ledger_index']) + 1) % 256 == 0:
+                await reset_potentially_omitted_values(validator)
             for key in validator.keys():
                 if key in message.keys():
                     validator[key] = message[key]

requirements.txt

@@ -1,15 +1,94 @@
-aiohttp==3.8.5
-aiosignal==1.3.1
-async-timeout==4.0.3
-attrs==23.1.0
-certifi==2023.7.22
+apparmor==3.0.4
+apt-xapian-index==0.49
+beautifulsoup4==4.10.0
+blinker==1.4
+certifi==2020.6.20
+cffi==1.15.1
+chardet==4.0.0
 charset-normalizer==3.2.0
-frozenlist==1.4.0
-idna==3.4
-multidict==6.0.4
-prettytable==3.9.0
-requests==2.31.0
-urllib3==2.0.5
-wcwidth==0.2.6
-websockets==11.0.3
-yarl==1.9.2
+click==8.0.3
+colorama==0.4.4
+command-not-found==0.3
+cryptography==41.0.3
+cupshelpers==1.0
+dbus-python==1.2.18
+deprecation==2.1.0
+distro==1.7.0
+distro-info==1.1+ubuntu0.2
+fuse-python==1.0.2
+gpg==1.16.0
+html5lib==1.1
+httplib2==0.20.2
+idna==3.3
+imageio==2.31.1
+img2pdf==0.4.4
+importlib-metadata==4.6.4
+imutils==0.5.4
+iotop==0.6
+jeepney==0.7.1
+keyring==23.5.0
+language-selector==0.1
+launchpadlib==1.10.16
+lazr.restfulclient==0.14.4
+lazr.uri==1.0.6
+lazy_loader==0.3
+LibAppArmor==3.0.4
+libvirt-python==8.0.0
+lxml==4.8.0
+markdown-it-py==3.0.0
+mdurl==0.1.2
+more-itertools==8.10.0
+netifaces==0.11.0
+networkx==3.1
+notify2==0.3
+numpy==1.25.2
+oauthlib==3.2.0
+ocrmypdf==14.4.0
+olefile==0.46
+opencv-python==4.8.0.76
+packaging==23.1
+pdfminer.six==20221105
+pikepdf==8.4.0
+Pillow==9.0.1
+pluggy==1.2.0
+pycairo==1.20.1
+pycparser==2.21
+pycups==2.0.1
+Pygments==2.16.1
+PyGObject==3.42.1
+PyJWT==2.3.0
+pylibacl==0.6.0
+pyparsing==2.4.7
+PyPDF2==3.0.1
+PyQt5==5.15.6
+PyQt5-sip==12.9.1
+python-apt==2.4.0+ubuntu3
+python-debian==0.1.43+ubuntu1.1
+PyWavelets==1.4.1
+pyxattr==0.7.2
+PyYAML==5.4.1
+reportlab==3.6.8
+requests==2.25.1
+rich==13.5.2
+scikit-image==0.21.0
+scipy==1.11.2
+scour==0.38.2
+SecretStorage==3.3.1
+six==1.16.0
+soupsieve==2.3.1
+speedtest-cli==2.1.3
+systemd-python==234
+tesserocr==2.6.1
+tifffile==2023.8.12
+tornado==6.1
+tqdm==4.66.1
+ubuntu-advantage-tools==8001
+ubuntu-drivers-common==0.0.0
+ufw==0.36.1
+unattended-upgrades==0.1
+urllib3==1.26.5
+usb-creator==0.3.7
+wadllib==1.3.6
+webencodings==0.5.1
+xkit==0.0.0
+zipp==1.0.0

settings_ex.py

@@ -28,32 +28,46 @@
 #### Console Output ####
 CONSOLE_OUT = True # Print a fancy table to the console
 CONSOLE_REFRESH_TIME = 5 # Time in seconds to wait before refreshing console output.
+PRINT_AMENDMENTS = True # Print output summarizing amendment voting.
 
 #### Random ####
 REMOVE_DUP_VALIDATORS = True # Allow the same validator master/eph keys to be tracked more than once
 
 LOG_VALIDATIONS_FROM = [] # Log validations that include a master_key defined in this list.
 
+#### Amendments ####
 AMENDMENTS = [
     {
         'id': '8CC0774A3BF66D1D22E76BBDA8E8A232E6B6313834301B3B23E8601196AE6455',
         'name': 'AMM',
     },
     {
-        'id': '56B241D7A43D40354D02A9DC4C8DF5C7A1F930D92A9035C4E12291B3CA3E1C2B',
-        'name': 'Clawback',
+        'id': 'C393B3AEEBF575E475F0C60D5E4241B2070CC4D0EB6C4846B1A07508FAEFC485',
+        'name': 'fixInnerObjTemplate',
     },
     {
-        'id': '27CD95EE8E1E5A537FF2F89B6CEB7C622E78E9374EBD7DCBEDFAE21CD6F16E0A',
-        'name': 'fixReducedOffersV1',
+        'id': '15D61F0C6DB6A2F86BCF96F1E2444FEC54E705923339EC175BD3E517C8B3FF91',
+        'name': 'fixDisallowIncomingV1',
+    },
+    {
+        'id': '3318EA0CF0755AF15DAC19F2B5C5BCBFF4B78BDD57609ACCAABE2C41309B051A',
+        'name': 'fixFillOrKill',
+    },
+    {
+        'id': 'C98D98EE9616ACD36E81FDEB8D41D349BF5F1B41DD64A0ABC1FE9AA5EA267E9C',
+        'name': 'XChainBridge',
+    },
+    {
+        'id': 'DB432C3A09D9D5DFC7859F39AE5FF767ABC59AED0A9FB441E83B814D8946C109',
+        'name': 'DID',
     },
     {
         'id': '93E516234E35E08CA689FA33A6D38E103881F8DCB53023F728C307AA89D515A7',
         'name': 'XRPFees',
     },
     {
-        'id': 'AE35ABDEFBDE520372B31C957020B34A7A4A9DC3115A69803A44016477C84D6E',
-        'name': 'fixNfTokenRemint',
+        'id': '03BDC0099C4E14163ADA272C1B6F6FABB448CC3E51F522F978041E4B57D9158C',
+        'name': 'fixNFTokenReserve',
     },
 ]
 

utils_for_humans/dict_convert.py

@@ -14,7 +14,7 @@
 OUTPUT_FILE = 'settings_new.py'
 
 NOTIFY_TWILIO = False
-NOTIFY_DISCORD = True
+NOTIFY_DISCORD = False
 NOTIFY_MATTERMOST = False
 NOTIFY_SLACK = False
 NOTIFY_SMTP = False
@@ -155,5 +155,5 @@ def build_notification_dict():
 if __name__ == '__main__':
     build_notification_dict()
     servers_new = build_servers(settings.SERVERS)
-    validators_new = build_validators(settings.VALIDATOR_KEYS)
+    validators_new = build_validators(settings.VALIDATORS)
     output_text(servers_new, validators_new)

utils_for_humans/to_monitor.py

@@ -11,7 +11,7 @@
     {"master_key": "nHUcNC5ni7XjVYfCMe38Rm3KQaq27jw7wJpcUYdo4miWwpNePRTw", "server_name": "Cabbit.Tech",},
     {"master_key": "nHUFE9prPXPrHcG3SkwP1UzAQbSphqyQkQK9ATXLZsfkezhhda3p", "server_name": "Alloy",},
     {"master_key": "nHB8QMKGt9VB4Vg71VszjBVQnDW3v3QudM4DwFaJfy96bj4Pv9fA", "server_name": "Bithomp",},
-    {"master_key": "nHUd8g4DWm6HgjGTjKKSfYiRyf8qCvEN1PXR7YDJ5QTFyAnZHkbW", "server_name": "Brex",},
+    {"master_key": "nHBVACxZaNbUjZZkBfj7gRxF3xgG2vbcP4m48KzVwntdTogi5Tfs", "server_name": "OnXRP.com",},
     {"master_key": "nHUwGQrfZfieeLFeGRdGnAmGpHBCZq9wvm5c59wTc2JhJMjoXmd8", "server_name": "XRPGoat",},
     {"master_key": "nHBWa56Vr7csoFcCnEPzCCKVvnDQw3L28mATgHYQMGtbEfUjuYyB", "server_name": "Vet",},
     {"master_key": "nHUge3GFusbqmfYAJjxfKgm2j4JXGxrRsfYMcEViHrFSzQDdk5Hq", "server_name": "Katczynski",},
@@ -25,7 +25,6 @@
     {"master_key": "nHUED59jjpQ5QbNhesXMhqii9gA8UfbBmv3i5StgyxG98qjsT4yn", "server_name": "Arrington",},
     {"master_key": "nHDB2PAPYqF86j9j3c6w1F1ZqwvQfiWcFShZ9Pokg9q4ohNDSkAz", "server_name": "XRPScan",},
     {"master_key": "nHUvcCcmoH1FJMMC6NtF9KKA4LpCWhjsxk2reCQidsp5AHQ7QY9H", "server_name": "Jon Nilsen",},
-    {"master_key": "nHUY14bKLLm72ukzo2t6AVnQiu4bCd1jkimwWyJk3txvLeGhvro5", "server_name": "Gatehub",},
     {"master_key": "nHUfPizyJyhAJZzeq3duRVrZmsTZfcLn7yLF5s2adzHdcHMb9HmQ", "server_name": "UNIC.ac.cy",},
     {"master_key": "nHUXeusfwk61c4xJPneb9Lgy7Ga6DVaVLEyB29ftUdt9k2KxD6Hw", "server_name": "XRPL-Labs",},
     {"master_key": "nHUq9tJvSyoXQKhRytuWeydpPjvTz3M9GfUpEqfsg9xsewM7KkkK", "server_name": "UCL.ac.uk",},
@@ -38,10 +37,10 @@
     {"master_key": "nHUVPzAmAmQ2QSc4oE1iLfsGi17qN2ado8PhxvgEkou76FLxAz7C", "server_name": "KU.edu",},
     {"master_key": "nHULqGBkJtWeNFjhTzYeAsHA3qKKS7HoBh8CV3BAGTGMZuepEhWC", "server_name": "Berkeley.edu",},
     {"master_key": "nHBdXSF6YHAHSZUk7rvox6jwbvvyqBnsWGcewBtq8x1XuH6KXKXr", "server_name": "FTSO.eu",},
-    {"master_key": "nHU95JxeaHJoSdpE7R49Mxp4611Yk5yL9SGEc12UDJLr4oEUN4NT", "server_name": "Flagship Solutions",},
+    {"master_key": "nHUDpRzvY8fSRfQkmJMqjmVSaFmMEVxBNn2tNQy5VAhFJ6is6GFk", "server_name": "Ekiserrepe.es",},
     {"master_key": "nHUpcmNsxAw47yt2ADDoNoQrzLyTJPgnyq16u6Qx2kRPA17oUNHz", "server_name": "ISRDC.in",},
     {"master_key": "nHUpJSKQTZdB1TDkbCREMuf8vEqFkk84BcvZDhsQsDufFDQVajam", "server_name": "Data443",},
-    {"master_key": "nHBgiH2aih5JoaL3wbiiqSQfhrC21vJjxXoCoD2fuqcNbriXsfLm", "server_name": "AtTokyo",},
-    {"validation_public_key": "n9M2UqXLK25h9YEQTskmCXbWPGhQmB1pFVqeXia38UwLaL838VbG", "server_name": "Gatehub.net (eph)"},
-    {"validation_public_key": "nHU3AenyRuJ4Yei4YHkh6frZg8y2RwXznkMAomUE1ptV5Spvqsih", "server_name": "aesthetes.art",},
+    {"master_key": "nHUfxETNHsA9reyYCVYwNztEbifMg6U9YUdcgVvzMwGNpphKSSf6", "server_name": "XRPKuwait.com",},
+    {"master_key": "nHU3AenyRuJ4Yei4YHkh6frZg8y2RwXznkMAomUE1ptV5Spvqsih", "server_name": "aesthetes.art",},
+    {"validation_public_key": "n9M2UqXLK25h9YEQTskmCXbWPGhQmB1pFVqeXia38UwLaL838VbG", "server_name": "Gatehub.net"},
 ]