update_list.py

"""Update the search_engines.pickle list contained within the package.
Use this before deploying an update"""
from __future__ import absolute_import, division, print_function

import os

try:
    import ujson as json
except ImportError:
    import json
import ruamel.yaml as yaml
from six.moves.urllib.request import urlopen


_here = lambda *paths: os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                    *paths)


def main():
    filename = _here('serpextract', 'search_engines.json')
    print('Updating search engine parser definitions.')

    url = urlopen('https://raw.githubusercontent.com/matomo-org/searchengine-and-social-list/master/SearchEngines.yml')
    matomo_engines = yaml.safe_load(url)
    with open(filename, 'w') as json_file:
        json.dump(matomo_engines, json_file, indent=2, sort_keys=True)

    print('Saved {} search engine parser definitions to {}.'
          .format(len(matomo_engines), filename))


if __name__ == '__main__':
    main()