Merge pull request #3 from akb89/develop

Adding note in README on how to avoid request rejection from Wikimedi…

README.md

@@ -46,6 +46,8 @@ The `--date` parameter expects a string corresponding to one of the dates
 found under the Wikimedia dump site corresponding to a given Wikipedia dump
 (e.g. https://dumps.wikimedia.org/enwiki/ for the English Wikipedia).
 
+**Important** Keep num-threads <= 3 to avoid rejection from Wikimedia servers
+
 ### Extract
 To extract the content of the downloaded .bz2 archives, do:
 

setup.py

@@ -16,7 +16,7 @@
     author_email='akb@3azouz.net',
     long_description=long_description,
     long_description_content_type='text/markdown',
-    version='0.1.2',
+    version='0.1.4',
     url='https://github.com/akb89/witokit',
     download_url='https://pypi.org/project/witokit/#files',
     license='MIT',

witokit/main.py

@@ -4,7 +4,6 @@
 """
 
 import os
-import sys
 import argparse
 import multiprocessing
 import urllib
@@ -38,9 +37,9 @@ def _download_href(output_dirpath, wiki_dump_url, href):
                                                           href)
     try:
         urllib.request.urlretrieve(url, output_filepath)
-    except urllib.error.HTTPError:
+    except urllib.error.HTTPError as error:
         logger.error('Could not download archive from {}'.format(url))
-        sys.exit(1)
+        raise error
 
 
 def _parallel_download(wiki_arxiv_hrefs, wiki_dump_url, num_threads,
@@ -69,11 +68,11 @@ def _collect_wiki_arxiv_hrefs(wiki_dump_url, lang, date):
             href = link.get('href')
             if re.match(pattern, href):
                 wiki_arxiv_hrefs.append(href)
-    except urllib.error.HTTPError:
+    except urllib.error.HTTPError as error:
         logger.error('HTTPError using lang = \'{}\' and date = \'{}\'. '
                      'Could not retrieve any Wikipedia data at URL = {}'
                      .format(lang, date, wiki_dump_url))
-        sys.exit(1)
+        raise error
     return wiki_arxiv_hrefs