Merge pull request #19 from NREL/sp/rhub_api

Sp/rhub api

elm/pdf.py

@@ -350,3 +350,28 @@ def clean_headers(self, char_thresh=0.6, page_thresh=0.8, split_on='\n',
  iheaders=iheaders)
  self.full = combine_pages(self.pages)
  return self.full
+
+ def convert_to_txt(self, txt_fp):
+ """Function to convert contents of pdf document to txt file.
+
+ Parameters
+ ----------
+ txt_fp: str
+ Directory for output txt file.
+
+ Returns
+ -------
+ text : str
+ Text string containing contents from pdf
+ """
+ text = self.clean_poppler(layout=True)
+ if self.is_double_col():
+ text = self.clean_poppler(layout=False)
+ text = self.clean_headers(char_thresh=0.6, page_thresh=0.8,
+ split_on='\n',
+ iheaders=[0, 1, 3, -3, -2, -1])
+ with open(txt_fp, 'w') as f:
+ f.write(text)
+ logger.info(f'Saved: {txt_fp}')
+
+ return text

elm/web/osti.py

@@ -178,9 +178,7 @@ def __init__(self, url, n_pages=1):
  self._n_pages = 0
  self._iter = 0
 
- records = self._get_first()
- for page in self._get_pages(n_pages=n_pages):
- records += page
+ records = self._get_all(n_pages)
  records = [OstiRecord(single) for single in records]
  super().__init__(records)
 
@@ -234,6 +232,27 @@ def _get_pages(self, n_pages):
  else:
  break
 
+ def _get_all(self, n_pages):
+ """Get all pages of records up to n_pages.
+
+ Parameters
+ ----------
+ n_pages : int
+ Number of pages to retrieve
+
+ Returns
+ -------
+ all_records : list
+ List of all records.
+ """
+ first_page = self._get_first()
+ records = first_page
+
+ for page in self._get_pages(n_pages):
+ records.extend(page)
+
+ return records
+
  def download(self, out_dir):
  """Download all PDFs from the records in this OSTI object into a
  directory. PDFs will be given file names based on their OSTI record ID