Merge pull request #1029 from jsvine/develop

Merge v0.10.3 into stable

CHANGELOG.md

@@ -2,6 +2,17 @@
 
 All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/).
 
+## [0.10.3] - 2023-10-26
+
+### Added
+
+- Add support for marked-content sequences, represented by `mcid` and `tag` attributes on `char`/`rect`/`line`/`curve`/`image` objects (h/t @dhdaines). ([#961](https://github.com/jsvine/pdfplumber/pulls/961))
+- Add `gs_path` argument to `pdfplumber.open(...)` and `pdfplumber.repair(...)`, to allow passing a custom Ghostscript path to be used for repairing. ([#953](https://github.com/jsvine/pdfplumber/issues/953))
+
+### Fixed
+
+- Respect `use_text_flow` in `extract_text` (h/t @dhdaines). ([#983](https://github.com/jsvine/pdfplumber/pulls/983))
+
 ## [0.10.2] - 2023-07-29
 
 ### Added

CITATION.cff

@@ -1,8 +1,8 @@
 cff-version: 1.2.0
 title: pdfplumber
 type: software
-version: 0.10.2
-date-released: "2023-07-29"
+version: 0.10.3
+date-released: "2023-10-26"
 authors:
  - family-names: "Singer-Vine"
  given-names: "Jeremy"

README.md

@@ -158,6 +158,8 @@ Each object is represented as a simple Python `dict`, with the following propert
 |`bottom`| Distance of bottom of the character from top of page.|
 |`doctop`| Distance of top of character from top of document.|
 |`matrix`| The "current transformation matrix" for this character. (See below for details.)|
+|`mcid`| The [marked content](https://ghostscript.com/~robin/pdf_reference17.pdf#page=850) section ID for this character if any (otherwise `None`). *Experimental attribute.*|
+|`tag`| The [marked content](https://ghostscript.com/~robin/pdf_reference17.pdf#page=850) section tag for this character if any (otherwise `None`). *Experimental attribute.*|
 |`ncs`|TKTK|
 |`stroking_pattern`|TKTK|
 |`non_stroking_pattern`|TKTK|
@@ -191,6 +193,8 @@ my_char_rotation = my_char_ctm.skew_x
 |`linewidth`| Thickness of line.|
 |`stroking_color`|The color of the line. See [docs/colors.md](docs/colors.md) for details.|
 |`non_stroking_color`|The non-stroking color specified for the line’s path. See [docs/colors.md](docs/colors.md) for details.|
+|`mcid`| The [marked content](https://ghostscript.com/~robin/pdf_reference17.pdf#page=850) section ID for this line if any (otherwise `None`). *Experimental attribute.*|
+|`tag`| The [marked content](https://ghostscript.com/~robin/pdf_reference17.pdf#page=850) section tag for this line if any (otherwise `None`). *Experimental attribute.*|
 |`object_type`| "line"|
 
 #### `rect` properties
@@ -210,6 +214,8 @@ my_char_rotation = my_char_ctm.skew_x
 |`linewidth`| Thickness of line.|
 |`stroking_color`|The color of the rectangle's outline. See [docs/colors.md](docs/colors.md) for details.|
 |`non_stroking_color`|The rectangle’s fill color. See [docs/colors.md](docs/colors.md) for details.|
+|`mcid`| The [marked content](https://ghostscript.com/~robin/pdf_reference17.pdf#page=850) section ID for this rect if any (otherwise `None`). *Experimental attribute.*|
+|`tag`| The [marked content](https://ghostscript.com/~robin/pdf_reference17.pdf#page=850) section tag for this rect if any (otherwise `None`). *Experimental attribute.*|
 |`object_type`| "rect"|
 
 #### `curve` properties
@@ -231,6 +237,8 @@ my_char_rotation = my_char_ctm.skew_x
 |`fill`| Whether the shape defined by the curve's path is filled.|
 |`stroking_color`|The color of the curve's outline. See [docs/colors.md](docs/colors.md) for details.|
 |`non_stroking_color`|The curve’s fill color. See [docs/colors.md](docs/colors.md) for details.|
+|`mcid`| The [marked content](https://ghostscript.com/~robin/pdf_reference17.pdf#page=850) section ID for this curve if any (otherwise `None`). *Experimental attribute.*|
+|`tag`| The [marked content](https://ghostscript.com/~robin/pdf_reference17.pdf#page=850) section tag for this curve if any (otherwise `None`). *Experimental attribute.*|
 |`object_type`| "curve"|
 
 #### Derived properties
@@ -531,6 +539,7 @@ Many thanks to the following users who've contributed ideas, features, and fixes
 - [Shannon Shen](https://github.com/lolipopshock)
 - [Matsumoto Toshi](https://github.com/toshi1127)
 - [John West](https://github.com/jwestwsj)
+- [David Huggins-Daines](https://github.com/dhdaines)
 - [Jeremy B. Merrill](https://github.com/jeremybmerrill)
 
 ## Contributing

docs/repairing.md

@@ -9,3 +9,7 @@ Malformed PDFs can often be [fixed via Ghostscript](https://superuser.com/questi
 - `pdfplumber.open(..., repair=True)` will repair your PDF on the fly (but not save the repaired version to disk).
 - `pdfplumber.repair(path_to_pdf)` will return a `BytesIO` object holding the bytes of a repaired version of the original file.
 - `pdfplumber.repair(path_to_pdf, outfile="path/to/repaired.pdf")` will write a repaired version of the original file to the indicated `outfile` path.
+
+## Custom parameters
+
+- `gs_path=...`: You can pass a custom path for the Ghostscript executable, helpful in case `pdfplumber` is unable to auto-detect your copy of Ghostscript.

pdfplumber/_version.py

@@ -1,2 +1,2 @@
-version_info = (0, 10, 2)
+version_info = (0, 10, 3)
 __version__ = ".".join(map(str, version_info))

pdfplumber/page.py

@@ -22,7 +22,7 @@
  LTPage,
  LTTextContainer,
 )
-from pdfminer.pdfinterp import PDFPageInterpreter
+from pdfminer.pdfinterp import PDFPageInterpreter, PDFStackT
 from pdfminer.pdfpage import PDFPage
 from pdfminer.psparser import PSLiteral
 
@@ -62,6 +62,8 @@
  "stream",
  "stroke",
  "stroking_color",
+ "mcid",
+ "tag",
  ]
 )
 
@@ -115,6 +117,56 @@ def normalize_color(
  return separate_pattern(tuplefied)
 
 
+class PDFPageAggregatorWithMarkedContent(PDFPageAggregator):
+ """Extract layout from a specific page, adding marked-content IDs to
+ objects where found."""
+
+ cur_mcid: Optional[int] = None
+ cur_tag: Optional[str] = None
+
+ def begin_tag(self, tag: PSLiteral, props: Optional[PDFStackT] = None) -> None:
+ """Handle beginning of tag, setting current MCID if any."""
+ self.cur_tag = decode_text(tag.name)
+ if isinstance(props, dict) and "MCID" in props:
+ self.cur_mcid = props["MCID"]
+ else:
+ self.cur_mcid = None
+
+ def end_tag(self) -> None:
+ """Handle beginning of tag, clearing current MCID."""
+ self.cur_tag = None
+ self.cur_mcid = None
+
+ def tag_cur_item(self) -> None:
+ """Add current MCID to what we hope to be the most recent object created
+ by pdfminer.six."""
+ # This is somewhat hacky and would not be necessary if
+ # pdfminer.six supported MCIDs. In reading the code it's
+ # clear that the `render_*` methods methods will only ever
+ # create one object, but that is far from being guaranteed.
+ # Even if pdfminer.six's API would just return the objects it
+ # creates, we wouldn't have to do this.
+ cur_obj = self.cur_item._objs[-1]
+ cur_obj.mcid = self.cur_mcid # type: ignore
+ cur_obj.tag = self.cur_tag # type: ignore
+
+ def render_char(self, *args, **kwargs) -> float: # type: ignore
+ """Hook for rendering characters, adding the `mcid` attribute."""
+ adv = super().render_char(*args, **kwargs)
+ self.tag_cur_item()
+ return adv
+
+ def render_image(self, *args, **kwargs) -> None: # type: ignore
+ """Hook for rendering images, adding the `mcid` attribute."""
+ super().render_image(*args, **kwargs)
+ self.tag_cur_item()
+
+ def paint_path(self, *args, **kwargs) -> None: # type: ignore
+ """Hook for rendering lines and curves, adding the `mcid` attribute."""
+ super().paint_path(*args, **kwargs)
+ self.tag_cur_item()
+
+
 class Page(Container):
  cached_properties: List[str] = Container.cached_properties + ["_layout"]
  is_original: bool = True
@@ -174,7 +226,7 @@ def height(self) -> T_num:
  def layout(self) -> LTPage:
  if hasattr(self, "_layout"):
  return self._layout
- device = PDFPageAggregator(
+ device = PDFPageAggregatorWithMarkedContent(
  self.pdf.rsrcmgr,
  pageno=self.page_number,
  laparams=self.pdf.laparams,

pdfplumber/pdf.py

@@ -70,12 +70,13 @@ def open(
  password: Optional[str] = None,
  strict_metadata: bool = False,
  repair: bool = False,
+ gs_path: Optional[Union[str, pathlib.Path]] = None,
  ) -> "PDF":
 
  stream: Union[BufferedReader, BytesIO]
 
  if repair:
- stream = _repair(path_or_fp, password=password)
+ stream = _repair(path_or_fp, password=password, gs_path=gs_path)
  stream_is_external = False
  # Although the original file has a path,
  # the repaired version does not

pdfplumber/repair.py

@@ -8,9 +8,10 @@
 def _repair(
  path_or_fp: Union[str, pathlib.Path, BufferedReader, BytesIO],
  password: Optional[str] = None,
+ gs_path: Optional[Union[str, pathlib.Path]] = None,
 ) -> BytesIO:
 
- executable = shutil.which("gs") or shutil.which("gswin32c")
+ executable = gs_path or shutil.which("gs") or shutil.which("gswin32c")
  if executable is None: # pragma: nocover
  raise Exception(
  "Cannot find Ghostscript, which is required for repairs.\n"
@@ -52,8 +53,9 @@ def repair(
  path_or_fp: Union[str, pathlib.Path, BufferedReader, BytesIO],
  outfile: Optional[Union[str, pathlib.Path]] = None,
  password: Optional[str] = None,
+ gs_path: Optional[Union[str, pathlib.Path]] = None,
 ) -> Optional[BytesIO]:
- repaired = _repair(path_or_fp, password)
+ repaired = _repair(path_or_fp, password, gs_path=gs_path)
  if outfile:
  with open(outfile, "wb") as f:
  f.write(repaired.read())

pdfplumber/utils/clustering.py

@@ -40,7 +40,10 @@ def make_cluster_dict(values: Iterable[T_num], tolerance: T_num) -> Dict[T_num,
 
 
 def cluster_objects(
- xs: List[R], key_fn: Union[Hashable, Callable[[R], T_num]], tolerance: T_num
+ xs: List[R],
+ key_fn: Union[Hashable, Callable[[R], T_num]],
+ tolerance: T_num,
+ preserve_order: bool = False,
 ) -> List[List[R]]:
 
  if not callable(key_fn):
@@ -51,7 +54,12 @@ def cluster_objects(
 
  get_0, get_1 = itemgetter(0), itemgetter(1)
 
- cluster_tuples = sorted(((x, cluster_dict.get(key_fn(x))) for x in xs), key=get_1)
+ if preserve_order:
+ cluster_tuples = [(x, cluster_dict.get(key_fn(x))) for x in xs]
+ else:
+ cluster_tuples = sorted(
+ ((x, cluster_dict.get(key_fn(x))) for x in xs), key=get_1
+ )
 
  grouped = itertools.groupby(cluster_tuples, key=get_1)
 

pdfplumber/utils/text.py

@@ -225,7 +225,10 @@ def to_textmap(
 
  for i, ws in enumerate(
  cluster_objects(
- words_sorted_doctop, lambda x: float(x[0]["doctop"]), y_tolerance
+ words_sorted_doctop,
+ lambda x: float(x[0]["doctop"]),
+ y_tolerance,
+ preserve_order=presorted or use_text_flow,
  )
  ):
  y_dist = (

tests/test_convert.py

@@ -70,7 +70,7 @@ def test_csv(self):
  assert c.split("\r\n")[9] == (
  "char,1,45.83,58.826,656.82,674.82,117.18,117.18,135.18,12.996,"
  '18.0,12.996,,,,,,TimesNewRomanPSMT,,,"(1, 0, 0, 1, 45.83, 660.69)"'
- ',DeviceRGB,"(0, 0, 0)",,,18.0,,,,,,Y,,1,'
+ ',,DeviceRGB,"(0, 0, 0)",,,18.0,,,,,,,Y,,1,'
  )
 
  io = StringIO()
@@ -125,7 +125,7 @@ def test_cli_csv(self):
  assert res.decode("utf-8").split("\r\n")[9] == (
  "char,1,45.83,58.826,656.82,674.82,117.18,117.18,135.18,12.996,"
  '18.0,12.996,,,,,,TimesNewRomanPSMT,,,"(1, 0, 0, 1, 45.83, 660.69)"'
- ',DeviceRGB,"(0, 0, 0)",,,18.0,,,,,,Y,,1,'
+ ',,DeviceRGB,"(0, 0, 0)",,,18.0,,,,,,,Y,,1,'
  )
 
  def test_cli_csv_exclude(self):
@@ -141,6 +141,7 @@ def test_cli_csv_exclude(self):
  "3",
  "--exclude-attrs",
  "matrix",
+ "mcid",
  "ncs",
  "non_stroking_pattern",
  "stroking_pattern",
@@ -150,7 +151,7 @@ def test_cli_csv_exclude(self):
  assert res.decode("utf-8").split("\r\n")[9] == (
  "char,1,45.83,58.826,656.82,674.82,117.18,117.18,135.18,12.996,"
  "18.0,12.996,,,,,,TimesNewRomanPSMT,"
- ',,"(0, 0, 0)",,18.0,,,,,Y,,1,'
+ ',,"(0, 0, 0)",,18.0,,,,,,Y,,1,'
  )
 
  def test_cli_csv_include(self):

tests/test_issues.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 import logging
 import os
+import re
 import unittest
 
 import pdfplumber
@@ -257,3 +258,20 @@ def test_issue_683(self):
  with pdfplumber.open(path) as pdf:
  page = pdf.pages[0]
  page.search(r"\d+", regex=True)
+
+ def test_issue_982(self):
+ """
+ extract_text(use_text_flow=True) apparently does nothing
+
+ This is because, while we took care not to sort the words by
+ `doctop` in `WordExtractor` and `WordMap`, no such precaution
+ was taken in `cluster_objects`. We thus add an option to
+ `cluster_objects` to preserve the ordering (which could come
+ from `use_text_flow` or from `presorted`) of the input objects.
+ """
+ path = os.path.join(HERE, "pdfs/issue-982-example.pdf")
+ with pdfplumber.open(path) as pdf:
+ page = pdf.pages[0]
+ text = re.sub(r"\s+", " ", page.extract_text(use_text_flow=True))
+ words = " ".join(w["text"] for w in page.extract_words(use_text_flow=True))
+ assert text[0:100] == words[0:100]

tests/test_mcids.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+import os
+import unittest
+
+import pdfplumber
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+
+
+class TestMCIDs(unittest.TestCase):
+ """Test MCID extraction."""
+
+ def test_mcids(self):
+ path = os.path.join(HERE, "pdfs/mcid_example.pdf")
+
+ pdf = pdfplumber.open(path)
+ page = pdf.pages[0]
+ # Check text of MCIDS
+ mcids = []
+ for c in page.chars:
+ if "mcid" in c:
+ while len(mcids) <= c["mcid"]:
+ mcids.append("")
+ if not mcids[c["mcid"]]:
+ mcids[c["mcid"]] = c["tag"] + ": "
+ mcids[c["mcid"]] += c["text"]
+ assert mcids == [
+ "Standard: Test of figures",
+ "",
+ "P: 1 ligne",
+ "P: 2 ligne",
+ "P: 3 ligne",
+ "P: 4 ligne",
+ "P: 0",
+ "P: 2",
+ "P: 4",
+ "P: 6",
+ "P: 8",
+ "P: 10",
+ "P: 12",
+ "P: Figure 1: Chart",
+ "",
+ "P: 1 colonne",
+ "P: 2 colonne",
+ "P: 3 colonne",
+ ]
+ # Check line and curve MCIDs
+ line_mcids = set(x["mcid"] for x in page.lines)
+ curve_mcids = set(x["mcid"] for x in page.curves)
+ assert all(x["tag"] == "Figure" for x in page.lines)
+ assert all(x["tag"] == "Figure" for x in page.curves)
+ assert line_mcids & {1, 14}
+ assert curve_mcids & {1, 14}
+ # No rects to test unfortunately!

tests/test_repair.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 import os
+import shutil
 import tempfile
 import unittest
 
@@ -56,3 +57,8 @@ def test_repair_password(self):
  path = os.path.join(HERE, "pdfs/password-example.pdf")
  with pdfplumber.open(path, repair=True, password="test") as pdf:
  assert len(pdf.pages[0].chars)
+
+ def test_repair_custom_path(self):
+ path = os.path.join(HERE, "pdfs/malformed-from-issue-932.pdf")
+ with pdfplumber.open(path, repair=True, gs_path=shutil.which("gs")) as pdf:
+ assert len(pdf.pages[0].chars)