Drop support for omitting pdfminer.six

[ci skip]

.gitattributes

@@ -9,5 +9,6 @@
 *.png	binary
 *.jpg	binary
 *.bin   binary
+*.afdesign  binary
 
 .git_archival.txt  export-subst

.gitignore

@@ -3,9 +3,10 @@
 .pylintrc
 .pytest_cache/
 .ruffus_history.sqlite
-.venv/
+.venv*/
 *.pyc
 *.sublime-*
+*.DS_Store
 
 # Package building
 .eggs/
@@ -42,3 +43,4 @@ tests/resources/private/
 tmp/
 /debug_tests.py
 *.traineddata
+/private

docs/installation.rst

@@ -364,6 +364,16 @@ The command line program should now be available:
 
     ocrmypdf --help
 
+Installing on FreeBSD
+---------------------
+
+FreeBSD 11.2 is known to work. Other versions likely work but have not been tested.
+
+In general it should work to:
+
+#. `Install and build pikepdf <https://pikepdf.readthedocs.io/en/latest/installation.html#installing-on-freebsd-11-2>`_.
+#. Install the equivalent list of dependencies for Linux.
+
 Installing the Docker image
 ---------------------------
 
@@ -374,9 +384,19 @@ See `OCRmyPDF Docker Image <docker>`_ for more information.
 Installing on Windows
 ---------------------
 
-Direct installation on Windows is not possible.  `Install the Docker <docker-install>`_ container as described above.  Ensure that your command prompt can run the docker "hello world" container.
+Direct installation on Windows is not possible, because there are a
+POSIX dependencies. Your options are:
 
-It would probably not be too difficult to port on Windows.  The main reason this has been avoided is the difficulty of packaging and installing the various non-Python dependencies: Tesseract, QPDF, Ghostscript, Leptonica.  Pull requests to add or improve Windows support would be quite welcome.
+* Install Ubuntu 18.04 in Windows 10 Subsystem for Linux, then follow
+  the Ubuntu 18.04 procedure.
+* `Install the Docker <docker-install>`__ container. Ensure that
+  your command prompt can run the docker "hello world" container.
+
+It would probably not be too difficult to port on Windows. The main
+reason this has been avoided is the difficulty of packaging and
+installing the various non-Python dependencies: Tesseract, QPDF,
+Ghostscript, Leptonica. Pull requests to add or improve Windows support
+would be quite welcome.
 
 Installing with Python pip
 --------------------------

docs/release_notes.rst

@@ -13,6 +13,19 @@ Note that it is licensed under GPLv3, so scripts that ``import ocrmypdf`` and ar
    find:    [^`]\#([0-9]{1,3})[^0-9]
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
+v8.3.2
+------
+
+-   Dropped workaround for macOS that allowed it work without pdfminer.six,
+    now a proper sdist release of pdfminer.six is available.
+
+-   pikepdf 1.5.0 is now required.
+
+v8.3.1
+------
+
+-   Fixed an issue where PDFs with malformed metadata would be rendered as blank pages. `#398 <https://github.com/jbarlow83/OCRmyPDF/issues/398>`_.
+
 v8.3.0
 ------
 

requirements/main.txt

@@ -5,7 +5,7 @@ chardet == 3.0.4
 cffi == 1.12.2
 img2pdf == 0.3.3
 pdfminer.six == 20181108
-pikepdf == 1.3.0
+pikepdf == 1.5.0.post0
 Pillow >= 5.0.0, != 5.1.0 ; sys_platform == "darwin"
 pycparser == 2.19
 python-xmp-toolkit == 2.0.1

setup.py

@@ -98,15 +98,14 @@ setup(
         'chardet >= 3.0.4, < 4',  # unlisted requirement of pdfminer.six 20181108
         'cffi >= 1.9.1',  # must be a setup and install requirement
         'img2pdf >= 0.3.0, < 0.4',  # pure Python, so track HEAD closely
-        'pdfminer.six == 20181108 ; sys_platform != "darwin"',
+        'pdfminer.six == 20181108',
-        'pikepdf >= 1.3.0, < 2',
+        'pikepdf >= 1.5.0, < 2',
         'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
         # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
         # block 5.1.0, broken wheels
         'reportlab >= 3.3.0',  # oldest released version with sane image handling
         'ruffus >= 2.7.0',
     ],
-    extras_require={'pdfminer': ['pdfminer.six == 20181108']},
     tests_require=tests_require,
     entry_points={'console_scripts': ['ocrmypdf = ocrmypdf.__main__:run_pipeline']},
     package_data={'ocrmypdf': ['data/sRGB.icc']},

src/ocrmypdf/_pipeline.py

@@ -816,15 +816,17 @@ def convert_to_pdfa(input_files_groups, output_file, log, context):
     # NULs in DocumentInfo seem to be common since older Acrobats included them.
     # pikepdf can deal with this, but we make the world a better place by
     # stamping them out as soon as possible.
+    modified = False
     with pikepdf.open(layers_file) as pdf_layers_file:
         if pdf_layers_file.docinfo:
-            modified = False
             for k, v in pdf_layers_file.docinfo.items():
                 if b'\x00' in bytes(v):
                     pdf_layers_file.docinfo[k] = bytes(v).replace(b'\x00', b'')
                     modified = True
-            if modified:
+        if modified:
-                pdf_layers_file.save(layers_file)
+            pdf_layers_file.save(layers_file + '_')
+    if modified:
+        os.replace(layers_file + '_', layers_file)
 
     ps = next((ii for ii in input_files if ii.endswith('.ps')), None)
     ghostscript.generate_pdfa(

src/ocrmypdf/pdfinfo/__init__.py

@@ -30,6 +30,7 @@ from pikepdf import PdfMatrix
 import pikepdf
 
 from . import ghosttext
+from .layout import get_page_analysis, get_text_boxes
 
 from ..exceptions import EncryptedPdfError, MissingDependencyError
 
@@ -564,14 +565,6 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
         )
         pageinfo['bboxes'] = bboxes
     else:
-        # pdfminer required for this section
-        try:
-            from .layout import get_page_analysis, get_text_boxes
-        except ImportError:
-            raise MissingDependencyError(
-                "pdfminer is required for this feature. Your distribution "
-                "may not have installed it."
-            )
         pscript5_mode = str(pdf.docinfo.get('/Creator')).startswith('PScript5')
         miner = get_page_analysis(infile, pageno, pscript5_mode)
         pageinfo['textboxes'] = list(simplify_textboxes(miner, get_text_boxes))

tests/conftest.py

@@ -62,16 +62,6 @@ def running_in_travis():
     return os.environ.get('TRAVIS') == 'true'
 
 
-@pytest.helpers.register
-def needs_pdfminer(fn):
-    try:
-        import pdfminer
-    except ImportError:
-        skip = pytest.mark.skipif(True, reason="pdfminer not available")
-        return skip(fn)
-    return fn
-
-
 @pytest.helpers.register
 def have_unpaper():
     try:

tests/test_main.py

@@ -219,7 +219,6 @@ def test_skip_ocr(spoof_tesseract_cache, resources, outpdf):
     assert pdfinfo[0].has_text
 
 
-@pytest.helpers.needs_pdfminer
 def test_redo_ocr(spoof_tesseract_cache, resources, outpdf):
     in_ = resources / 'graph_ocred.pdf'
     before = PdfInfo(in_, detailed_page_analysis=True)

tests/test_metadata.py

@@ -22,7 +22,7 @@ import logging
 import mmap
 from os import fspath
 from pathlib import Path
-from shutil import copyfile
+from shutil import copyfile, move
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -308,10 +308,11 @@ def test_metadata_fixup_warning(resources, outdir, caplog):
         assert record.levelname != 'WARNING'
 
     # Now add some metadata that will not be copyable
-    graph = pikepdf.open(outdir / 'graph.repaired.pdf')
+    with pikepdf.open(outdir / 'graph.repaired.pdf') as graph:
-    with graph.open_metadata() as meta:
+        with graph.open_metadata() as meta:
-        meta['prism2:publicationName'] = 'OCRmyPDF Test'
+            meta['prism2:publicationName'] = 'OCRmyPDF Test'
-    graph.save(outdir / 'graph.repaired.pdf')
+        graph.save(outdir / 'graph.repaired.modified.pdf')
+    move(outdir / 'graph.repaired.modified.pdf', outdir / 'graph.repaired.pdf')
 
     log = logging.getLogger()
     context = MagicMock()

tests/test_pdfinfo.py

@@ -183,12 +183,7 @@ def test_ocr_detection(resources):
 @pytest.mark.parametrize(
     'testfile', ('truetype_font_nomapping.pdf', 'type3_font_nomapping.pdf')
 )
-@pytest.helpers.needs_pdfminer  # pylint: disable=e1101
 def test_corrupt_font_detection(resources, testfile):
-    try:
-        import pdfminer
-    except ImportError:
-        pytest.skip("Needs pdfminer")
     filename = resources / testfile
     with pytest.raises(NotImplementedError):
         pdf = pdfinfo.PdfInfo(filename)

tests/test_weave.py

@@ -44,7 +44,6 @@ def test_no_glyphless_weave(resources, outdir):
     )
 
 
-@pytest.helpers.needs_pdfminer
 def test_links(resources, outpdf):
     check_ocrmypdf(
         resources / 'link.pdf',