optimize: fix error in Py3.5

Eliminates PyPDF2 and defusedxml as dependencies.

.gitignore

@@ -41,4 +41,5 @@ tmp/
 pdfbox-app*.jar
 .vscode/
 IDEAS
-_Dockerfile.local
+_Dockerfile.local
+scratch.py

.travis.yml

@@ -17,6 +17,7 @@ addons:
     - libavcodec56
     - libavformat56
     - libavutil54
+    - libexempi3
     - libffi-dev
     - pngquant
     - poppler-utils
@@ -100,19 +101,6 @@ deploy:
     condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux" && $EXTRAS == ""
   skip_upload_docs: true
 
-  # test pypi
-- provider: pypi
-  server: https://testpypi.pypi.org/legacy/
-  user: ocrmypdf-travis
-  password:
-    secure: "DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo="
-  distributions: "sdist"
-  on:
-    branch: develop
-    tags: false
-    condition: $TRAVIS_OS_NAME == "osx"
-  skip_upload_docs: true
-
   # null deploy for osx
   # we really just want to run after_deploy *after* pypi upload is done, but
   # after_deploy on runs if a given box deployed

.travis/Brewfile

@@ -4,6 +4,7 @@ brew 'ghostscript'
 brew 'jbig2dec'
 brew 'jbig2enc'
 brew 'leptonica'
+brew 'exempi'
 brew 'libffi'
 brew 'libtiff'
 brew 'libxml2'

docs/release_notes.rst

@@ -6,9 +6,48 @@ OCRmyPDF uses `semantic versioning <http://semver.org/>`_ for its command line i
 The OCRmyPDF package itself does not contain a public API, although it is fairly stable and breaking changes are usually timed with a major release. A future release will clearly define the stable public API.
 
 .. Issue regex
-   find:    [^`]\#([0-9]{1,3})[^0-9]  
+   find:    [^`]\#([0-9]{1,3})[^0-9]
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
+
+v7
+--
+
+-   The core algorithm for combining OCR layers with existing PDF pages has been rewritten and improved considerably.  The new algorithm uses less temporary disk space and eliminates object duplication and deduplication that was required when processing certain PDFs.
+
+-   New dependency: `pikepdf <https://github.com/pikepdf>`_. pikepdf is a  powerful new Python PDF library driving the latest OCRmyPDF features, built on QPDF's proven and mature libqpdf C++ library.
+
+-   New feature: PDF optimization with ``-O`` or ``--optimize``.  After OCR, OCRmyPDF will perform image optimizations relevant to OCR PDFs.
+
+    +   If a JBIG2 encoder is available, then monochrome images will be converted, with the potential for huge savings on large black and white files, since JBIG2 is far more efficient than any other monochrome (bi-level) compression. (All known US patents related to JBIG2 have probably expired, but it remains the responsibility of the user to supply a JBIG2 encoder such as `jbig2enc <https://github.com/agl/jbig2enc>`_. OCRmyPDF does not implement JBIG2 encoding.)
+
+    +   If ``pngquant`` is installed, OCRmyPDF will optionally use it to perform lossy quantization and compression of PNG images.
+
+    +   The quality of JPEGs can also be lowered, on the assumption that a lower quality image may be suitable for storage after OCR.
+
+    +   This component will eventually be offered as an independent command line utility.
+
+    +   Optimization ranges from ``-O0`` through ``-O3``, where ``0`` disables optimization and ``3`` implements all options. ``1``, the default, performs only safe and lossless optimizations. (This is similar to GCC's optimization parameter.) The exact type of optimizations performed will vary over time.
+
+-   Small amounts of text in the margins of a page, such as watermarks, page numbers, or digital stamps, will no longer prevent the rest of a page from being OCRed when ``--skip-text`` is issued. This behavior is based on a heuristic.
+
+-   Removed features
+
+    +   The deprecated ``tesseract`` PDF renderer was removed, due to changes in how we construct PDFs.
+
+    +   ``-g``, the option to generate debug text pages, was removed because it was a maintenance burden and only worked in isolated cases. HOCR pages can still be previewed by running the hocrtransform.py with appropriate settings.
+
+-   Removed dependencies
+
+    +   ``PyPDF2`` is no longer used except in the test suite.
+
+    +   ``defusedxml`` is no longer used anywhere.
+
+-   The ``sandwich`` PDF renderer can be used with all supported versions of Tesseract, including that those priority to v3.05 which don't support ``-c textonly``
+
+-   ``--pdf-renderer auto`` option and the diagnostics used to select a PDF renderer now work better with old versions, but may make different decisions than past versions.
+
+
 v6.2.0
 ------
 
@@ -34,7 +73,7 @@ v6.1.5
 v6.1.4
 ------
 
--   Fix issue `#248 <https://github.com/jbarlow83/OCRmyPDF/issues/248>`_ ``--clean`` argument may remove OCR from left column of text on certain documents. We now set ``--layout none`` to suppress this.  
+-   Fix issue `#248 <https://github.com/jbarlow83/OCRmyPDF/issues/248>`_ ``--clean`` argument may remove OCR from left column of text on certain documents. We now set ``--layout none`` to suppress this.
 
 -   The test cache was updated to reflect the change above.
 
@@ -165,7 +204,7 @@ v5.5
 -   Add new argument ``--max-image-mpixels``. Pillow 5.0 now raises an exception when images may be decompression bombs. This argument can be used to override the limit Pillow sets.
 -   Fix output page cropped when using the sandwich renderer and OCR is skipped on a rotated and image-processed page
 -   A warning is now issued when old versions of Ghostscript are used in cases known to cause issues with non-Latin characters
--   Fix a few parameter validation checks for ``-output-type pdfa-1`` and ``pdfa-2`` 
+-   Fix a few parameter validation checks for ``-output-type pdfa-1`` and ``pdfa-2``
 
 
 v5.4.4
@@ -175,7 +214,7 @@ v5.4.4
 -   Fix issue `#200 <https://github.com/jbarlow83/OCRmyPDF/issues/200>`_: an uncommon syntax for formatting decimal numbers in a PDF would cause qpdf to issue a warning, which ocrmypdf treated as an error. Now this the warning is relayed.
 -   Fix an issue where intermediate PDFs would be created at version 1.3 instead of the version of the original file. It's possible but unlikely this had side effects.
 -   A warning is now issued when older versions of qpdf are used since issues like `#200 <https://github.com/jbarlow83/OCRmyPDF/issues/200>`_ cause qpdf to infinite-loop
--   Address issue `#140 <https://github.com/jbarlow83/OCRmyPDF/issues/140>`_: if Tesseract outputs invalid UTF-8, escape it and print its message instead of aborting with a Unicode error 
+-   Address issue `#140 <https://github.com/jbarlow83/OCRmyPDF/issues/140>`_: if Tesseract outputs invalid UTF-8, escape it and print its message instead of aborting with a Unicode error
 -   Adding previously unlisted setup requirement, pytest-runner
 -   Update documentation: fix an error in the example script for Synology with Docker images, improved security guidance, advised ``pip install --user``
 
@@ -344,7 +383,7 @@ v4.4.1
 
 -   To prevent a `TIFF output error <https://github.com/python-pillow/Pillow/issues/2206>`_ caused by img2pdf >= 0.2.1 and Pillow <= 3.4.2, dependencies have been tightened
 -   The Tesseract 4.00 simultaneous process limit was increased from 1 to 2, since it was observed that 1 lowers performance
--   Documentation improvements to describe the ``--tesseract-config`` feature 
+-   Documentation improvements to describe the ``--tesseract-config`` feature
 -   Added test cases and fixed error handling for ``--tesseract-config``
 -   Tweaks to setup.py to deal with issues in the v4.4 release
 
@@ -407,7 +446,7 @@ v4.3
 
 -   New feature ``--remove-background`` to detect and erase the background of color and grayscale images
 -   Better documentation
--   Fixed an issue with PDFs that draw images when the raster stack depth is zero 
+-   Fixed an issue with PDFs that draw images when the raster stack depth is zero
 -   ocrmypdf can now redirect its output to stdout for use in a shell pipeline
 
     +  This does not improve performance since temporary files are still used for buffering
@@ -475,7 +514,7 @@ v4.2
     +  fixes issue `#82 <https://github.com/jbarlow83/OCRmyPDF/issues/82>`_
 
 -   Fixes an issue where, with certain settings, monochrome images in PDFs would be converted to 8-bit grayscale, increasing file size (`#79 <https://github.com/jbarlow83/OCRmyPDF/issues/79>`_)
--   Support for Ubuntu 12.04 LTS "precise" has been dropped in favor of (roughly) Ubuntu 14.04 LTS "trusty" 
+-   Support for Ubuntu 12.04 LTS "precise" has been dropped in favor of (roughly) Ubuntu 14.04 LTS "trusty"
 
     +  Some Ubuntu "PPAs" (backports) are needed to make it work
 
@@ -590,7 +629,7 @@ New features
 
 -   Automatic page rotation (``-r``) is now available. It uses ignores any prior rotation information
     on PDFs and sets rotation based on the dominant orientation of detectable text. This feature is
-    fairly reliable but some false positives occur especially if there is not much text to work with. (`#4 <https://github.com/jbarlow83/OCRmyPDF/issues/4>`_) 
+    fairly reliable but some false positives occur especially if there is not much text to work with. (`#4 <https://github.com/jbarlow83/OCRmyPDF/issues/4>`_)
 -   Deskewing is now performed using Leptonica instead of unpaper. Leptonica is faster and more reliable
     at image deskewing than unpaper.
 
@@ -633,7 +672,7 @@ v3.2
 New features
 ^^^^^^^^^^^^
 
--   Lossless reconstruction: when possible, OCRmyPDF will inject text layers without 
+-   Lossless reconstruction: when possible, OCRmyPDF will inject text layers without
     otherwise manipulating the content and layout of a PDF page. For example, a PDF containing a mix
     of vector and raster content would see the vector content preserved. Images may still be transcoded
     during PDF/A conversion.  (``--deskew`` and ``--clean-final`` disable this mode, necessarily.)
@@ -668,10 +707,10 @@ Changes
     needed to implement support
 -   Improved some error messages related to missing input files
 -   Fixed issue `#20 <https://github.com/jbarlow83/OCRmyPDF/issues/20>`_ - uppercase .PDF extension not accepted
--   Fixed an issue where OCRmyPDF failed to text that certain pages contained previously OCR'ed text, 
+-   Fixed an issue where OCRmyPDF failed to text that certain pages contained previously OCR'ed text,
     such as OCR text produced by Tesseract 3.04
 -   Inserts /Creator tag into PDFs so that errors can be traced back to this project
--   Added new option ``--pdf-renderer=auto``, to let OCRmyPDF pick the best PDF renderer. 
+-   Added new option ``--pdf-renderer=auto``, to let OCRmyPDF pick the best PDF renderer.
     Currently it always chooses the 'hocrtransform' renderer but that behavior may change.
 -   Set up Travis CI automatic integration testing
 
@@ -681,19 +720,19 @@ v3.0
 New features
 ^^^^^^^^^^^^
 
--   Easier installation with a Docker container or Python's ``pip`` package manager 
+-   Easier installation with a Docker container or Python's ``pip`` package manager
 -   Eliminated many external dependencies, so it's easier to setup
 -   Now installs ``ocrmypdf`` to ``/usr/local/bin`` or equivalent for system-wide
     access and easier typing
 -   Improved command line syntax and usage help (``--help``)
 -   Tesseract 3.03+ PDF page rendering can be used instead for better positioning
     of recognized text (``--pdf-renderer tesseract``)
--   PDF metadata (title, author, keywords) are now transferred to the 
+-   PDF metadata (title, author, keywords) are now transferred to the
     output PDF
 -   PDF metadata can also be set from the command line (``--title``, etc.)
 -   Automatic repairs malformed input PDFs if possible
 -   Added test cases to confirm everything is working
--   Added option to skip extremely large pages that take too long to OCR and are 
+-   Added option to skip extremely large pages that take too long to OCR and are
     often not OCRable (e.g. large scanned maps or diagrams); other pages are still
     processed (``--skip-big``)
 -   Added option to kill Tesseract OCR process if it seems to be taking too long on
@@ -712,9 +751,9 @@ Changes
     available CPUs, increasing performance
 -   The ``-o DPI`` argument has been phased out, in favor of ``--oversample DPI``, in
     case we need ``-o OUTPUTFILE`` in the future
--   Removed several dependencies, so it's easier to install.  We no 
+-   Removed several dependencies, so it's easier to install.  We no
     longer use:
-    
+
     - GNU parallel_
     - ImageMagick_
     - Python 2.7
@@ -730,7 +769,7 @@ Changes
     - qpdf_ 5.0.0+
     - Unpaper_ 6.1 (optional)
     - some automatically managed Python packages
-  
+
 .. _ruffus: http://www.ruffus.org.uk/index.html
 .. _parallel: https://www.gnu.org/software/parallel/
 .. _ImageMagick: http://www.imagemagick.org/script/index.php
@@ -776,7 +815,7 @@ Release candidates
     - fixed some installer issues and errors in installation instructions
     - improve performance: run Ghostscript with multithreaded rendering
     - improve performance: use multiple cores by default
-    - bug fix: checking for wrong exception on process timeout 
+    - bug fix: checking for wrong exception on process timeout
 
 -   rc3: skipping version number intentionally to avoid confusion with Tesseract
 -   rc2: first release for public testing to test-PyPI, Github
@@ -799,11 +838,11 @@ where ``settings.txt`` contains *one argument per line*, for example:
 
 ::
 
-    -l 
-    deu 
-    --author 
-    A. Merkel 
-    --pdf-renderer 
+    -l
+    deu
+    --author
+    A. Merkel
+    --pdf-renderer
     tesseract
 
 

docs/security.rst

@@ -1,7 +1,7 @@
 PDF security issues
 ===================
 
-	OCRmyPDF should only be used on PDFs you trust. It is not designed to protect you against malware. 
+	OCRmyPDF should only be used on PDFs you trust. It is not designed to protect you against malware.
 
 Recognizing that many users have an interest in handling PDFs and applying OCR to PDFs they did not generate themselves, this article discusses the security implications of PDFs and how users can protect themselves.
 
@@ -19,9 +19,7 @@ This `article <https://theinvisiblethings.blogspot.ca/2013/02/converting-untrust
 How OCRmyPDF processes PDFs
 ---------------------------
 
-OCRmyPDF must open and interpret your PDF in order to insert an OCR layer. First, it runs all PDFs through `qpdf <https://github.com/qpdf/qpdf>`_, a program that repairs PDFs with syntax errors. This is done because, in the author's experience, a significant number of PDFs in the wild especially those created by scanners are not well-formed files. qpdf makes it more likely that OCRmyPDF will succeed, but offers no security guarantees. qpdf is also used to split the PDF into single page PDFs.
-
-After qpdf, OCRmyPDF examines each page using `PyPDF2 <https://github.com/mstamy2/PyPDF2>`_. This library also has no warranties or guarantees. OCRmyPDF works with qpdf 5.0 and up, but version 7.0 is recommended because of known security vulnerabilities in early versions.
+OCRmyPDF must open and interpret your PDF in order to insert an OCR layer. First, it runs all PDFs through `pikepdf <https://github.com/pikepdf/pikepdf>`_, a library based on `qpdf <https://github.com/qpdf/qpdf>`_, a program that repairs PDFs with syntax errors. This is done because, in the author's experience, a significant number of PDFs in the wild especially those created by scanners are not well-formed files. qpdf makes it more likely that OCRmyPDF will succeed, but offers no security guarantees. qpdf is also used to split the PDF into single page PDFs.
 
 Finally, OCRmyPDF rasterizes each page of the PDF using `Ghostscript <http://ghostscript.com/>`_ in ``-dSAFER`` mode.
 
@@ -60,13 +58,13 @@ Commercial alternatives
 
 The author also provides professional services that include OCR and building databases around PDFs, and is happy to provide consultation.
 
-Abbyy Cloud OCR is a viable commercial alternative with a web services API. 
+Abbyy Cloud OCR is a viable commercial alternative with a web services API.
 
 
 Password protection, digital signatures and certification
 ---------------------------------------------------------
 
-Password protected PDFs usually have two passwords, and owner and user password. When the user password is set to empty, PDF readers will open the file automatically and marked it as "(SECURED)". While not as reliable as a digital signature, this indicates that whoever set the password approved of the file at that time. When the user password is set, the document cannot be viewed without the password. 
+Password protected PDFs usually have two passwords, and owner and user password. When the user password is set to empty, PDF readers will open the file automatically and marked it as "(SECURED)". While not as reliable as a digital signature, this indicates that whoever set the password approved of the file at that time. When the user password is set, the document cannot be viewed without the password.
 
 Either way, OCRmyPDF does not remove passwords from PDFs and exits with an error on encountering them.
 
@@ -76,4 +74,4 @@ After OCR is applied, password protection is not permitted on PDF/A documents bu
 
 Many programs exist which are capable of inserting an image of someone's signature. On its own, this offers no security guarantees. It is trivial to remove the signature image and apply it to other files. This practice offers no real security.
 
-Important documents can be digitally signed and certified to attest to their authorship. OCRmyPDF cannot do this. Open source tools such as pdfbox (Java) have this capability as does Adobe Acrobat. 
+Important documents can be digitally signed and certified to attest to their authorship. OCRmyPDF cannot do this. Open source tools such as pdfbox (Java) have this capability as does Adobe Acrobat.

setup.py

@@ -185,13 +185,17 @@ if not forced and command.startswith('install') or \
         package='unpaper',
         optional=True
     )
-    if not os.environ.get('OCRMYPDF_QPDF_APPIMAGE'):
-        check_external_program(
-            program='qpdf',
-            need_version='7.0.0', # test suite known to fail on 5.1.1
-            package='qpdf',
-            version_check_args=['--version']
-        )
+    check_external_program(
+        program='qpdf',
+        need_version='8.0.2', # test suite known to fail on 5.1.1
+        package='qpdf'
+    )
+    check_external_program(
+        program='pngquant',
+        need_version='2.0.0',
+        package='pngquant',
+        optional=True
+    )
 
 if 'upload' in sys.argv[1:]:
     print('Use twine to upload the package - setup.py upload is insecure')
@@ -243,13 +247,12 @@ setup(
     ],
     install_requires=[
         'cffi >= 1.9.1',          # must be a setup and install requirement
-        'defusedxml >= 0.5.0',    # pure Python, so track HEAD closely
         'img2pdf >= 0.2.4',       # pure Python, so track HEAD closely
         'pikepdf',
-        'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',        
+        'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
                                   # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
                                   # block 5.1.0, broken wheels
-        'PyPDF2 >= 1.26',         # pure Python, so track HEAD closely
+        'python-xmp-toolkit >= 2, < 3',
         'reportlab >= 3.3.0',     # oldest released version with sane image handling
         'ruffus == 2.6.3',        # pinned - ocrmypdf implements a 2.6.3 workaround
     ],

src/ocrmypdf/__main__.py

@@ -22,14 +22,12 @@ from pathlib import Path
 import sys
 import os
 import re
-import warnings
 import multiprocessing
 import atexit
 import textwrap
 import logging
 import argparse
 
-import PyPDF2 as pypdf
 import PIL
 
 import ruffus.ruffus_exceptions as ruffus_exceptions
@@ -50,8 +48,6 @@ from .exceptions import ExitCode, ExitCodeException, MissingDependencyError, \
 from . import exceptions as ocrmypdf_exceptions
 from ._unicodefun import verify_python3_env
 
-warnings.simplefilter('ignore', pypdf.utils.PdfReadWarning)
-
 
 # -------------
 # External dependencies
@@ -70,7 +66,7 @@ def complain(message):
 if 'IDE_PROJECT_ROOTS' in os.environ:
     os.environ['PATH'] = '/usr/local/bin:' + os.environ['PATH']
 
-# -------- 
+# --------
 # Critical environment tests
 
 verify_python3_env()
@@ -171,7 +167,7 @@ parser.add_argument(
     '--image-dpi', metavar='DPI', type=int,
     help="For input image instead of PDF, use this DPI instead of file's.")
 parser.add_argument(
-    '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'], 
+    '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'],
     default='pdfa',
     help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for "
          "long term archiving (default, recommended) but may not suitable "
@@ -314,12 +310,12 @@ advanced.add_argument(
          "choose.  See documentation for discussion."
     )
 advanced.add_argument(
-    '--tesseract-timeout', default=180.0, type=numeric(float, 0), 
+    '--tesseract-timeout', default=180.0, type=numeric(float, 0),
     metavar='SECONDS',
     help='Give up on OCR after the timeout, but copy the preprocessed page '
          'into the final output')
 advanced.add_argument(
-    '--rotate-pages-threshold', default=14.0, type=numeric(float, 0, 1000), 
+    '--rotate-pages-threshold', default=14.0, type=numeric(float, 0, 1000),
     metavar='CONFIDENCE',
     help="Only rotate pages when confidence is above this value (arbitrary "
          "units reported by tesseract)")
@@ -394,10 +390,10 @@ def check_options_output(options, log):
     if options.pdf_renderer == 'hocr' and not is_latin:
         msg = (
             "The 'hocr' PDF renderer is known to cause problems with one "
-            "or more of the languages in your document.  Use " 
+            "or more of the languages in your document.  Use "
             "--pdf-renderer auto (the default) to avoid this issue.")
         log.warning(msg)
-    
+
     if ghostscript.version() < '9.20' \
             and options.output_type != 'pdf' \
             and not is_latin:
@@ -478,13 +474,13 @@ def check_options_preprocessing(options, log):
         _optional_program_check(
             'unpaper', unpaper.version, '6.1', '--clean, --clean-final'
         )
-    
+
 
 def check_options_ocr_behavior(options, log):
     if options.force_ocr and options.skip_text:
         raise argparse.ArgumentError(
             None,
-            "Error: --force-ocr and --skip-text are mutually incompatible.")
+            "Error: --force-ocr and --skip-text are mutually exclusive.")
 
 
 def check_options_optimizing(options, log):
@@ -508,7 +504,7 @@ def check_options_advanced(options, log):
             "--pdfa-image-compression argument has no effect when "
             "--output-type is not 'pdfa', 'pdfa-1', or 'pdfa-2'"
         )
-    
+
     if tesseract.v4() and (options.user_words or options.user_patterns):
         log.warning(
             'Tesseract 4.x ignores --user-words, so this has no effect')
@@ -597,7 +593,7 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
     if exc_name == 'builtins.SystemExit':
         match = re.search(r"\.(.+?)\)", exc_value)
         exit_code_name = match.groups()[0]
-        exit_code = getattr(ExitCode, exit_code_name, 'other_error')        
+        exit_code = getattr(ExitCode, exit_code_name, 'other_error')
     elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':
         log.error(cleanup_ruffus_error_message(exc_value))
         exit_code = ExitCode.input_file
@@ -616,12 +612,10 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
         msg = "Error occurred while running this command:"
         log.error(msg + '\n' + exc_value)
         exit_code = ExitCode.child_process_error
-    elif (exc_name == 'PyPDF2.utils.PdfReadError' and \
-            'not been decrypted' in exc_value) or \
-            (exc_name == 'ocrmypdf.exceptions.EncryptedPdfError'):
+    elif exc_name == 'ocrmypdf.exceptions.EncryptedPdfError':
         log.error(textwrap.dedent("""\
             Input PDF is encrypted. The encryption must be removed to
-            perform OCR. 
+            perform OCR.
 
             For information about this PDF's security use
                 qpdf --show-encryption infilename
@@ -630,7 +624,7 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
                 qpdf --decrypt [--password=[password]] infilename
 
             """))
-        exit_code = ExitCode.encrypted_pdf        
+        exit_code = ExitCode.encrypted_pdf
     elif exc_name == 'ocrmypdf.exceptions.PdfMergeFailedError':
         log.error(textwrap.dedent("""\
             Failed to merge PDF image layer with OCR layer
@@ -669,12 +663,12 @@ def traverse_ruffus_exception(e_args, options, log):
       (task, job, exc, value, stack)
     or something like:
       [[(task, job, exc, value, stack)]]
-    
+
     Generally cross-process exception marshalling doesn't work well
     and ruffus doesn't support because BaseException has its own
     implementation of __reduce__ that attempts to reconstruct the
     exception based on e.__init__(e.args).
-    
+
     Attempting to log the exception directly marshalls it to the logger
     which is probably in another process, so it's better to log only
     data from the exception at this point.
@@ -772,7 +766,7 @@ def check_environ(options, _log):
     for k in old_envvars:
         if k in os.environ:
             _log.warning(textwrap.dedent("""\
-                OCRmyPDF no longer uses the environment variable {}. 
+                OCRmyPDF no longer uses the environment variable {}.
                 Change PATH to select alternate programs.""".format(k)))
 
 
@@ -815,14 +809,12 @@ def report_output_file_size(options, _log, input_file, output_file):
     ratio = output_size / input_size
     if ratio < 1.35 or input_size < 25000:
         return  # Seems fine
-    
+
     reasons = []
-    if not fitz:
-        reasons.append("The optional dependency PyMuPDF is not installed.")
     image_preproc = {
-        'deskew', 
-        'clean_final', 
-        'remove_background', 
+        'deskew',
+        'clean_final',
+        'remove_background',
         'oversample',
         'force_ocr'
     }
@@ -943,7 +935,7 @@ def run_pipeline():
             _log.warning('Output file: The generated PDF is INVALID')
             return ExitCode.invalid_output_pdf
 
-        report_output_file_size(options, _log, start_input_file, 
+        report_output_file_size(options, _log, start_input_file,
                                 options.output_file)
 
     pdfinfo = context.get_pdfinfo()

src/ocrmypdf/_optimize.py

@@ -26,12 +26,11 @@ import sys
 from io import BytesIO
 from PIL import Image
 
-from .lib import fitz
 import pikepdf
 
 from ._jobcontext import JobContext
 from . import leptonica
-from .helpers import re_symlink
+from .helpers import re_symlink, fspath
 from .exec import pngquant, jbig2enc
 
 PAGE_GROUP_SIZE = 10
@@ -55,7 +54,7 @@ def tif_name(root, xref):
     return img_name(root, xref, '.tif')
 
 
-def extract_image(*, doc, pike, root, log, image, xref, jbig2s, 
+def extract_image(*, pike, root, log, image, xref, jbig2s,
                   pngs, jpegs, options):
     if image.Subtype != '/Image':
         return False
@@ -85,7 +84,7 @@ def extract_image(*, doc, pike, root, log, image, xref, jbig2s,
                 ext = pim.extract_to(stream=f)
             imgname.rename(imgname.with_suffix(ext))
         except pikepdf.UnsupportedImageTypeError:
-            return False        
+            return False
         jbig2s.append((xref, ext))
     elif filtdp[0] == '/DCTDecode' \
             and options.optimize >= 2:
@@ -111,33 +110,27 @@ def extract_image(*, doc, pike, root, log, image, xref, jbig2s,
                 ext = pim.extract_to(stream=f)
             imgname.rename(imgname.with_suffix(ext))
         except pikepdf.UnsupportedImageTypeError:
-            return False        
+            return False
         jpegs.append(xref)
     elif pim.indexed \
             and pim.colorspace in pim.SIMPLE_COLORSPACES \
-            and options.optimize >= 3 \
-            and fitz:
+            and options.optimize >= 3:
         # Try to improve on indexed images - these are far from low hanging
         # fruit in most cases
-        pix = fitz.Pixmap(doc, xref)
-        pix.writePNG(png_name(root, xref), savealpha=False)
-        pngs.append(xref)        
-    elif pim.colorspace in pim.SIMPLE_COLORSPACES and fitz:
-        # For any 'inferior' filter including /FlateDecode we extract
-        # and recode as /FlateDecode
-        # raw_png = pike._get_object_id(xref, 0)
-        # raw_png_data = raw_png.read_raw_bytes()
-        # (root / '{:08d}.png'.format(xref)).write_bytes(raw_png_data)
-        pix = fitz.Pixmap(doc, xref)
-        pix.writePNG(png_name(root, xref), savealpha=False)
+        pim.as_pil_image().save(png_name(root, xref))
+        pngs.append(xref)
+    elif not pim.indexed and pim.colorspace in pim.SIMPLE_COLORSPACES:
+        # An optimization opportunity here, not currently taken, is directly
+        # generating a PNG from compressed data
+        pim.as_pil_image().save(png_name(root, xref))
         pngs.append(xref)
     else:
         return False
-    
+
     return True
 
 
-def extract_images(doc, pike, root, log, options):
+def extract_images(pike, root, log, options):
     # Extract images we can improve
     changed_xrefs = set()
     jbig2_groups = defaultdict(lambda: [])
@@ -156,8 +149,8 @@ def extract_images(doc, pike, root, log, options):
                 continue  # Don't improve same image twice
             try:
                 result = extract_image(
-                    doc=doc, pike=pike, root=root, log=log, image=image, 
-                    xref=xref, jbig2s=jbig2_groups[group], pngs=pngs, 
+                    pike=pike, root=root, log=log, image=image,
+                    xref=xref, jbig2s=jbig2_groups[group], pngs=pngs,
                     jpegs=jpegs, options=options
                 )
                 if result:
@@ -171,7 +164,7 @@ def extract_images(doc, pike, root, log, options):
     jbig2_groups = {group: xrefs for group, xrefs in jbig2_groups.items()
                     if len(xrefs) > 0}
     log.debug(
-        "Optimizable images: " 
+        "Optimizable images: "
         "JBIG2 groups: {} JPEGs: {} PNGs: {} Errors: {}".format(
             len(jbig2_groups), len(jpegs), len(pngs), errors
     ))
@@ -186,7 +179,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
     We use a group because JBIG2 works best with a symbol dictionary that spans
     multiple pages. When inserted back into the PDF, each JBIG2 must reference
     the symbol dictionary it is associated with. So convert a group at a time,
-    and replace their streams with a parameter set that points to the 
+    and replace their streams with a parameter set that points to the
     appropriate dictionary.
 
     If too many pages shared the same dictionary JBIG2 encoding becomes more
@@ -199,7 +192,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
         for group, xref_exts in jbig2_groups.items():
             prefix = 'group{:08d}'.format(group)
             future = executor.submit(
-                jbig2enc.convert_group, 
+                jbig2enc.convert_group,
                 cwd=str(root),
                 infiles=(img_name(root, xref, ext) for xref, ext in xref_exts),
                 out_prefix=prefix
@@ -220,7 +213,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
             jbig2_im_data = jbig2_im_file.read_bytes()
             im_obj = pike._get_object_id(xref, 0)
             im_obj.write(
-                jbig2_im_data, pikepdf.Name('/JBIG2Decode'), 
+                jbig2_im_data, pikepdf.Name('/JBIG2Decode'),
                 pikepdf.Dictionary({
                     '/JBIG2Globals': jbig2_globals
                 })
@@ -237,7 +230,7 @@ def transcode_jpegs(pike, jpegs, root, log, options):
         # 'close'.  Seems to be mostly harmless
         # https://github.com/python-pillow/Pillow/issues/1144
         with Image.open(str(in_jpg)) as im:
-            im.save(str(opt_jpg), 
+            im.save(str(opt_jpg),
                     optimize=True,
                     quality=JPEG_QUALITY)
         if opt_jpg.stat().st_size > in_jpg.stat().st_size:
@@ -258,8 +251,8 @@ def transcode_pngs(pike, pngs, root, log, options):
                 max_workers=options.jobs) as executor:
             for xref in pngs:
                 executor.submit(
-                    pngquant.quantize, 
-                    png_name(root, xref), png_name(root, xref), 
+                    pngquant.quantize,
+                    png_name(root, xref), png_name(root, xref),
                     PNG_QUALITY[0], PNG_QUALITY[1])
 
     for xref in pngs:
@@ -274,7 +267,7 @@ def transcode_pngs(pike, pngs, root, log, options):
         except leptonica.LeptonicaError as e:
             log.error(e)
             continue
-        
+
         # This is what we should be doing: open the compressed data without
         # transcoding. However this shifts each pixel row by one for some
         # reason.
@@ -326,23 +319,19 @@ def optimize(
         PNG_QUALITY = (20, 40)
         JPEG_QUALITY = 40
 
-    if fitz:
-        doc = fitz.open(str(input_file))
-    else:
-        doc = None
     pike = pikepdf.Pdf.open(input_file)
 
     root = Path(output_file).parent / 'images'
     root.mkdir(exist_ok=True)
     changed_xrefs, jbig2_groups, jpegs, pngs = extract_images(
-        doc, pike, root, log, options)
+        pike, root, log, options)
 
     convert_to_jbig2(pike, jbig2_groups, root, log, options)
     transcode_jpegs(pike, jpegs, root, log, options)
     transcode_pngs(pike, pngs, root, log, options)
 
     # Not object_stream_mode + preserve_pdfa generates noncompliant PDFs
-    target_file = output_file + '_opt.pdf'
+    target_file = Path(output_file).with_suffix('.opt.pdf')
     pike.save(target_file, preserve_pdfa=True)
 
     input_size = Path(input_file).stat().st_size
@@ -351,15 +340,17 @@ def optimize(
     savings = 1 - output_size / input_size
     log.info("Optimize ratio: {:.2f} savings: {:.1f}%".format(
         ratio, 100 * savings))
-    
+
     if savings < 0:
         log.info("Optimize did not improve the file - discarded")
         re_symlink(input_file, output_file, log)
     else:
         re_symlink(target_file, output_file, log)
-        
+
 
 def main(infile, outfile, level, jobs=1):
+    from tempfile import TemporaryDirectory
+    from shutil import copy
     Options = namedtuple('Options', 'jobs optimize')
 
     logging.basicConfig(level=logging.DEBUG)
@@ -369,8 +360,11 @@ def main(infile, outfile, level, jobs=1):
     options = Options(jobs=jobs, optimize=int(level))
     ctx.set_options(options)
 
-    optimize(infile, outfile, log, ctx)
+    with TemporaryDirectory() as td:
+        tmpout = Path(td) / 'out.pdf'
+        optimize(infile, tmpout, log, ctx)
+        copy(fspath(tmpout), fspath(outfile))
 
 
 if __name__ == '__main__':
-    main(sys.argv[1], sys.argv[2], sys.argv[3])
+    main(sys.argv[1], sys.argv[2], sys.argv[3])

src/ocrmypdf/_weave.py

@@ -21,6 +21,7 @@ from itertools import groupby
 import pikepdf
 
 from .helpers import flatten_groups, page_number
+from .exec import tesseract
 
 
 def _update_page_resources(*, page, font, font_key, procset):
@@ -54,17 +55,27 @@ def _weave_layers_graft(
     pdf_text = pikepdf.open(text)
     pdf_text_contents = pdf_text.pages[0].Contents.read_bytes()
 
+    if not tesseract.has_textonly_pdf():
+        # If we don't have textonly_pdf, edit the stream to delete the
+        # instruction to draw the image Tesseract generated, which we do not
+        # use.
+        stream = bytearray(pdf_text_contents)
+        pattern = b'/Im1 Do'
+        idx = stream.find(pattern)
+        stream[idx:(idx + len(pattern))] = b' ' * len(pattern)
+        pdf_text_contents = bytes(stream)
+
     base_page = pdf_base.pages.p(page_num)
 
     # The text page always will be oriented up by this stage but the original
     # content may have a rotation applied. Wrap the text stream with a rotation
     # so it will be oriented the same way as the rest of the page content.
-    # (Previous versions OCRmyPDF rotated the content layer to match the text.)    
-    mediabox = [float(pdf_text.pages[0].MediaBox[v].decode()) 
+    # (Previous versions OCRmyPDF rotated the content layer to match the text.)
+    mediabox = [float(pdf_text.pages[0].MediaBox[v].decode())
                 for v in range(4)]
     wt, ht = mediabox[2] - mediabox[0], mediabox[3] - mediabox[1]
 
-    mediabox = [float(base_page.MediaBox[v].decode()) 
+    mediabox = [float(base_page.MediaBox[v].decode())
                 for v in range(4)]
     wp, hp = mediabox[2] - mediabox[0], mediabox[3] - mediabox[1]
 
@@ -83,16 +94,16 @@ def _weave_layers_graft(
         c, s = 0, -1
     else:
         raise NotImplementedError("rotation to arbitrary angle")
-    
+
     rotate = pikepdf.PdfMatrix((c, s, -s, c, 0, 0))
 
     # Because of rounding of DPI, we might get a text layer that is not
     # identically sized to the target page. Scale to adjust. Normally this
     # is within 0.998.
+    if rotation in (90, 270):
+        wt, ht = ht, wt
     scale_x = wp / wt
     scale_y = hp / ht
-    if rotation % 90 == 0:
-        scale_x, scale_y = scale_y, scale_x
 
     log.debug('%r', (scale_x, scale_y))
     scale = pikepdf.PdfMatrix((scale_x, 0, 0, scale_y, 0, 0))
@@ -100,7 +111,7 @@ def _weave_layers_graft(
     # Translate the text so it is centered at (0, 0), rotate it there, adjust
     # for a size different between initial and text PDF, then untranslate
     ctm = translate @ rotate @ scale @ untranslate
-        
+
     pdf_text_contents = (
         b'q %s cm\n' % ctm.encode() +
         pdf_text_contents +
@@ -152,7 +163,7 @@ def _fix_toc(pdf_base, pageref_remap, log):
     the two types of object in the table of contents that can be page bookmarks
     and update the page entry.
 
-    It may ultimately be better to find a way to rebuild a page in place.    
+    It may ultimately be better to find a way to rebuild a page in place.
 
     """
 
@@ -191,7 +202,7 @@ def _fix_toc(pdf_base, pageref_remap, log):
             objgen = item._objgen
             if objgen not in visited:
                 queue.add(objgen)
-        
+
         if '/Dest' in node:
             remap_dest(node['/Dest'])
         elif '/A' in node:
@@ -205,7 +216,7 @@ def weave_layers(
         log,
         context):
     """Apply text layer and/or image layer changes to baseline file
-    
+
     This is where the magic happens. infiles will be the main PDF to modify,
     and optional .text.pdf and .image-layer.pdf files, organized however ruffus
     organizes them.
@@ -224,7 +235,7 @@ def weave_layers(
     doesn't actually copy the data until asked to write, so all the resources
     it may need to remain available.
 
-    For completeness, we set up a /ProcSet on every page, although it's 
+    For completeness, we set up a /ProcSet on every page, although it's
     unlikely any PDF viewer cares about this anymore.
 
     """
@@ -246,7 +257,7 @@ def weave_layers(
     font, font_key, procset = None, None, None
     pdfinfo = context.get_pdfinfo()
     pagerefs = {}
-    
+
     procset = pdf_base.make_indirect(
         pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]'))
 
@@ -294,14 +305,14 @@ def weave_layers(
         text_rotation = autorotate_correction
         text_misaligned = (text_rotation - content_rotation) % 360
         log.debug('%r', [
-            text_rotation, autorotate_correction, text_misaligned, 
+            text_rotation, autorotate_correction, text_misaligned,
             content_rotation]
         )
 
         if text and font:
             # Graft the text layer onto this page, whether new or old
             _weave_layers_graft(
-                pdf_base=pdf_base, page_num=page_num, text=text, font=font, 
+                pdf_base=pdf_base, page_num=page_num, text=text, font=font,
                 font_key=font_key, rotation=text_misaligned, procset=procset,
                 log=log
             )
@@ -329,4 +340,4 @@ def weave_layers(
             font = pdf_base.pages[0].Resources.Font.get(font_key)
 
     _fix_toc(pdf_base, pagerefs, log)
-    pdf_base.save(output_file)
+    pdf_base.save(output_file)

src/ocrmypdf/exec/qpdf.py

@@ -30,8 +30,6 @@ from ..helpers import re_symlink
 
 @lru_cache(maxsize=1)
 def version():
-    if 'OCRMYPDF_QPDF_APPIMAGE' in os.environ:
-        return os.environ['OCRMYPDF_QPDF_APPIMAGE']
     return get_version('qpdf', regex=r'qpdf version (.+)')
 
 
@@ -46,7 +44,7 @@ def check(input_file, log=None):
         import logging as log
 
     try:
-        run(args_qpdf, stderr=STDOUT, stdout=PIPE, universal_newlines=True, 
+        run(args_qpdf, stderr=STDOUT, stdout=PIPE, universal_newlines=True,
             check=True)
     except CalledProcessError as e:
         if e.returncode == 2:
@@ -76,7 +74,7 @@ def repair(input_file, output_file, log):
         'qpdf', input_file, output_file
     ]
     try:
-        run(args_qpdf, stderr=STDOUT, stdout=PIPE, universal_newlines=True, 
+        run(args_qpdf, stderr=STDOUT, stdout=PIPE, universal_newlines=True,
             check=True)
     except CalledProcessError as e:
         if e.returncode == 3 and e.output.find("operation succeeded"):
@@ -131,7 +129,7 @@ def extract_page(input_file, output_file, pageno):
         output_file
     ]
     run(args_qpdf, check=True)
-    
+
 
 def _merge_inner(input_files, output_file, min_version=None, log=None):
     """Merge the list of input files (all filenames) into the output file.
@@ -173,10 +171,10 @@ def merge(input_files, output_file, min_version=None, log=None, max_files=None):
     The input files may contain one or more pages.
 
     """
-    # qpdf requires that every file that contributes to the output has a file 
-    # descriptor that remains open. That means, given our approach of one 
-    # intermediate PDF per, we can practically hit the number of file 
-    # descriptors. 
+    # qpdf requires that every file that contributes to the output has a file
+    # descriptor that remains open. That means, given our approach of one
+    # intermediate PDF per, we can practically hit the number of file
+    # descriptors.
 
     if max_files is None or max_files < 2:
         # Find out how many open file descriptors we can get away with
@@ -188,7 +186,7 @@ def merge(input_files, output_file, min_version=None, log=None, max_files=None):
     output_dir = os.path.dirname(output_file)
 
     import random
-    import string    
+    import string
 
     def randstr():
         return ''.join(random.sample(string.ascii_lowercase, 6))
@@ -212,7 +210,7 @@ def merge(input_files, output_file, min_version=None, log=None, max_files=None):
         counter += 1
         _merge_inner(job, merge_file, min_version=min_version, log=log)
 
-        # On the next 
+        # On the next
         next_workqueue.append(merge_file)
         log.debug('next_workqueue ' + repr(next_workqueue))
 
@@ -225,5 +223,3 @@ def merge(input_files, output_file, min_version=None, log=None, max_files=None):
             next_workqueue = []
 
     re_symlink(workqueue.pop(), output_file, log)
-
-

src/ocrmypdf/exec/tesseract.py

@@ -26,8 +26,6 @@ from subprocess import PIPE, CalledProcessError, \
     TimeoutExpired, check_output, STDOUT
 from contextlib import suppress
 
-import PyPDF2 as pypdf
-
 from ..exceptions import MissingDependencyError, TesseractConfigError
 from ..helpers import page_number
 from . import get_version
@@ -67,8 +65,8 @@ def v4():
 @lru_cache(maxsize=1)
 def has_textonly_pdf():
     """Does Tesseract have textonly_pdf capability?
-    
-    Available in 3.05.01, and v4.00.00alpha since January 2017. Best to 
+
+    Available in 3.05.01, and v4.00.00alpha since January 2017. Best to
     parse the parameter list
     """
     args_tess = [
@@ -190,6 +188,10 @@ def tesseract_log_output(log, stdout, input_file):
             log.warning(prefix + "lots of diacritics - possibly poor OCR")
         elif line.startswith('OSD: Weak margin'):
             log.warning(prefix + "unsure about page orientation")
+        elif 'Error in pixScanForForeground' in line:
+            pass  # Appears to be spurious/problem with nonwhite borders
+        elif 'Error in boxClipToRectangle' in line:
+            pass  # Always appears with pixScanForForeground message
         elif 'error' in line.lower() or 'exception' in line.lower():
             log.error(prefix + line.strip())
         elif 'warning' in line.lower():

src/ocrmypdf/helpers.py

@@ -29,7 +29,8 @@ def re_symlink(input_file, soft_link_name, log=None):
     """
     Helper function: relinks soft symbolic link if necessary
     """
-
+    input_file = str(input_file)  # For Py3.5
+    soft_link_name = str(soft_link_name)
     if log is None:
         prdebug = partial(print, file=sys.stderr)
     else:
@@ -103,7 +104,7 @@ def is_file_writable(test_file):
 
     if p.is_symlink():
         # Python 3.5 does not accept parameters for Path.resolve() and behaves
-        # as if strict=True (throws an exception on failure). Python 3.6 
+        # as if strict=True (throws an exception on failure). Python 3.6
         # defaults to strict=False. This implements strict=False like behavior
         # for Python 3.5.
         if sys.version_info[0:2] <= (3, 5):

src/ocrmypdf/pdfa.py

@@ -20,11 +20,9 @@
 from string import Template
 from binascii import hexlify
 from datetime import datetime
-from xml.parsers.expat import ExpatError
 import pkg_resources
-import PyPDF2 as pypdf
-from defusedxml.minidom import parseString as defused_parseString
-from unittest.mock import patch
+from libxmp.utils import file_to_dict
+from libxmp import consts
 
 ICC_PROFILE_RELPATH = 'data/sRGB.icc'
 
@@ -105,7 +103,7 @@ def encode_text_string(s: str) -> str:
 def encode_pdf_date(d: datetime) -> str:
     """Encode Python datetime object as PDF date string
 
-    From Adobe pdfmark manual:    
+    From Adobe pdfmark manual:
     (D:YYYYMMDDHHmmSSOHH'mm')
     D: is an optional prefix. YYYY is the year. All fields after the year are
     optional. MM is the month (01-12), DD is the day (01-31), HH is the
@@ -140,13 +138,13 @@ def decode_pdf_date(s: str) -> datetime:
     if s.startswith('D:'):
         s = s[2:]
 
-    # Literal Z00'00', is incorrect but found in the wild, 
+    # Literal Z00'00', is incorrect but found in the wild,
     # probably made by OS X Quartz -- standardize
     if s.endswith("Z00'00'"):
         s = s.replace("Z00'00'", '+0000')
     elif s.endswith('Z'):
         s = s.replace('Z', '+0000')
-    
+
     s = s.replace("'", "")  # Remove apos from PDF time strings
 
     return datetime.strptime(s, r'%Y%m%d%H%M%S%z')
@@ -154,7 +152,7 @@ def decode_pdf_date(s: str) -> datetime:
 
 def _get_pdfmark_dates(pdfmark):
     """Encode dates for pdfmark Postscript.  The best way to deal with a
-    missing date entry is set it to null, because if the key is omitted 
+    missing date entry is set it to null, because if the key is omitted
     Ghostscript will set it to now - we do not want to erase the fact that
     the value was unknown.  Setting to an empty string breaks Ghostscript
     9.22 as reported here:
@@ -172,7 +170,7 @@ def _get_pdfmark_dates(pdfmark):
             date_str = date_str[2:]
         try:
             yield '  {} (D:{})'.format(
-                    key, 
+                    key,
                     encode_pdf_date(decode_pdf_date(date_str)))
         except ValueError:
             yield '  {} null'.format(key)
@@ -180,7 +178,7 @@ def _get_pdfmark_dates(pdfmark):
 
 def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
     """Create a Postscript file for Ghostscript.  pdfmark contains the various
-    objects as strings; these must be encoded in ASCII, and dates have a 
+    objects as strings; these must be encoded in ASCII, and dates have a
     special format."""
 
     # Ghostscript <= 9.21 has a bug where null entries in DOCINFO might produce
@@ -229,38 +227,35 @@ def file_claims_pdfa(filename):
 
     This checks if the XMP metadata contains a PDF/A marker.
     """
-    pdf = pypdf.PdfFileReader(filename)
-    try:
-        # Monkeypatch PyPDF2 to use defusedxml as its XML parser, for safety
-        with patch('xml.dom.minidom.parseString', new=defused_parseString):
-            xmp = pdf.getXmpMetadata()
-    except ExpatError:
-        return {'pass': False, 'output': 'pdf',
-                'conformance': 'Invalid XML metadata'}
 
-    try:
-        pdfa_nodes = xmp.getNodesInNamespace(
-            aboutUri='',
-            namespace='http://www.aiim.org/pdfa/ns/id/')
-    except AttributeError:
+    xmp = file_to_dict(filename)
+    if not xmp:
         return {'pass': False, 'output': 'pdf',
                 'conformance': 'No XMP metadata'}
 
-    pdfa_dict = {attr.localName: attr.value for attr in pdfa_nodes}
-    if not pdfa_dict:
+    if not consts.XMP_NS_PDFA_ID in xmp:
         return {'pass': False, 'output': 'pdf',
-                'conformance': 'No XMP metadata'}
+                'conformance': 'No PDF/A metadata in XMP'}
 
-    part_conformance = pdfa_dict['part'] + pdfa_dict['conformance']
+    pdfa_node = xmp[consts.XMP_NS_PDFA_ID]
+    def read_node(node, key):
+        return next(
+            (v for k, v, meta in pdfa_node if k == key), ''
+        )
+
+    part = read_node(pdfa_node, 'pdfaid:part')
+    conformance = read_node(pdfa_node, 'pdfaid:conformance')
+
+    part_conformance = part + conformance
     valid_part_conforms = {'1A', '1B', '2A', '2B', '2U', '3A', '3B', '3U'}
 
     conformance = 'PDF/A-{}'.format(
         part_conformance)
 
+    pdfa_dict = {}
     if part_conformance in valid_part_conforms:
         pdfa_dict['pass'] = True
         pdfa_dict['output'] = 'pdfa'
     pdfa_dict['conformance'] = conformance
 
     return pdfa_dict
-

src/ocrmypdf/pdfinfo.py

@@ -28,13 +28,11 @@ from pathlib import Path
 from enum import Enum
 from contextlib import contextmanager
 
-import PyPDF2 as pypdf
 from .lib import fitz
 from .helpers import universal_open, fspath
 
-
-
-matrix_mult = pypdf.pdf.utils.matrixMultiply
+from pikepdf import PdfMatrix
+import pikepdf
 
 Colorspace = Enum('Colorspace',
                   'gray rgb cmyk lab icc index sep devn pattern jpeg2000')
@@ -118,24 +116,20 @@ XobjectSettings = namedtuple('XobjectSettings',
     ['name', 'shorthand', 'stack_depth'])
 
 InlineSettings = namedtuple('InlineSettings',
-    ['settings', 'shorthand', 'stack_depth'])
+    ['iimage', 'shorthand', 'stack_depth'])
 
-ContentsInfo = namedtuple('ContentsInfo', 
+ContentsInfo = namedtuple('ContentsInfo',
     ['xobject_settings', 'inline_images', 'found_text'])
 
 
+
 def _normalize_stack(operations):
-    """Fix runs of qQ's in the stack
-
-    For some reason PyPDF2 converts runs of qqq, QQ, QQQq, etc. into single
-    operations.  Break this silliness up and issue each stack operation
-    individually so we don't lose count.
-
-    """
+    """Convert runs of qQ's in the stack into single operations"""
     for operands, command in operations:
-        if re.match(br'Q*q+$', command):   # Zero or more Q, one or more q
-            for char in command:           # Split into individual bytes
-                yield ([], bytes([char]))  # Yield individual bytes
+        command = str(command)
+        if re.match(r'Q*q+$', command):   # Zero or more Q, one or more q
+            for char in command:          # Split into individual
+                yield ([], char)          # Yield individual
         else:
             yield (operands, command)
 
@@ -159,46 +153,46 @@ def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
 
     PDF units suit our needs so we initialize ctm to the identity matrix.
 
-    PyPDF2 replaces inline images with a fake "INLINE IMAGE" operator.
-
     """
 
-    operations = contentstream.operations
     stack = []
-    ctm = _matrix_from_shorthand(initial_shorthand)
+    ctm = PdfMatrix(initial_shorthand)
     xobject_settings = []
     inline_images = []
     found_text = False
+    text_operators = set(['Tj', 'TJ', '"', "'"])
+    operator_whitelist = """q Q Do cm TJ Tj " ' BI ID EI"""
 
-    for n, op in enumerate(_normalize_stack(operations)):
+    for n, op in enumerate(_normalize_stack(
+            pikepdf.parse_content_stream(contentstream, operator_whitelist))):
         operands, command = op
-        if command == b'q':
+
+        if command == 'q':
             stack.append(ctm)
             if len(stack) > 32:
                 raise RuntimeError(
                     "PDF graphics stack overflow, command %i" % n)
-        elif command == b'Q':
+        elif command == 'Q':
             try:
                 ctm = stack.pop()
             except IndexError:
                 raise RuntimeError(
                     "PDF graphics stack underflow, command %i" % n)
-        elif command == b'cm':
-            ctm = matrix_mult(
-                _matrix_from_shorthand(operands), ctm)
-        elif command == b'Do':
+        elif command == 'cm':
+            ctm = PdfMatrix(operands) @ ctm
+        elif command == 'Do':
             image_name = operands[0]
             settings = XobjectSettings(
-                name=image_name, shorthand=_shorthand_from_matrix(ctm),
+                name=image_name, shorthand=ctm.shorthand,
                 stack_depth=len(stack))
             xobject_settings.append(settings)
-        elif command == b'INLINE IMAGE':
-            settings = operands['settings']
+        elif command == 'INLINE IMAGE':
+            iimage = operands[0]
             inline = InlineSettings(
-                settings=settings, shorthand=_shorthand_from_matrix(ctm),
+                iimage=iimage, shorthand=ctm.shorthand,
                 stack_depth=len(stack))
             inline_images.append(inline)
-        elif command in (b'Tj', b'TJ', b'"', b"'"):
+        elif command in text_operators:
             found_text = True
 
 
@@ -278,73 +272,47 @@ class ImageInfo:
     def __init__(self, *, name='', pdfimage=None, inline=None,
                  shorthand=None):
 
-        self._name = name
+        self._name = str(name)
         self._shorthand = shorthand
-        if inline:
-            # Fixme does not work for inline images with non abbreviated
-            # fields
+
+        if inline is not None:
             self._origin = 'inline'
-            self._width = inline.settings['/W']
-            self._height = inline.settings['/H']
-            self._type = 'stencil' if inline.settings.get('/IM') else 'image'
-            default_bpc = 8 if self._type == 'image' else 1
-            self._bpc = inline.settings.get('/BPC', default_bpc)
-            try:
-                self._color = FRIENDLY_COLORSPACE[inline.settings['/CS']]
-            except Exception:
-                self._color = '-'
-            self._comp = FRIENDLY_COMP.get(self._color, '?')
-            if '/F' in inline.settings:
-                filter_ = inline.settings['/F']
-                if isinstance(filter_, pypdf.generic.ArrayObject):
-                    filter_ = filter_[0]
-                self._enc = FRIENDLY_ENCODING.get(filter_, 'image')
-            else:
-                self._enc = 'image'
-        elif pdfimage:
+            pim = inline.iimage
+        elif pdfimage is not None:
             self._origin = 'xobject'
-            self._width = pdfimage['/Width']
-            self._height = pdfimage['/Height']
+            pim = pikepdf.PdfImage(pdfimage)
+        self._width = pim.width
+        self._height = pim.height
 
-            # If /ImageMask is true, then this image is a stencil mask
-            # (Images that draw with this stencil mask will have a reference to
-            # it in their /Mask, but we don't actually need that information)
-            if '/ImageMask' in pdfimage:
-                self._type = 'stencil' if pdfimage['/ImageMask'].value \
-                    else 'image'
-            else:
-                self._type = 'image'
+        # If /ImageMask is true, then this image is a stencil mask
+        # (Images that draw with this stencil mask will have a reference to
+        # it in their /Mask, but we don't actually need that information)
+        if pim.image_mask:
+            self._type = 'stencil'
+        else:
+            self._type = 'image'
 
-            default_bpc = 8 if self._type == 'image' else 1
-            if '/BitsPerComponent' in pdfimage:
-                self._bpc = pdfimage['/BitsPerComponent']
-            else:
-                self._bpc = default_bpc
+        self._bpc = int(pim.bits_per_component)
+        try:
+            self._enc = FRIENDLY_ENCODING.get(pim.filters[0], 'image')
+        except IndexError as e:
+            self._enc = '?'
 
-            if '/Filter' in pdfimage:
-                filter_ = pdfimage['/Filter']
-                if isinstance(filter_, pypdf.generic.ArrayObject):
-                    filter_ = filter_[0]
-                self._enc = FRIENDLY_ENCODING.get(filter_, 'image')
-            else:
-                self._enc = 'image'
-            if '/ColorSpace' in pdfimage:
-                cs = pdfimage['/ColorSpace']
-                if isinstance(cs, pypdf.generic.ArrayObject):
-                    cs = cs[0]
-                self._color = FRIENDLY_COLORSPACE.get(cs, '-')
-            else:
-                self._color = FRIENDLY_COLORSPACE[Colorspace.jpeg2000] \
-                    if self._enc == Encoding.jpeg2000 else '?'
+        try:
+            self._color = FRIENDLY_COLORSPACE.get(pim.colorspace, '?')
+        except NotImplementedError:
+            self._color = '?'
+        if self._enc == Encoding.jpeg2000:
+            self._color = Colorspace.jpeg2000
 
-            self._comp = FRIENDLY_COMP.get(self._color, '?')
+        self._comp = FRIENDLY_COMP.get(self._color, '?')
 
-            # Bit of a hack... infer grayscale if component count is uncertain
-            # but encoding must be monochrome. This happens if a monochrome image
-            # has an ICC profile attached. Better solution would be to examine
-            # the ICC profile.
-            if self._comp == '?' and self._enc in (Encoding.ccitt, 'jbig2'):
-                self._comp = FRIENDLY_COMP[Colorspace.gray]
+        # Bit of a hack... infer grayscale if component count is uncertain
+        # but encoding must be monochrome. This happens if a monochrome image
+        # has an ICC profile attached. Better solution would be to examine
+        # the ICC profile.
+        if self._comp == '?' and self._enc in (Encoding.ccitt, 'jbig2'):
+            self._comp = FRIENDLY_COMP[Colorspace.gray]
 
     @property
     def name(self):
@@ -386,20 +354,6 @@ class ImageInfo:
     def yres(self):
         return _get_dpi(self._shorthand, (self._width, self._height))[1]
 
-    def __getitem__(self, item):
-        warnings.warn("ImageInfo.__getitem__", DeprecationWarning)
-        if item in ('name', 'width', 'height', 'bpc', 'color', 'comp', 'enc'):
-            return getattr(self, item)
-        elif item == 'dpi_w':
-            return Decimal(self.xres).quantize(self.DPI_PREC)
-        elif item == 'dpi_h':
-            return Decimal(self.yres).quantize(self.DPI_PREC)
-        elif item == 'dpi':
-            return Decimal(self.xres * self.yres).sqrt().quantize(
-                self.DPI_PREC)
-        else:
-            raise KeyError(item)
-
     def __repr__(self):
         class_locals = {attr: getattr(self, attr, None) for attr in dir(self)
                         if not attr.startswith('_')}
@@ -434,8 +388,9 @@ def _image_xobjects(container):
     resources = container['/Resources']
     if '/XObject' not in resources:
         return
-    for xobj in resources['/XObject']:
-        candidate = resources['/XObject'][xobj]
+    xobjs = resources['/XObject'].as_dict()
+    for xobj in xobjs:
+        candidate = xobjs[xobj]
         if candidate['/Subtype'] == '/Image':
             pdfimage = candidate
             yield (pdfimage, xobj)
@@ -482,8 +437,9 @@ def _find_form_xobject_images(pdf, container, contentsinfo):
     resources = container['/Resources']
     if '/XObject' not in resources:
         return
-    for xobj in resources['/XObject']:
-        candidate = resources['/XObject'][xobj]
+    xobjs = resources['/XObject'].as_dict()
+    for xobj in xobjs:
+        candidate = xobjs[xobj]
         if candidate['/Subtype'] != '/Form':
             continue
 
@@ -521,32 +477,26 @@ def _find_images(*, pdf, container, shorthand=None):
     """
 
     if container.get('/Type') == '/Page' and '/Contents' in container:
-        # For a /Page the content stream is attached to the page's /Contents
-        page = container
-        contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
         initial_shorthand = shorthand or UNIT_SQUARE
     elif container.get('/Type') == '/XObject' and \
             container['/Subtype'] == '/Form':
-        # For a Form XObject that content stream is attached to the XObject
-        contentstream = pypdf.pdf.ContentStream(container, pdf)
-
         # Set the CTM to the state it was when the "Do" operator was
         # encountered that is drawing this instance of the Form XObject
-        ctm = _matrix_from_shorthand(shorthand or UNIT_SQUARE)
+        ctm = PdfMatrix(shorthand) if shorthand else PdfMatrix.identity()
 
         # A Form XObject may provide its own matrix to map form space into
         # user space. Get this if one exists
-        form_matrix = _matrix_from_shorthand(
-                container.get('/Matrix', UNIT_SQUARE))
+        form_shorthand = container.get('/Matrix', PdfMatrix.identity())
+        form_matrix = PdfMatrix(form_shorthand)
 
         # Concatenate form matrix with CTM to ensure CTM is correct for
         # drawing this instance of the XObject
-        ctm = matrix_mult(form_matrix, ctm)
-        initial_shorthand = _shorthand_from_matrix(ctm)
+        ctm = form_matrix @ ctm
+        initial_shorthand = ctm.shorthand
     else:
         return
 
-    contentsinfo = _interpret_contents(contentstream, initial_shorthand)
+    contentsinfo = _interpret_contents(container, initial_shorthand)
 
     yield from _find_inline_images(contentsinfo)
     yield from _find_regular_images(container, contentsinfo)
@@ -558,9 +508,8 @@ def _naive_find_text(*, pdf, page):
         # Not a page, or has no /Contents => no text
         return False
 
-    # First we check the main content stream    
-    contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
-    contentsinfo = _interpret_contents(contentstream, UNIT_SQUARE)
+    # First we check the main content stream
+    contentsinfo = _interpret_contents(page, UNIT_SQUARE)
     if contentsinfo.found_text:
         return True
 
@@ -570,16 +519,16 @@ def _naive_find_text(*, pdf, page):
     # rare.
     if '/Resources' in page:
         resources = page['/Resources']
-        if '/XObject' in resources:    
-            for xobj in resources['/XObject']:
-                candidate = resources['/XObject'][xobj]
+        if '/XObject' in resources:
+            xobjs = resources['/XObject'].as_dict()
+            for xobj in xobjs:
+                candidate = xobjs[xobj]
                 if candidate['/Subtype'] != '/Form':
                     continue
-                form_xobject = candidate                
+                form_xobject = candidate
                 # Content stream is attached to Form XObject dictionary
-                contentstream = pypdf.pdf.ContentStream(form_xobject, pdf)
                 sub_contentsinfo = _interpret_contents(
-                    contentstream, UNIT_SQUARE)
+                    form_xobject, UNIT_SQUARE)
                 if sub_contentsinfo.found_text:
                     return True
     return False
@@ -604,7 +553,7 @@ def _page_get_textblocks(infile, pageno):
 
 def _page_has_text(text):
     "Smarter text detection that ignores text in margins"
-    
+
     pw, ph = text['width'], text['height']
 
     margin_ratio = 0.125
@@ -626,11 +575,6 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile):
     pageinfo['pageno'] = pageno
     pageinfo['images'] = []
 
-    if isinstance(pdf, Path):
-        pdf = pypdf.PdfFileReader(str(pdf))
-    elif isinstance(pdf, str):
-        pdf = pypdf.PdfFileReader(pdf)
-
     page = pdf.pages[pageno]
 
     if fitz:
@@ -639,10 +583,13 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile):
     else:
         pageinfo['has_text'] = _naive_find_text(pdf=pdf, page=page)
 
-    width_pt = page.mediaBox.getWidth()
-    height_pt = page.mediaBox.getHeight()
+    mediabox = [Decimal(d.decode()) for d in page.MediaBox.as_list()]
+    width_pt = mediabox[2] - mediabox[0]
+    height_pt = mediabox[3] - mediabox[1]
 
     userunit = page.get('/UserUnit', Decimal(1.0))
+    if not isinstance(userunit, Decimal):
+        userunit = userunit.decode()
     pageinfo['userunit'] = userunit
     pageinfo['width_inches'] = width_pt * userunit / Decimal(72.0)
     pageinfo['height_inches'] = height_pt * userunit / Decimal(72.0)
@@ -657,8 +604,8 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile):
                           _find_images(pdf=pdf, container=page,
                                        shorthand=userunit_shorthand)]
     if pageinfo['images']:
-        xres = max(image['dpi_w'] for image in pageinfo['images'])
-        yres = max(image['dpi_h'] for image in pageinfo['images'])
+        xres = Decimal(max(image.xres for image in pageinfo['images']))
+        yres = Decimal(max(image.yres for image in pageinfo['images']))
         pageinfo['xres'], pageinfo['yres'] = xres, yres
         pageinfo['width_pixels'] = \
             int(round(xres * pageinfo['width_inches']))
@@ -669,9 +616,8 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile):
 
 
 def _pdf_get_all_pageinfo(infile):
-    with universal_open(infile, 'rb') as f:
-        pdf = pypdf.PdfFileReader(f)
-        return [PageInfo(pdf, n, infile) for n in range(pdf.numPages)]
+    pdf = pikepdf.open(infile)
+    return [PageInfo(pdf, n, infile) for n in range(len(pdf.pages))]
 
 
 class PageInfo:
@@ -686,7 +632,7 @@ class PageInfo:
 
     @property
     def has_text(self):
-        return self._pageinfo['has_text']                
+        return self._pageinfo['has_text']
 
     @property
     def width_inches(self):
@@ -755,7 +701,7 @@ class PageInfo:
 
 class PdfInfo:
     """Get summary information about a PDF
-    
+
     """
     def __init__(self, infile):
         self._infile = infile

src/ocrmypdf/pipeline.py

@@ -26,7 +26,6 @@ import shutil
 import re
 
 import img2pdf
-import PyPDF2 as pypdf
 import pikepdf
 
 from PIL import Image
@@ -116,7 +115,7 @@ def triage_image_file(input_file, output_file, log, options):
 
 def _pdf_guess_version(input_file, search_window=1024):
     """Try to find version signature at start of file.
-    
+
     Not robust enough to deal with appended files.
 
     Returns empty string if not found, indicating file is probably not PDF.
@@ -175,7 +174,7 @@ def repair_and_parse_pdf(
             "output these files.)  Use --output-type=pdf instead."
         )
         raise InputFileError()
-    
+
     if len(pdfinfo.pages) > 2000 and sys.version_info[0:2] <= (3, 5):
         log.warning(
             "Performance regressions are known occur with Python 3.5 for "
@@ -242,7 +241,7 @@ def is_ocr_required(pageinfo, log, options):
         # We found a page with no images and no text. That means it may
         # have vector art that the user wants to OCR. If we determined
         # lossless reconstruction is not possible then we have to rasterize
-        # the image. So if OCR is being forced, take that to mean YES, go 
+        # the image. So if OCR is being forced, take that to mean YES, go
         # ahead and rasterize. If not forced, then pretend there's no text
         # on the page at all so we don't lose anything.
         # This could be made smarter by explicitly searching for vector art.
@@ -332,14 +331,14 @@ def ocr_or_skip(
     options = context.get_options()
     work_folder = context.get_work_folder()
     pdfinfo =  context.get_pdfinfo()
-    
+
     for input_file in input_files:
         pageno = page_number(input_file) - 1
         pageinfo = pdfinfo[pageno]
         alt_suffix = \
             '.ocr.page.pdf' if is_ocr_required(pageinfo, log, options) \
             else '.skip.page.pdf'
-        
+
         re_symlink(
             input_file,
             os.path.join(
@@ -540,14 +539,13 @@ def select_ocr_image(
         white = ImageColor.getcolor('#ffffff', im.mode)
         draw = ImageDraw.ImageDraw(im)
 
-        xres = im.width / pageinfo.width_inches
-        yres = im.height / pageinfo.height_inches
+        xres, yres = im.info['dpi']
         for textarea in pageinfo.get_textareas():
             # Calculate resolution based on the image size and page dimensions
             # without regard whatever resolution is in pageinfo (may differ or
             # be None)
             bbox = textarea['bbox']
-            log.debug('calculated resolution %r %r', xres, yres)
+            log.debug('resolution %r %r', xres, yres)
 
             pixcoords = [Decimal(bbox[0]) / Decimal(72) * xres,
                          Decimal(bbox[1]) / Decimal(72) * yres,
@@ -556,7 +554,7 @@ def select_ocr_image(
             pixcoords = [int(c) for c in pixcoords]
             log.debug('blanking %r', pixcoords)
             draw.rectangle(pixcoords, fill=white)
-        
+
         del draw
 
         # Pillow requires integer DPI
@@ -604,7 +602,7 @@ def select_visible_page_image(
 
     pageinfo = get_pageinfo(image, context)
     if pageinfo.images and \
-            all(im['enc'] == 'jpeg' for im in pageinfo.images):
+            all(im.enc == 'jpeg' for im in pageinfo.images):
         log.debug('{:4d}: JPEG input -> JPEG output'.format(
             page_number(image)))
         # If all images were JPEGs originally, produce a JPEG as output
@@ -716,7 +714,7 @@ def get_pdfmark(base_pdf, options):
         except (KeyError, TypeError):
             return ''
 
-    pdfmark = {k: from_document_info(k) for k in 
+    pdfmark = {k: from_document_info(k) for k in
         ('/Title', '/Author', '/Keywords', '/Subject', '/CreationDate')}
     if options.title:
         pdfmark['/Title'] = options.title
@@ -784,8 +782,8 @@ def metadata_fixup(
         metadata = pikepdf.open(metadata_file)
         pdfmark = get_pdfmark(metadata, options)
         pdf = pikepdf.open(layers_file)
-        pdf.metadata = pikepdf.Dictionary(pdfmark)
-        pdf.save(output_file)
+        pdf.metadata = pdf.make_indirect(pikepdf.Dictionary(pdfmark))
+        pdf.save(output_file, stream_data_mode=pikepdf.StreamDataMode.compress)
 
 
 def optimize_pdf(
@@ -817,7 +815,7 @@ def merge_sidecars(
             if txt_file:
                 with open(txt_file, 'r', encoding="utf-8") as in_:
                     txt = in_.read()
-                    # Tesseract v4 alpha started adding form feeds in 
+                    # Tesseract v4 alpha started adding form feeds in
                     # commit aa6eb6b
                     # No obvious way to detect what binaries will do this, so
                     # for consistency just ignore its form feeds and insert our
@@ -844,14 +842,14 @@ def copy_final(
         log,
         context):
     input_file = next((ii for ii in input_files if ii.endswith('.pdf')))
-
+    log.debug('%s -> %s', input_file, output_file)
     with open(input_file, 'rb') as input_stream:
         if output_file == '-':
             copyfileobj(input_stream, sys.stdout.buffer)
             sys.stdout.flush()
         else:
             # At this point we overwrite the output_file specified by the user
-            # use copyfileobj because then we use open() to create the file and 
+            # use copyfileobj because then we use open() to create the file and
             # get the appropriate umask, ownership, etc.
             with open(output_file, 'wb') as output_stream:
                 copyfileobj(input_stream, output_stream)
@@ -1028,7 +1026,7 @@ def build_pipeline(options, work_folder, log, context):
     task_metadata_fixup = main_pipeline.merge(
         task_func=metadata_fixup,
         input=[task_repair_and_parse_pdf,
-               task_weave_layers,               
+               task_weave_layers,
                task_generate_postscript_stub],
         output=os.path.join(work_folder, 'metafix.pdf'),
         extras=[log, context]

test_requirements.txt

@@ -4,4 +4,5 @@ pytest-xdist
 pytest-cov
 pytest-timeout
 python-xmp-toolkit   # requires apt-get install libexempi3
-                     # or brew install exempi
+                     # or brew install exempi
+PyPDF2 >= 1.26.0

tests/spoof/tesseract_badutf8.py

@@ -22,14 +22,13 @@
 
 import sys
 import img2pdf
-import PyPDF2 as pypdf
 from PIL import Image
 
 
 """Tesseract bad utf8 spoof
 
 In 'hocr' mode or 'pdf' mode, return error code 1 and some non-Unicode
-text because tesseract seems to do that in some cases related to 
+text because tesseract seems to do that in some cases related to
 language pack version mismatches
 
 """

tests/test_main.py

@@ -109,7 +109,7 @@ def test_deskew(spoof_tesseract_noop, resources, outdir):
         raster_device='pngmono',
         log=log,
         pageno=1)
-    
+
     pix = Pix.read(str(deskewed_png))
     skew_angle, skew_confidence = pix.find_skew()
 
@@ -220,8 +220,8 @@ def test_ocr_timeout(renderer, resources, outpdf):
 
 
 def test_skip_big(spoof_tesseract_cache, resources, outpdf):
-    out = check_ocrmypdf(resources / 'enormous.pdf', outpdf,
-                         '--skip-big', '10', env=spoof_tesseract_cache)
+    out = check_ocrmypdf(resources / 'jbig2.pdf', outpdf,
+                         '--skip-big', '1', env=spoof_tesseract_cache)
     pdfinfo = PdfInfo(out)
     assert not pdfinfo[0].has_text
 
@@ -383,14 +383,14 @@ def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error,
                                  resources, outpdf):
     check_ocrmypdf(
         resources / 'hugemono.pdf', outpdf, '-r',
-        '--pdf-renderer', renderer, 
+        '--pdf-renderer', renderer,
         '--max-image-mpixels', '0',
         env=spoof_tesseract_big_image_error)
 
 
-def test_algo4(resources, no_outpdf):
-    p, _, _ = run_ocrmypdf(resources / 'encrypted_algo4.pdf', no_outpdf)
-    assert p.returncode == ExitCode.encrypted_pdf
+def test_algo4(resources, spoof_tesseract_noop, outpdf):
+    p, _, _ = run_ocrmypdf(resources / 'encrypted_algo4.pdf', outpdf)
+    assert p.returncode == ExitCode.ok
 
 
 @pytest.mark.parametrize('renderer', RENDERERS)
@@ -439,8 +439,8 @@ def test_convert_to_square_resolution(renderer, spoof_tesseract_cache,
 
     # Because we rasterized the page to produce a new image, it should occupy
     # the entire page
-    out_im_w = out_p0.images[0]['width'] / out_p0.images[0]['dpi_w']
-    out_im_h = out_p0.images[0]['height'] / out_p0.images[0]['dpi_h']
+    out_im_w = out_p0.images[0].width / out_p0.images[0].xres
+    out_im_h = out_p0.images[0].height / out_p0.images[0].yres
     assert isclose(out_p0.width_inches, out_im_w)
     assert isclose(out_p0.height_inches, out_im_h)
 
@@ -651,7 +651,7 @@ def test_form_xobject(spoof_tesseract_noop, resources, outpdf):
 
 @pytest.mark.parametrize('renderer', RENDERERS)
 def test_pagesize_consistency(renderer, resources, outpdf):
-    
+
 
     first_page_dimensions = pytest.helpers.first_page_dimensions
 
@@ -893,10 +893,10 @@ def test_text_curves(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(
         resources / 'vector.pdf', outpdf, '--force-ocr',
         env=spoof_tesseract_noop)
-    
+
     info = PdfInfo(outpdf)
     assert len(info.pages[0].images) != 0, "force did not rasterize"
-    
+
 
 def test_dev_null(spoof_tesseract_noop, resources):
     p, out, err = run_ocrmypdf(

tests/test_optimize.py

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+from pathlib import Path
+
 import pytest
 
 import pikepdf
@@ -22,7 +24,9 @@ import pikepdf
 from ocrmypdf import _optimize as opt
 
 
-def test_multipage(resources, outpdf):
-    opt.main(resources / 'multipage.pdf', outpdf, level=3)
-
+@pytest.mark.parametrize('pdf', ['multipage.pdf', 'palette.pdf'])
+def test_basic(resources, pdf, outpdf):
+    infile = resources / pdf
+    opt.main(infile, outpdf, level=3)
 
+    assert Path(outpdf).stat().st_size <= Path(infile).stat().st_size

tests/test_pageinfo.py

@@ -28,6 +28,8 @@ import pytest
 import img2pdf
 import sys
 import PyPDF2 as pypdf
+import pikepdf
+import pickle
 
 
 def test_single_page_text(outdir):
@@ -127,8 +129,8 @@ def test_form_xobject(resources):
 
 def test_naive_find_text(resources):
     filename = resources / 'formxobject.pdf'
-    reader = pypdf.PdfFileReader(str(filename))
-    page = reader.getPage(0)
+    reader = pikepdf.open(filename)
+    page = reader.pages[0]
     assert pdfinfo._naive_find_text(pdf=reader, page=page)
 
 
@@ -144,3 +146,12 @@ def test_oversized_page(resources):
     pdf = pdfinfo.PdfInfo(resources / 'poster.pdf')
     image = pdf[0].images[0]
     assert image.width * image.xres > 200, "this is supposed to be oversized"
+
+
+def test_pickle(resources):
+    # For multiprocessing we must be able to pickle our information - if
+    # this fails then we are probably storing some unpickleabe pikepdf or
+    # other external data around
+    filename = resources / 'formxobject.pdf'
+    pdf = pdfinfo.PdfInfo(filename)
+    pickle.dumps(pdf)