v9.0.1 release notes

.docker/alpine.dockerfile

@@ -56,7 +56,7 @@ RUN \
   # Add runtime dependencies
   && apk add --update \
   python3 \
-  jbig2enc@testing \
+  jbig2enc@community \
   ghostscript \
   qpdf@community \
   qpdf-dev@community \

.travis.yml

@@ -1,7 +1,7 @@
 cache:
   pip: true
   directories:
-  - $HOME/Library/Caches/Homebrew
+    - $HOME/Library/Caches/Homebrew
 
 matrix:
   include:
@@ -12,27 +12,58 @@ matrix:
       python: "3.6"
       env:
         - DIST=trusty
-      addons: &trusty_apt
+        - MINIMAL=true
+      addons:
         apt:
           update: true
           sources:
-          - sourceline: 'ppa:alex-p/tesseract-ocr'
+            - sourceline: "ppa:alex-p/tesseract-ocr"
-          - sourceline: 'ppa:heyarje/libav-11'
+            - sourceline: "ppa:vshn/ghostscript"
-          - sourceline: 'ppa:vshn/ghostscript'
           packages:
-          - ghostscript
+            - ghostscript
-          - libavcodec56
+            - libffi-dev
-          - libavformat56
+            - qpdf
-          - libavutil54
+            - tesseract-ocr
-          - libexempi3
+            - tesseract-ocr-deu
-          - libffi-dev
+            - tesseract-ocr-eng
-          - pngquant
+            - tesseract-ocr-fra
-          - poppler-utils
+      before_install: |
-          - qpdf
+        pip3 install --upgrade pip
-          - tesseract-ocr
+        pip3 install --upgrade wheel
-          - tesseract-ocr-deu
+    - os: linux
-          - tesseract-ocr-eng
+      dist: trusty
-          - tesseract-ocr-fra
+      sudo: required
+      language: python
+      python: "3.6"
+      env:
+        - DIST=trusty
+      addons:
+        apt:
+          update: true
+          sources:
+            - sourceline: "ppa:alex-p/tesseract-ocr"
+            - sourceline: "ppa:heyarje/libav-11"
+            - sourceline: "ppa:vshn/ghostscript"
+          packages:
+            - ghostscript
+            - libavcodec56
+            - libavformat56
+            - libavutil54
+            - libffi-dev
+            - qpdf
+            - tesseract-ocr
+            - tesseract-ocr-deu
+            - tesseract-ocr-eng
+            - tesseract-ocr-fra
+            - libexempi3 # --- optional extras from here ---
+            - pngquant
+            - poppler-utils
+      before_install: |
+        mkdir -p bin packages
+        pip3 install --upgrade pip
+        pip3 install --upgrade wheel
+        wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
+        sudo dpkg -i packages/unpaper_6.1-1.deb
     - os: linux
       dist: xenial
       sudo: required
@@ -44,19 +75,19 @@ matrix:
         apt:
           update: true
           sources:
-          - sourceline: 'ppa:alex-p/tesseract-ocr'
+            - sourceline: "ppa:alex-p/tesseract-ocr"
           packages:
-          - ghostscript
+            - ghostscript
-          - libexempi3
+            - libexempi3
-          - libffi-dev
+            - libffi-dev
-          - pngquant
+            - pngquant
-          - poppler-utils
+            - poppler-utils
-          - qpdf
+            - qpdf
-          - tesseract-ocr
+            - tesseract-ocr
-          - tesseract-ocr-deu
+            - tesseract-ocr-deu
-          - tesseract-ocr-eng
+            - tesseract-ocr-eng
-          - tesseract-ocr-fra
+            - tesseract-ocr-fra
-          - unpaper
+            - unpaper
     - os: osx
       osx_image: xcode9.2
       language: generic
@@ -74,59 +105,25 @@ matrix:
             - qpdf
             - tesseract
             - unpaper
-    - os: osx
+      before_install: |
-      osx_image: xcode9.2
+        pip3 install --upgrade pip
-      language: generic
+        pip3 install wheel
-      env:
-        - ADD_PDFMINER=1
-      addons:
-        homebrew:
-          update: true
-          packages:
-            - exempi
-            - ghostscript
-            - jbig2enc
-            - leptonica
-            - openjpeg
-            - pngquant
-            - python
-            - qpdf
-            - tesseract
-            - unpaper
 
 before_cache:
-- rm -f $HOME/.cache/pip/log/debug.log
+  - rm -f $HOME/.cache/pip/log/debug.log
-
-before_install: |
-  mkdir -p bin
-  if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-    pip3 install --upgrade pip
-    pip3 install --upgrade wheel
-    if [[ "$DIST" == "trusty" ]]; then
-      mkdir -p packages
-      wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
-      sudo dpkg -i packages/unpaper_6.1-1.deb
-    fi
-  elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    pip3 install --upgrade pip
-    pip3 install wheel
-  fi
 
 install:
-- export PATH=$PWD/bin:$PATH
+  - mkdir -p bin
-- pip3 install pycparser  # py3.7 workaround for https://github.com/eliben/pycparser/issues/251
+  - export PATH=$PWD/bin:$PATH
-- pip3 install -r requirements/main.txt
+  - pip3 install pycparser # py3.7 workaround for https://github.com/eliben/pycparser/issues/251
-- pip3 install --no-deps .
+  - pip3 install -r requirements/main.txt
-- |
+  - pip3 install --no-deps .
-  if [[ "$ADD_PDFMINER" == "1" ]]; then
+  - pip3 install -r requirements/test.txt
-    pip3 install --no-deps .[pdfminer]
-  fi
-- pip3 install -r requirements/test.txt
 
 script:
-- tesseract --version
+  - tesseract --version
-- qpdf --version
+  - qpdf --version
-- pytest -n auto
+  - pytest -n auto
 
 deploy:
   # release for main pypi
@@ -134,13 +131,13 @@ deploy:
   # a race and all versions will try to deploy
   # OTOH if we ever need separate binary wheels then each version needs its
   # own deploy
-- provider: pypi
+  - provider: pypi
-  user: ocrmypdf-travis
+    user: ocrmypdf-travis
-  password:
+    password:
-    secure: "DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo="
+      secure: "DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo="
-  distributions: "sdist bdist_wheel"
+    distributions: "sdist bdist_wheel"
-  on:
+    on:
-    branch: master
+      branch: master
-    tags: true
+      tags: true
-    condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux"
+      condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux"
-  skip_upload_docs: true
+    skip_upload_docs: true

docs/installation.rst

@@ -87,7 +87,6 @@ Fedora 29 or newer
 .. |fedora-rawhide| image:: https://repology.org/badge/version-for-repo/fedora_rawhide/ocrmypdf.svg
     :alt: Fedore Rawhide
 
-
 +------------------------------+
 | **OCRmyPDF version**         |
 +------------------------------+
@@ -403,14 +402,19 @@ The command line program should now be available:
 Installing on FreeBSD
 =====================
 
-FreeBSD 11.2 is known to work. Other versions likely work but have not
+.. image:: https://repology.org/badge/version-for-repo/freebsd/python:ocrmypdf.svg
-been tested.
+    :alt: FreeBSD
+    :target: https://repology.org/project/python:ocrmypdf/versions
 
-In general it should work to:
+FreeBSD 11.2, 11.3, 12.0-RELEASE and 13.0-CURRENT are supported. Other
+versions likely work but have not been tested.
 
-#. `Install and build
+.. code-block:: bash
-   pikepdf <https://pikepdf.readthedocs.io/en/latest/installation.html#installing-on-freebsd-11-2>`__.
+
-#. Install the equivalent list of dependencies for Linux.
+    pkg install py36-ocrmypdf
+
+To install a more recent version, you could attempt to first install the system
+version with ``pkg``, then use ``pip install --user ocrmypdf``.
 
 Installing the Docker image
 ===========================

docs/release_notes.rst

@@ -13,6 +13,15 @@ Note that it is licensed under GPLv3, so scripts that
 ``import ocrmypdf`` and are released publicly should probably also be
 licensed under GPLv3.
 
+v9.0.1
+======
+
+-  Fixed test suite failing when either of optional dependencies unpaper and
+   pngquant were missing.
+-  Fixed Alpine Docker image build.
+-  Documented that FreeBSD ports are now available.
+-  Changed to pikepdf 1.6.1 (also for Alpine Docker).
+
 v9.0.0
 ======
 
@@ -59,6 +68,8 @@ v9.0.0
 -  Updated Docker images to use newer versions.
 -  Fixed images encoded as JBIG2 with a colorspace other than ``/DeviceGray``
    were not interpreted correctly.
+-  Fixed a OCR text-image registration (i.e. alignment) problem when the page
+   when MediaBox had a nonzero corner.
 
 v8.3.2
 ======

requirements/main.txt

@@ -5,7 +5,7 @@ chardet == 3.0.4
 cffi == 1.12.2
 img2pdf == 0.3.3
 pdfminer.six == 20181108
-pikepdf == 1.6.0
+pikepdf == 1.6.1
 Pillow >= 5.0.0, != 5.1.0 ; sys_platform == "darwin"
 pycparser == 2.19
 python-xmp-toolkit == 2.0.1

src/ocrmypdf/_pipeline.py

@@ -129,7 +129,7 @@ def triage(input_file, output_file, options, log):
         if _pdf_guess_version(input_file):
             if options.image_dpi:
                 log.warning(
-                    "Argument --image-dpi ignored because the "
+                    "Argument --image-dpi is being ignored because the "
                     "input file is a PDF, not an image."
                 )
             # Origin file is a pdf create a symlink with pdf extension

tests/conftest.py

@@ -176,10 +176,9 @@ def no_outpdf(tmp_path):
 def check_ocrmypdf(input_file, output_file, *args, env=None):
     """Run ocrmypdf and confirmed that a valid file was created"""
 
-    # p, out, err = run_ocrmypdf(input_file, output_file, *args, env=env)
-
     options = cli.parser.parse_args(
-        [str(input_file), str(output_file)] + [str(arg) for arg in args]
+        [str(input_file), str(output_file)]
+        + [str(arg) for arg in args if arg is not None]
     )
     api.check_options(options)
     if env:

tests/test_main.py

@@ -1,4 +1,4 @@
-# © 2015-17 James R. Barlow: github.com/jbarlow83
+# © 2015-19 James R. Barlow: github.com/jbarlow83
 #
 # This file is part of OCRmyPDF.
 #
@@ -18,10 +18,9 @@
 import logging
 import os
 import shutil
-import sys
 from math import isclose
 from pathlib import Path
-from subprocess import DEVNULL, PIPE, run, Popen
+from subprocess import PIPE, run
 
 import PIL
 import pytest
@@ -84,11 +83,6 @@ def spoof_no_tess_gs_raster_fail(tmp_path_factory):
     )
 
 
-@pytest.fixture(scope='session')
-def spoof_tess_bad_utf8(tmp_path_factory):
-    return spoof(tmp_path_factory, tesseract='tesseract_badutf8.py')
-
-
 def test_quick(spoof_tesseract_cache, resources, outpdf):
     check_ocrmypdf(resources / 'ccitt.pdf', outpdf, env=spoof_tesseract_cache)
 
@@ -550,68 +544,6 @@ def test_jbig2_passthrough(spoof_tesseract_cache, resources, outpdf):
     assert out_pageinfo[0].images[0].enc == Encoding.jbig2
 
 
-def test_stdin(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
-    input_file = str(resources / 'francais.pdf')
-    output_file = str(outpdf)
-
-    # Runs: ocrmypdf - output.pdf < testfile.pdf
-    with open(input_file, 'rb') as input_stream:
-        p_args = ocrmypdf_exec + ['-', output_file]
-        p = run(
-            p_args,
-            stdout=PIPE,
-            stderr=PIPE,
-            stdin=input_stream,
-            env=spoof_tesseract_noop,
-        )
-        assert p.returncode == ExitCode.ok
-
-
-def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
-    input_file = str(resources / 'francais.pdf')
-    output_file = str(outpdf)
-
-    # Runs: ocrmypdf francais.pdf - > test_stdout.pdf
-    with open(output_file, 'wb') as output_stream:
-        p_args = ocrmypdf_exec + [input_file, '-']
-        p = run(
-            p_args,
-            stdout=output_stream,
-            stderr=PIPE,
-            stdin=DEVNULL,
-            env=spoof_tesseract_noop,
-        )
-        assert p.returncode == ExitCode.ok
-
-    assert qpdf.check(output_file, log=None)
-
-
-@pytest.mark.skipif(
-    sys.version_info[0:3] >= (3, 6, 4), reason="issue fixed in Python 3.6.4"
-)
-def test_closed_streams(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
-    input_file = str(resources / 'francais.pdf')
-    output_file = str(outpdf)
-
-    def evil_closer():
-        os.close(0)
-        os.close(1)
-
-    p_args = ocrmypdf_exec + [input_file, output_file]
-    p = Popen(  # pylint: disable=subprocess-popen-preexec-fn
-        p_args,
-        close_fds=True,
-        stdout=None,
-        stderr=PIPE,
-        stdin=None,
-        env=spoof_tesseract_noop,
-        preexec_fn=evil_closer,
-    )
-    out, err = p.communicate()
-    print(err.decode())
-    assert p.returncode == ExitCode.ok
-
-
 def test_masks(spoof_tesseract_noop, resources, outpdf):
     assert (
         ocrmypdf.ocr(
@@ -993,36 +925,6 @@ def test_pdfa_n(spoof_tesseract_cache, pdfa_level, resources, outpdf):
     assert pdfa_info['conformance'] == f'PDF/A-{pdfa_level}B'
 
 
-@pytest.mark.skipif(sys.version_info >= (3, 7, 0), reason='better utf-8')
-@pytest.mark.skipif(
-    Path('/etc/alpine-release').exists(), reason="invalid test on alpine"
-)
-def test_bad_locale():
-    env = os.environ.copy()
-    env['LC_ALL'] = 'C'
-
-    p, out, err = run_ocrmypdf('a', 'b', env=env)
-    assert out == '', "stdout not clean"
-    assert p.returncode != 0
-    assert 'configured to use ASCII as encoding' in err, "should whine"
-
-
-@pytest.mark.parametrize('renderer', RENDERERS)
-def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf):
-    p, out, err = run_ocrmypdf(
-        resources / 'ccitt.pdf',
-        no_outpdf,
-        '--pdf-renderer',
-        renderer,
-        env=spoof_tess_bad_utf8,
-    )
-
-    assert out == '', "stdout not clean"
-    assert p.returncode != 0
-    assert 'not utf-8' in err, "should whine about utf-8"
-    assert '\\x96' in err, 'should repeat backslash encoded output'
-
-
 @pytest.mark.skipif(
     PIL.__version__ < '5.0.0', reason="Pillow < 5.0.0 doesn't raise the exception"
 )
@@ -1051,14 +953,6 @@ def test_text_curves(spoof_tesseract_noop, resources, outpdf):
     assert len(info.pages[0].images) != 0, "force did not rasterize"
 
 
-def test_dev_null(spoof_tesseract_noop, resources):
-    p, out, err = run_ocrmypdf(
-        resources / 'trivial.pdf', os.devnull, '--force-ocr', env=spoof_tesseract_noop
-    )
-    assert p.returncode == 0, "could not send output to /dev/null"
-    assert len(out) == 0, "wrote to stdout"
-
-
 def test_output_is_dir(spoof_tesseract_noop, resources, outdir):
     p, out, err = run_ocrmypdf(
         resources / 'trivial.pdf', outdir, '--force-ocr', env=spoof_tesseract_noop
@@ -1125,3 +1019,14 @@ def test_fast_web_view(
     )
     with pikepdf.open(outpdf) as pdf:
         assert pdf.is_linearized == expected
+
+
+def test_image_dpi_not_image(caplog, spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(
+        resources / 'trivial.pdf',
+        outpdf,
+        '--image-dpi',
+        '100',
+        env=spoof_tesseract_noop,
+    )
+    assert '--image-dpi is being ignored' in caplog.text

tests/test_optimize.py

@@ -55,6 +55,7 @@ def test_mono_not_inverted(resources, outdir):
     assert im.getpixel((0, 0)) == 255, "Expected white background"
 
 
+@pytest.mark.skipif(not pngquant.available(), reason='need pngquant')
 def test_jpg_png_params(resources, outpdf, spoof_tesseract_noop):
     check_ocrmypdf(
         resources / 'crom.png',

tests/test_stdio.py

@@ -0,0 +1,137 @@
+# © 2019 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+from pathlib import Path
+from subprocess import DEVNULL, PIPE, run, Popen
+
+import pytest
+
+from ocrmypdf.exceptions import ExitCode
+from ocrmypdf.exec import qpdf
+
+# pytest.helpers is dynamic
+# pylint: disable=no-member,redefined-outer-name
+
+run_ocrmypdf = pytest.helpers.run_ocrmypdf
+spoof = pytest.helpers.spoof
+
+
+@pytest.fixture(scope='session')
+def spoof_tess_bad_utf8(tmp_path_factory):
+    return spoof(tmp_path_factory, tesseract='tesseract_badutf8.py')
+
+
+def test_stdin(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
+    input_file = str(resources / 'francais.pdf')
+    output_file = str(outpdf)
+
+    # Runs: ocrmypdf - output.pdf < testfile.pdf
+    with open(input_file, 'rb') as input_stream:
+        p_args = ocrmypdf_exec + ['-', output_file]
+        p = run(
+            p_args,
+            stdout=PIPE,
+            stderr=PIPE,
+            stdin=input_stream,
+            env=spoof_tesseract_noop,
+        )
+        assert p.returncode == ExitCode.ok
+
+
+def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
+    input_file = str(resources / 'francais.pdf')
+    output_file = str(outpdf)
+
+    # Runs: ocrmypdf francais.pdf - > test_stdout.pdf
+    with open(output_file, 'wb') as output_stream:
+        p_args = ocrmypdf_exec + [input_file, '-']
+        p = run(
+            p_args,
+            stdout=output_stream,
+            stderr=PIPE,
+            stdin=DEVNULL,
+            env=spoof_tesseract_noop,
+        )
+        assert p.returncode == ExitCode.ok
+
+    assert qpdf.check(output_file, log=None)
+
+
+@pytest.mark.skipif(
+    sys.version_info[0:3] >= (3, 6, 4), reason="issue fixed in Python 3.6.4"
+)
+def test_closed_streams(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
+    input_file = str(resources / 'francais.pdf')
+    output_file = str(outpdf)
+
+    def evil_closer():
+        os.close(0)
+        os.close(1)
+
+    p_args = ocrmypdf_exec + [input_file, output_file]
+    p = Popen(  # pylint: disable=subprocess-popen-preexec-fn
+        p_args,
+        close_fds=True,
+        stdout=None,
+        stderr=PIPE,
+        stdin=None,
+        env=spoof_tesseract_noop,
+        preexec_fn=evil_closer,
+    )
+    out, err = p.communicate()
+    print(err.decode())
+    assert p.returncode == ExitCode.ok
+
+
+@pytest.mark.skipif(sys.version_info >= (3, 7, 0), reason='better utf-8')
+@pytest.mark.skipif(
+    Path('/etc/alpine-release').exists(), reason="invalid test on alpine"
+)
+def test_bad_locale():
+    env = os.environ.copy()
+    env['LC_ALL'] = 'C'
+
+    p, out, err = run_ocrmypdf('a', 'b', env=env)
+    assert out == '', "stdout not clean"
+    assert p.returncode != 0
+    assert 'configured to use ASCII as encoding' in err, "should whine"
+
+
+@pytest.mark.parametrize('renderer', ['hocr', 'sandwich'])
+def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / 'ccitt.pdf',
+        no_outpdf,
+        '--pdf-renderer',
+        renderer,
+        env=spoof_tess_bad_utf8,
+    )
+
+    assert out == '', "stdout not clean"
+    assert p.returncode != 0
+    assert 'not utf-8' in err, "should whine about utf-8"
+    assert '\\x96' in err, 'should repeat backslash encoded output'
+
+
+def test_dev_null(spoof_tesseract_noop, resources):
+    p, out, err = run_ocrmypdf(
+        resources / 'trivial.pdf', os.devnull, '--force-ocr', env=spoof_tesseract_noop
+    )
+    assert p.returncode == 0, "could not send output to /dev/null"
+    assert len(out) == 0, "wrote to stdout"