v9.8.0 release notes

Output not guaranteed to be UTF-8.

Fixes #543.

azure-pipelines.yml

@@ -153,12 +153,13 @@ stages:
           matrix:
             Python37:
               python.version: "3.7"
-            Python38:
-              python.version: "3.8"
+            # Python38:
+            #   python.version: "3.8"
         steps:
-          - task: UsePythonVersion@0
-            inputs:
-              versionSpec: "$(python.version)"
+          # https://github.com/actions/virtual-environments/issues/664
+          # - task: UsePythonVersion@0
+          #   inputs:
+          #     versionSpec: "$(python.version)"
           - bash: |
               brew update
               brew unlink python@2
@@ -169,6 +170,7 @@ stages:
                 leptonica \
                 openjpeg \
                 pngquant \
+                python \
                 tesseract \
                 unpaper
             displayName: "Install system packages"

docs/installation.rst

@@ -137,11 +137,12 @@ Installing the latest version on Ubuntu 18.04 LTS
 -------------------------------------------------
 
 Ubuntu 18.04 includes ocrmypdf 6.1.2 - you can install that with ``apt``, but
-it is quite old now. To install a more recent version, first install several
-system dependencies:
+it is quite old now. To install a more recent version, uninstall the old version
+of ocrmypdf, and install the following dependencies:
 
 .. code-block:: bash
 
+    sudo apt-get -y remove ocrmypdf
     sudo apt-get -y update
     sudo apt-get -y install \
         ghostscript \

docs/release_notes.rst

@@ -13,6 +13,14 @@ Note that it is licensed under GPLv3, so scripts that
 ``import ocrmypdf`` and are released publicly should probably also be
 licensed under GPLv3.
 
+v9.8.0
+======
+
+-  Fixed issue where only the first PNG (FlateDecode) image in a file would be
+   considered for optimization. File sizes should be improved from here on.
+-  Fixed a startup crash when the chosen language was Japanese (#543).
+-  Added options to configure polling and log level to watcher.py.
+
 v9.7.2
 ======
 

misc/watcher.py

@@ -25,6 +25,7 @@ from pathlib import Path
 import pikepdf
 from watchdog.events import PatternMatchingEventHandler
 from watchdog.observers import Observer
+from watchdog.observers.polling import PollingObserver
 
 import ocrmypdf
 
@@ -37,7 +38,8 @@ ON_SUCCESS_DELETE = bool(os.getenv('OCR_ON_SUCCESS_DELETE', False))
 DESKEW = bool(os.getenv('OCR_DESKEW', False))
 OCR_JSON_SETTINGS = json.loads(os.getenv('OCR_JSON_SETTINGS', '{}'))
 POLL_NEW_FILE_SECONDS = os.getenv('OCR_POLL_NEW_FILE_SECONDS', 1)
-LOGLEVEL = os.environ.get('OCR_LOGLEVEL', 'INFO').upper()
+USE_POLLING = bool(os.getenv('OCR_USE_POLLING', False))
+LOGLEVEL = os.getenv('OCR_LOGLEVEL', 'INFO').upper()
 PATTERNS = ['*.pdf']
 
 log = logging.getLogger('ocrmypdf-watcher')
@@ -112,6 +114,7 @@ def main():
     ocrmypdf.configure_logging(
         verbosity=ocrmypdf.Verbosity.default, manage_root_logger=True
     )
+    log.setLevel(LOGLEVEL)
     log.info(
         f"Starting OCRmyPDF watcher with config:\n"
         f"Input Directory: {INPUT_DIRECTORY}\n"
@@ -126,6 +129,7 @@ def main():
         f"DESKEW: {DESKEW}\n"
         f"ARGS: {OCR_JSON_SETTINGS}\n"
         f"POLL_NEW_FILE_SECONDS: {POLL_NEW_FILE_SECONDS}\n"
+        f"USE_POLLING: {USE_POLLING}\n"
         f"LOGLEVEL: {LOGLEVEL}\n"
     )
 
@@ -134,7 +138,10 @@ def main():
         sys.exit(1)
 
     handler = HandleObserverEvent(patterns=PATTERNS)
-    observer = Observer()
+    if USE_POLLING:
+        observer = PollingObserver()
+    else:
+        observer = Observer()
     observer.schedule(handler, INPUT_DIRECTORY, recursive=True)
     observer.start()
     try:

requirements/dev.txt

@@ -1,2 +0,0 @@
-twine >= 1.8.1
-coverage >= 4.5

requirements/main.txt

@@ -2,9 +2,9 @@
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
 cffi == 1.14.0
-img2pdf == 0.3.3
+img2pdf == 0.3.4
 pdfminer.six == 20200402
-pikepdf == 1.10.2
-Pillow == 7.0.0
+pikepdf == 1.11.1
+Pillow == 7.1.1
 reportlab == 3.5.34
-tqdm == 4.42.1
+tqdm == 4.45.0

src/ocrmypdf/exec/tesseract.py

@@ -77,20 +77,14 @@ def has_textonly_pdf(tesseract_env=None, langs=None):
     args_tess = tess_base_args(langs, engine_mode=None) + ['--print-parameters', 'pdf']
     params = ''
     try:
-        proc = run(
-            args_tess,
-            check=True,
-            universal_newlines=True,
-            stdout=PIPE,
-            stderr=STDOUT,
-            env=tesseract_env,
-        )
+        # print-parameters can return non-UTF8 if the parameters are so initialized
+        proc = run(args_tess, check=True, stdout=PIPE, stderr=STDOUT, env=tesseract_env)
         params = proc.stdout
     except CalledProcessError as e:
         raise MissingDependencyError(
             "Could not --print-parameters from tesseract"
         ) from e
-    if 'textonly_pdf' in params:
+    if b'textonly_pdf' in params:
         return True
     return False
 

src/ocrmypdf/optimize.py

@@ -421,9 +421,9 @@ def transcode_pngs(pike, images, image_name_fn, root, log, options):
             )
             continue
         if compdata.type == leptonica.lept.L_FLATE_ENCODE:
-            return rewrite_png(pike, im_obj, compdata, log)
+            rewrite_png(pike, im_obj, compdata, log)
         elif compdata.type == leptonica.lept.L_G4_ENCODE:
-            return rewrite_png_as_g4(pike, im_obj, compdata, log)
+            rewrite_png_as_g4(pike, im_obj, compdata, log)
 
 
 def rewrite_png_as_g4(pike, im_obj, compdata, log):

tests/spoof/tesseract_badutf8.py

@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-# © 2017 James R. Barlow: github.com/jbarlow83
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-import sys
-
-"""Tesseract bad utf8 spoof
-
-In 'hocr' mode or 'pdf' mode, return error code 1 and some non-Unicode
-text because tesseract seems to do that in some cases related to
-language pack version mismatches
-
-"""
-
-
-VERSION_STRING = '''tesseract 4.0.0
- leptonica-1.77.0
-  libjpeg 9c : libpng 1.6.35 : libtiff 4.0.10 : zlib 1.2.11 : libopenjp2 2.3.0
- Found AVX2
- Found AVX
- Found SSE
-SPOOFED
-'''
-
-# Japanese "Invalid UTF-8" encoded in Shift JIS
-BAD_UTF8 = b'\x96\xb3\x8c\xf8\x82\xc8UTF-8\x0a'
-
-
-def main():
-    if sys.argv[1] == '--version':
-        print(VERSION_STRING, file=sys.stderr)
-        sys.exit(0)
-    elif sys.argv[1] == '--list-langs':
-        print('List of available languages (1):\neng', file=sys.stderr)
-        sys.exit(0)
-    elif sys.argv[-2] == '--print-parameters':
-        print("Some parameters", file=sys.stderr)
-        print("textonly_pdf\t1\tSome help text")
-        sys.exit(0)
-    elif sys.argv[-2] in ('hocr', 'pdf'):
-        sys.stdout.buffer.write(BAD_UTF8)
-        sys.exit(1)
-    elif sys.argv[-1] == 'stdout':
-        # input file is at sys.argv[-2] but we don't look at it
-        print(
-            """Orientation: 0
-Orientation in degrees: 0
-Orientation confidence: 100.00
-Script: 1
-Script confidence: 100.00""",
-            file=sys.stderr,
-        )
-    else:
-        print("Spoof doesn't understand arguments", file=sys.stderr)
-        print(sys.argv, file=sys.stderr)
-        sys.exit(1)
-
-    sys.exit(0)
-
-
-if __name__ == '__main__':
-    main()

tests/test_optimize.py

@@ -18,10 +18,12 @@
 import logging
 from os import fspath
 from pathlib import Path
+from unittest.mock import patch
 
+import img2pdf
 import pikepdf
 import pytest
-from PIL import Image
+from PIL import Image, ImageDraw
 
 from ocrmypdf import optimize as opt
 from ocrmypdf.exec import jbig2enc, pngquant
@@ -130,3 +132,41 @@ def test_flate_to_jbig2(resources, outdir, spoof_tesseract_noop):
     pdf = pikepdf.open(outdir / 'out.pdf')
     pim = pikepdf.PdfImage(next(iter(pdf.pages[0].images.values())))
     assert pim.filters[0] == '/JBIG2Decode'
+
+
+def test_multiple_pngs(resources, outdir, spoof_tesseract_noop):
+    with Path.open(outdir / 'in.pdf', 'wb') as inpdf:
+        img2pdf.convert(
+            fspath(resources / 'baiona_colormapped.png'),
+            fspath(resources / 'baiona_gray.png'),
+            with_pdfrw=False,
+            outputstream=inpdf,
+        )
+
+    def mockquant(input_file, output_file, _quality_min, _quality_max):
+        with Image.open(input_file) as im:
+            draw = ImageDraw.Draw(im)
+            draw.rectangle((0, 0, im.width, im.height), fill=128)
+            im.save(output_file)
+
+    with patch('ocrmypdf.optimize.pngquant.quantize', new=mockquant):
+        check_ocrmypdf(
+            outdir / 'in.pdf',
+            outdir / 'out.pdf',
+            '--optimize',
+            '3',
+            '--jobs',
+            '1',
+            '--use-threads',
+            '--output-type',
+            'pdf',
+            env=spoof_tesseract_noop,
+        )
+
+    with pikepdf.open(outdir / 'in.pdf') as inpdf, pikepdf.open(
+        outdir / 'out.pdf'
+    ) as outpdf:
+        for n in range(len(inpdf.pages)):
+            inim = next(iter(inpdf.pages[n].images.values()))
+            outim = next(iter(outpdf.pages[n].images.values()))
+            assert len(outim.read_raw_bytes()) < len(inim.read_raw_bytes()), n

tests/test_stdio.py

@@ -33,11 +33,6 @@ run_ocrmypdf_api = pytest.helpers.run_ocrmypdf
 spoof = pytest.helpers.spoof
 
 
-@pytest.fixture
-def spoof_tess_bad_utf8(tmp_path_factory):
-    return spoof(tmp_path_factory, tesseract='tesseract_badutf8.py')
-
-
 def test_stdin(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
     input_file = str(resources / 'francais.pdf')
     output_file = str(outpdf)