Fix inverted colors during JBIG2 optimization on paletted images

Fixes #640

docs/installation.rst

@@ -190,7 +190,8 @@ of ocrmypdf, and install the following dependencies:
         python3-reportlab \
         qpdf \
         tesseract-ocr \
-        zlib1g
+        zlib1g \
+        unpaper
 
 We will need a newer version of ``pip`` then was available for Ubuntu 18.04:
 

docs/release_notes.rst

@@ -12,12 +12,42 @@ may be unreliable. Use the API to depend on precise behavior.
 The public API may be useful in scripts that launch OCRmyPDF processes or that
 wish to use some of its features for working with PDFs.
 
+v11.2.1
+=======
+
+-  Fixed an issue where optimization of a 1-bit image with a color palette or
+   associated ICC that was optimized to JBIG2 could have its colors inverted.
+
+v11.2.0
+=======
+
+-  Fixed an issue with optimizing PNG-type images that had soft masks or image masks.
+   This is a regression introduced in (or about) v11.1.0.
+-  Improved type checking of the ``plugins`` parameter for the ``ocrmypdf.ocr``
+   API call.
+
+v11.1.2
+=======
+
+-  Fixed hOCR renderer writing the text in roughly reverse order. This should not
+   affect reasonably smart PDF readers that properly locate the position of all
+   text, but may confuse those that rely on the order of objects in the content
+   stream. (#642)
+
+v11.1.1
+=======
+
+-  We now avoid using named temporary files when using pngquant allowing containerized
+   pngquant installs to be used.
+-  Clarified an error message.
+-  Highest number of 1's in a release ever!
+
 v11.1.0
 =======
 
 -  Fixed page rotation issues: #634, #589.
 -  Fixed some cases where optimization created an invalid image such as a
-   1-bit "RGB" iamge: #629, #620.
+   1-bit "RGB" image: #629, #620.
 -  Page numbers are now displayed in debug logs when pages are being grafted.
 -  ocrmypdf.optimize.rewrite_png and ocrmypdf.optimize.rewrite_png_as_g4 were
    marked deprecated. Strictly speaking these should have been internal APIs,

misc/example_plugin.py

@@ -18,6 +18,25 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+"""
+An example of an OCRmyPDF plugin.
+
+This plugin adds two new command line arguments
+    --grayscale-ocr: converts the image to grayscale before performing OCR on it
+        (This is occasionally useful for images whose color confounds OCR. It only
+        affects the image shown to OCR. The image is not saved.)
+    --mono-page: converts pages all pages in the output file to black and white
+
+To use this from the command line:
+    ocrmypdf --plugin path/to/example_plugin.py --mono-page input.pdf output.pdf
+
+To use this as an API:
+    import ocrmypdf
+    ocrmypdf.ocr('input.pdf', 'output.pdf',
+        plugins=['path/to/example_plugin.py'], mono_page=True
+    )
+"""
+
 import logging
 
 from PIL import Image

src/ocrmypdf/_exec/pngquant.py

@@ -7,7 +7,11 @@
 
 """Interface to pngquant executable"""
 
+from contextlib import contextmanager
+from io import BytesIO
 from os import fspath
+from pathlib import Path
+from subprocess import PIPE
 from tempfile import NamedTemporaryFile
 
 from PIL import Image
@@ -28,34 +32,32 @@ def available():
     return True
 
 
-def quantize(input_file, output_file, quality_min, quality_max):
-    input_file = fspath(input_file)
-    output_file = fspath(output_file)
-    if input_file.endswith('.jpg'):
-        with Image.open(input_file) as im, NamedTemporaryFile(suffix='.png') as tmp:
-            im.save(tmp)
-            args = [
-                'pngquant',
-                '--force',
-                '--skip-if-larger',
-                '--output',
-                output_file,
-                '--quality',
-                f'{quality_min}-{quality_max}',
-                '--',
-                tmp.name,
-            ]
-            run(args)
+@contextmanager
+def input_as_png(input_file: Path):
+    if not input_file.name.endswith('.png'):
+        with Image.open(input_file) as im:
+            bio = BytesIO()
+            im.save(bio, format='png')
+            bio.seek(0)
+            yield bio
     else:
+        with open(input_file, 'rb') as f:
+            yield f
+
+
+def quantize(input_file: Path, output_file: Path, quality_min: int, quality_max: int):
+    with input_as_png(input_file) as input_stream:
         args = [
             'pngquant',
             '--force',
             '--skip-if-larger',
-            '--output',
-            output_file,
             '--quality',
             f'{quality_min}-{quality_max}',
-            '--',
-            input_file,
+            '--',  # pngquant: stop processing arguments
+            '-',  # pngquant: stream input and output
         ]
-        run(args)
+        result = run(args, stdin=input_stream, stdout=PIPE, stderr=PIPE, check=False)
+
+    if result.returncode == 0:
+        # input_file could be the same as output_file, so we defer the write
+        output_file.write_bytes(result.stdout)

src/ocrmypdf/_exec/unpaper.py

@@ -107,7 +107,7 @@ def run(input_file, output_file, dpi, mode_args):
 
 def validate_custom_args(args: str):
     unpaper_args = shlex.split(args)
-    if any('/' in arg for arg in unpaper_args):
+    if any(('/' in arg or arg == '.' or arg == '..') for arg in unpaper_args):
         raise ValueError('No filenames allowed in --unpaper-args')
     return unpaper_args
 

src/ocrmypdf/api.py

@@ -226,7 +226,7 @@ def ocr(  # pylint: disable=unused-argument
     user_words: os.PathLike = None,
     user_patterns: os.PathLike = None,
     fast_web_view: float = None,
-    plugins: Iterable[str] = None,
+    plugins: Iterable[Union[str, Path]] = None,
     keep_temporary_files: bool = None,
     progress_bar: bool = None,
     **kwargs,
@@ -280,6 +280,8 @@ def ocr(  # pylint: disable=unused-argument
     """
     if not plugins:
         plugins = []
+    elif isinstance(plugins, (str, Path)):
+        plugins = [plugins]
     else:
         plugins = list(plugins)
 

src/ocrmypdf/builtin_plugins/ghostscript.py

@@ -39,13 +39,11 @@ def check_options(options):
     if gs_version < '9.20' and options.output_type != 'pdf' and not is_latin:
         # https://bugs.ghostscript.com/show_bug.cgi?id=696874
         # Ghostscript < 9.20 fails to encode multibyte characters properly
-        msg = (
-            "The installed version of Ghostscript does not work correctly "
-            "with the OCR languages you specified. Use --output-type pdf or "
+        log.warning(
+            f"The installed version of Ghostscript ({gs_version}) does not work "
+            "correctly with the OCR languages you specified. Use --output-type pdf or "
             "upgrade to Ghostscript 9.20 or later to avoid this issue."
         )
-        msg += f"Found Ghostscript {gs_version}"
-        log.warning(msg)
 
     if options.output_type == 'pdfa':
         options.output_type = 'pdfa-2'

src/ocrmypdf/hocrtransform.py

@@ -167,7 +167,10 @@ class HocrTransform:
     def topdown_position(self, element):
         pxl_line_coords = self.element_coordinates(element)
         line_box = self.pt_from_pixel(pxl_line_coords)
-        return -line_box.y2
+        # Coordinates here are still in the hocr coordinate system, so 0 on the y axis
+        # is the top of the page and increasing values of y will move towards the
+        # bottom of the page.
+        return line_box.y2
 
     def to_pdf(
         self,

src/ocrmypdf/optimize.py

@@ -77,23 +77,26 @@ def extract_image_filter(
     if image.Subtype != Name.Image:
         return None
     if image.Length < 100:
-        log.debug("Skipping small image, xref %s", xref)
+        log.debug(f"Skipping small image, xref {xref}")
         return None
 
     pim = PdfImage(image)
 
     if len(pim.filter_decodeparms) > 1:
-        log.debug("Skipping multiply filtered, xref %s", xref)
+        log.debug(f"Skipping multiply filtered image, xref {xref}")
         return None
     filtdp = pim.filter_decodeparms[0]
 
     if pim.bits_per_component > 8:
+        log.debug(f"Skipping wide gamut image, xref {xref}")
         return None  # Don't mess with wide gamut images
 
     if filtdp[0] == Name.JPXDecode:
+        log.debug(f"Skipping JPEG2000 iamge, xref {xref}")
         return None  # Don't do JPEG2000
 
     if Name.Decode in image:
+        log.debug(f"Skipping image with Decode table, xref {xref}")
         return None  # Don't mess with custom Decode tables
 
     return pim, filtdp
@@ -113,12 +116,23 @@ def extract_image_jbig2(
         and jbig2enc.available()
     ):
         try:
+            # Save any colorspace associated with the image, so that we
+            # will export a pure 1-bit PNG with no palette or ICC profile.
+            # Showing the palette or ICC to jbig2enc will cause it to perform
+            # colorspace transform to 1bpp, which will conflict the palette or
+            # ICC if it exists.
+            colorspace = pim.obj.ColorSpace
+            # Set to DeviceGray temporarily; we already in 1 bpc.
+            pim.obj.ColorSpace = pikepdf.Name.DeviceGray
             imgname = root / f'{xref:08d}'
             with imgname.open('wb') as f:
                 ext = pim.extract_to(stream=f)
             imgname.rename(imgname.with_suffix(ext))
         except pikepdf.UnsupportedImageTypeError:
             return None
+        finally:
+            # Restore image colorspace after temporarily setting it to DeviceGray
+            pim.obj.ColorSpace = colorspace
         return XrefExt(xref, ext)
     return None
 
@@ -229,7 +243,9 @@ def extract_images(
                 # Ignore soft masks
                 smask_xref = Xref(image.SMask.objgen[0])
                 exclude_xrefs.add(smask_xref)
+                log.debug(f"Skipping image {smask_xref} because it is an SMask")
             include_xrefs.add(xref)
+            log.debug(f"Treating {xref} as an optimization candidate")
             if xref not in pageno_for_xref:
                 pageno_for_xref[xref] = pageno
 
@@ -411,12 +427,29 @@ def _transcode_png(pike: Pdf, filename: Path, xref: Xref) -> bool:
             decode_parms=local_image.DecodeParms,
         )
 
+        # Don't copy keys from the new image...
         del_keys = set(im_obj.keys()) - set(local_image.keys())
+        # ...except for the keep_fields, which are essential to displaying
+        # the image correctly and preserving its metadata. (/Decode arrays
+        # and /SMaskInData are implicitly discarded prior to this point.)
+        keep_fields = {
+            '/ID',
+            '/Intent',
+            '/Interpolate',
+            '/Mask',
+            '/Metadata',
+            '/OC',
+            '/OPI',
+            '/SMask',
+            '/StructParent',
+        }
+        del_keys -= keep_fields
         for key in local_image.keys():
-            if key != Name.Length:
+            if key != Name.Length and str(key) not in keep_fields:
                 im_obj[key] = local_image[key]
         for key in del_keys:
             del im_obj[key]
+    return True
 
 
 def transcode_pngs(

tests/test_validation.py

@@ -55,7 +55,7 @@ def test_old_ghostscript(caplog):
         vd._check_options(
             *make_opts_pm(language='chi_sim', output_type='pdfa'), {'chi_sim'}
         )
-        assert 'Ghostscript does not work correctly' in caplog.text
+        assert 'does not work correctly' in caplog.text
 
     with patch('ocrmypdf._exec.ghostscript.version', return_value='9.18'), patch(
         'ocrmypdf._exec.tesseract.has_textonly_pdf', return_value=True