diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 00ef700a..3bd9bcd9 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -1,22 +1,18 @@ # OCRmyPDF # -FROM ubuntu:17.10 +FROM ubuntu:18.04 RUN apt-get update && apt-get install -y --no-install-recommends \ - software-properties-common python-software-properties \ - python3-wheel \ - python3-reportlab \ - python3-venv \ - ghostscript \ - qpdf \ - poppler-utils \ + ocrmypdf \ unpaper \ - libffi-dev \ - tesseract-ocr \ tesseract-ocr-eng \ tesseract-ocr-fra \ + tesseract-ocr-deu \ tesseract-ocr-spa \ - tesseract-ocr-deu + tesseract-ocr-por \ + tesseract-ocr-chi-sim \ + python3-venv \ + python3-pip ENV LANG=C.UTF-8 diff --git a/.docker/Dockerfile.tess4 b/.docker/Dockerfile.tess4 deleted file mode 100644 index 1f1b447b..00000000 --- a/.docker/Dockerfile.tess4 +++ /dev/null @@ -1,64 +0,0 @@ -# OCRmyPDF -# -FROM ubuntu:17.10 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - software-properties-common python-software-properties \ - python3-wheel \ - python3-reportlab \ - python3-venv \ - ghostscript \ - qpdf \ - poppler-utils \ - unpaper \ - libffi-dev - -RUN add-apt-repository ppa:alex-p/tesseract-ocr - -RUN apt-get update \ - && apt-get autoremove -y \ - && apt-get install -y --no-install-recommends \ - tesseract-ocr \ - tesseract-ocr-eng \ - tesseract-ocr-fra \ - tesseract-ocr-deu \ - tesseract-ocr-spa \ - tesseract-ocr-por \ - tesseract-ocr-ara \ - tesseract-ocr-rus \ - tesseract-ocr-chi-sim - -ENV LANG=C.UTF-8 - -RUN python3 -m venv --system-site-packages /appenv - -# This installs the latest binary wheel instead of the code in the current -# folder. Installing from source will fail, apparently because cffi needs -# build-essentials (gcc) to do a source installation -# (i.e. "pip install ."). It's unclear to me why this is the case. -RUN . /appenv/bin/activate; \ - pip install --upgrade pip \ - && pip install ocrmypdf[fitz] - -# Now copy the application in, mainly to get the test suite. -# Do this now to make the best use of Docker cache. -COPY . /application -RUN . /appenv/bin/activate; \ - pip install -r /application/test_requirements.txt - -# Remove the junk, including the source version of application since it was -# already installed -RUN rm -rf /tmp/* /var/tmp/* /root/* /application/ocrmypdf \ - && apt-get autoremove -y \ - && apt-get autoclean -y - -RUN useradd docker \ - && mkdir /home/docker \ - && chown docker:docker /home/docker - -USER docker -WORKDIR /home/docker - -# Must use array form of ENTRYPOINT -# Non-array form does not append other arguments, because that is "intuitive" -ENTRYPOINT ["/application/.docker/docker-wrapper.sh"] diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE index a526bfc9..4d0dc4c9 100644 --- a/.github/ISSUE_TEMPLATE +++ b/.github/ISSUE_TEMPLATE @@ -1,3 +1,12 @@ -Please include the command line and (if needed) a test file with your issue report. +Please include the command line and a test file with your issue report. If possible, please use a test file that we can include in future test cases (no personal information, no copyrighted material). + +If you wish to encrypt a test file so that only the maintainer of OCRmyPDF can view it, you may use: + +```bash + +gpg --recv-keys 4434eb74c4a35f7f --keyserver pgp.mit.edu +gpg --output test.pdf.gpg --encrypt --receipient 4434eb74c4a35f7f test.pdf + +``` diff --git a/docs/cookbook.rst b/docs/cookbook.rst index a3c4dbef..3f79152c 100644 --- a/docs/cookbook.rst +++ b/docs/cookbook.rst @@ -65,8 +65,8 @@ By default OCRmyPDF assumes the document is English. .. code-block:: bash - ocrmypdf -l fre LeParisien.pdf LeParisien.pdf - ocrmypdf -l eng+fre Bilingual-English-French.pdf Bilingual-English-French.pdf + ocrmypdf -l fra LeParisien.pdf LeParisien.pdf + ocrmypdf -l eng+fra Bilingual-English-French.pdf Bilingual-English-French.pdf Language packs must be installed for all languages specified. See :ref:`Installing additional language packs `. diff --git a/docs/installation.rst b/docs/installation.rst index d858d926..8efaac2e 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -77,25 +77,22 @@ Assuming you have a Docker engine running, you can download one of the three ava - Notes * - ocrmypdf - ``docker pull jbarlow83/ocrmypdf`` - - Latest ocrmypdf with Tesseract 3.x. Includes English, French, German, Spanish. + - Latest ocrmypdf with Tesseract 4.0.0-beta1 on Ubuntu 18.04. Includes English, French, German, Spanish, Portugeuse and Simplified Chinese. * - ocrmypdf-polyglot - ``docker pull jbarlow83/ocrmypdf-polyglot`` - As above, with all available language packs. - * - ocrmypdf-tess4 - - ``docker pull jbarlow83/ocrmypdf-tess4`` - - Latest ocrmypdf with Tesseract 4.x and English, French, German, Spanish, Portuguese, Chinese Simplified, Arabic and Russian (the top 8). For example: .. code-block:: bash - docker pull jbarlow83/ocrmypdf-tess4 + docker pull jbarlow83/ocrmypdf Then tag it to give a more convenient name, just ocrmypdf: .. code-block:: bash - docker tag jbarlow83/ocrmypdf-tess4 ocrmypdf + docker tag jbarlow83/ocrmypdf ocrmypdf .. _docker-polyglot: @@ -211,6 +208,26 @@ The command line program should now be available: ocrmypdf --help +Installing the latest version on Ubuntu 18.04 LTS +------------------------------------------------- + +Ubuntu 18.04 includes ocrmypdf 6.1.2. To install a more recent version, first +install the system version to get all the dependencies: + +.. code-block:: bash + + sudo apt-get update + sudo apt-get install \ + ocrmypdf \ + python3-pip + +Then install ocrmypdf 6.1.5 for the local user and set the user's ``PATH`` to check for the user's Python packages. + +.. code-block:: bash + + export PATH=$HOME/.local/bin:$PATH + pip3 install --user ocrmypdf[fitz] + Installing on Ubuntu 16.04 LTS ------------------------------ @@ -383,7 +400,7 @@ The following dependencies are recommended: These are in addition to the Python packaging dependencies, meaning that unfortunately, the ``pip install`` command cannot satisfy all of them. -Python 3.6 and Tesseract 4.x are recommended for best OCR results and best performance. +Python 3.6 and Tesseract 4.0.0-beta.1 are recommended for best OCR results and best performance. The library PyMuPDF is not widely available in platform distributions, and it improves OCRmyPDF in certain conditions. Consider installing OCRmyPDF from the Python binary wheels, which include a precompiled version of this library. diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 5bc73c3a..bb980dd4 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -9,10 +9,24 @@ The OCRmyPDF package itself does not contain a public API, although it is fairly find: [^`]\#([0-9]{1,3})[^0-9] replace: `#$1 `_ +v6.2.0 +------ + +- **Docker**: The Docker image ``ocrmypdf-tess4`` has been removed. The main Docker images, ``ocrmypdf`` and ``ocrmypdf-polyglot`` now use Ubuntu 18.04 as a base image, and as such Tesseract 4.0.0-beta1 is now the Tesseract version they use. There is no Docker image based on Tesseract 3.05 anymore. + +- Creation of PDF/A-3 is now supported. However, there is no ability to attach files to PDF/A-3. + +- List more reasons why the file size might grow. + +- Fix issue `#262 `_, ``--remove-background`` error on PDFs contained colormapped (paletted) images. + +- Fix another XMP metadata validation issue, in cases where the input file's creation date has no timezone and the creation date is not overridden. + + v6.1.5 ------ -- Fix issue `#253 `_, a possible division by zero when using the ``hocr`` renderer. +- Fix issue `#253 `_, a possible division by zero when using the ``hocr`` renderer. - Fix incorrectly formatted ```` field inside XMP metadata for PDF/As. veraPDF flags this as a PDF/A validation failure. The error is caused the timezone and final digit of the seconds of modified time to be omitted, so at worst the modification time stamp is rounded to the nearest 10 seconds. diff --git a/src/ocrmypdf/__main__.py b/src/ocrmypdf/__main__.py index ee4018bb..b7739938 100755 --- a/src/ocrmypdf/__main__.py +++ b/src/ocrmypdf/__main__.py @@ -156,14 +156,15 @@ parser.add_argument( '--image-dpi', metavar='DPI', type=int, help="For input image instead of PDF, use this DPI instead of file's.") parser.add_argument( - '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2'], + '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'], default='pdfa', help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for " "long term archiving (default, recommended) but may not suitable " "for users who want their file altered as little as possible. 'pdfa' " "also has problems with full Unicode text. 'pdf' attempts to " "preserve file contents as much as possible. 'pdf-a1' creates a " - "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'." + "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'. 'pdf-a3' creates a " + "PDF/A3-b file." ) # Use null string '\0' as sentinel to indicate the user supplied no argument, @@ -404,6 +405,14 @@ def check_options_output(options, log): raise MissingDependencyError(msg) log.warning(msg) + if options.output_type == 'pdfa': + options.output_type = 'pdfa-2' + + if options.output_type == 'pdfa-3' and ghostscript.version() < '9.19': + raise MissingDependencyError( + "--output-type pdfa-3 requires Ghostscript 9.19 or later" + ) + lossless_reconstruction = False if options.pdf_renderer in ('hocr', 'sandwich'): if not any((options.deskew, options.clean_final, options.force_ocr, @@ -758,8 +767,20 @@ def report_output_file_size(options, _log, input_file, output_file): reasons = [] if not fitz: reasons.append("The optional dependency PyMuPDF is not installed.") - if options.force_ocr: - reasons.append("The argument --force-ocr was issued.") + image_preproc = { + 'deskew', + 'clean_final', + 'remove_background', + 'oversample', + 'force_ocr' + } + for arg in image_preproc: + attr = getattr(options, arg, None) + if not attr: + continue + reasons.append( + "The argument --{} was issued, causing transcoding.".format( + arg.replace('_', '-'))) if reasons: explanation = ( diff --git a/src/ocrmypdf/helpers.py b/src/ocrmypdf/helpers.py index b7833426..745549a8 100644 --- a/src/ocrmypdf/helpers.py +++ b/src/ocrmypdf/helpers.py @@ -22,6 +22,7 @@ from pathlib import Path import sys import os import multiprocessing +import warnings def re_symlink(input_file, soft_link_name, log=None): @@ -70,7 +71,7 @@ def is_iterable_notstr(thing): def page_number(input_file): "Get one-based page number implied by filename (000002.pdf -> 2)" - return int(os.path.basename(input_file)[0:6]) + return int(os.path.basename(fspath(input_file))[0:6]) def available_cpu_count(): @@ -85,7 +86,7 @@ def available_cpu_count(): except (ImportError, AttributeError): pass - complain( + warnings.warn( "Could not get CPU count. Assuming one (1) CPU." "Use -j N to set manually.") return 1 diff --git a/src/ocrmypdf/leptonica.py b/src/ocrmypdf/leptonica.py index 6ee9747e..26abd32f 100644 --- a/src/ocrmypdf/leptonica.py +++ b/src/ocrmypdf/leptonica.py @@ -108,13 +108,6 @@ class LeptonicaIOError(LeptonicaError): pass -class RemoveColormap(Enum): - to_binary = 0 - to_grayscale = 1 - to_full_color = 2 - based_on_src = 3 - - class Pix: """Wrapper around leptonica's PIX object. @@ -141,9 +134,10 @@ class Pix: def __repr__(self): if self._pix: - s = "" + s = "" return s.format(self._pix.w, self._pix.h, self._pix.d, - int(ffi.cast("intptr_t", self._pix))) + int(ffi.cast('intptr_t', self._pix)), + '(colormapped)' if self._pix.colormap else '') else: return "" @@ -328,13 +322,16 @@ class Pix: return None def remove_colormap(self, removal_type): - """Remove a palette + """Remove a palette (colormap); if no colormap, returns a copy of this + image + + removal_type - any of lept.REMOVE_CMAP_* - removal_type - RemovalColormap() """ - - with LeptonicaErrorTrap(): - return Pix(lept.pixRemoveColormap(self._pix, removal_type)) + return self + # with LeptonicaErrorTrap(): + # return Pix(lept.pixRemoveColormapGeneral( + # self._pix, removal_type, lept.L_COPY)) def otsu_adaptive_threshold( self, tile_size=(300, 300), kernel_size=(4, 4), scorefract=0.1): @@ -426,9 +423,11 @@ class Pix: def background_norm( self, mask=None, grayscale=None, tile_size=(10, 15), fg_threshold=60, min_count=40, bg_val=200, smooth_kernel=(2, 1)): + # Background norm doesn't work on color mapped Pix, so remove colormap + target_pix = self.remove_colormap(lept.REMOVE_CMAP_BASED_ON_SRC) with LeptonicaErrorTrap(): return Pix(lept.pixBackgroundNorm( - self._pix, + target_pix._pix, mask or ffi.NULL, grayscale or ffi.NULL, tile_size[0], @@ -450,8 +449,8 @@ class Pix: if get_leptonica_version() < 'leptonica-1.72': # Older versions of Leptonica (pre-1.72) have a buggy # implementation of pixCorrelationBinary that overflows on larger - # images. Ubuntu trusty has 1.70. Ubuntu PPA - # ppa:rebuntu16/avidemux+unofficial has "leptonlib" 1.73. + # images. Ubuntu 14.04/trusty has 1.70. Ubuntu PPA + # ppa:alex-p/tesseract-ocr has leptonlib 1.75. pix1_count = ffi.new('l_int32 *') pix2_count = ffi.new('l_int32 *') pixn_count = ffi.new('l_int32 *') diff --git a/src/ocrmypdf/lib/_leptonica.py b/src/ocrmypdf/lib/_leptonica.py index 9bdb3aed..80e67386 100644 --- a/src/ocrmypdf/lib/_leptonica.py +++ b/src/ocrmypdf/lib/_leptonica.py @@ -3,8 +3,9 @@ import _cffi_backend ffi = _cffi_backend.FFI('ocrmypdf.lib._leptonica', _version = 0x2601, - _types = b'\x00\x00\x0F\x0D\x00\x00\xBB\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xBE\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\xBA\x03\x00\x00\x0F\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x02\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xC7\x03\x00\x00\x1C\x01\x00\x00\x00\x0F\x00\x00\xBD\x0D\x00\x00\x00\x0F\x00\x00\x49\x0D\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x1F\x03\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x49\x11\x00\x00\x49\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x49\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x01\x03\x00\x00\x9D\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\xC6\x03\x00\x00\x65\x03\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x08\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\xC4\x03\x00\x00\xA3\x11\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\xCA\x0D\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x00\xCA\x0D\x00\x00\x9D\x11\x00\x00\x00\x0F\x00\x00\xCA\x0D\x00\x00\xCA\x03\x00\x00\x00\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x09\x00\x00\xBE\x03\x00\x00\x02\x01\x00\x00\x0E\x01\x00\x00\x0B\x01\x00\x00\x05\x01\x00\x00\x03\x01\x00\x00\xBC\x03\x00\x00\xC5\x03\x00\x00\x04\x01\x00\x00\xC7\x03\x00\x00\x08\x01\x00\x00\x0C\x01\x00\x00\x06\x01\x00\x00\x00\x01', - _globals = (b'\x00\x00\xB1\x23boxDestroy',0,b'\x00\x00\x67\x23getLeptonicaVersion',0,b'\x00\x00\xB7\x23lept_free',0,b'\x00\x00\x69\x23makePixelSumTab8',0,b'\x00\x00\x16\x23pixAnd',0,b'\x00\x00\x23\x23pixBackgroundNorm',0,b'\x00\x00\x1B\x23pixCleanBackgroundToWhite',0,b'\x00\x00\x0D\x23pixClipRectangle',0,b'\x00\x00\x8D\x23pixColorFraction',0,b'\x00\x00\x59\x23pixColorMagnitude',0,b'\x00\x00\x0A\x23pixConvertRGBToLuminance',0,b'\x00\x00\x6B\x23pixCorrelationBinary',0,b'\x00\x00\x80\x23pixCountPixels',0,b'\x00\x00\x63\x23pixDeserializeFromMemory',0,b'\x00\x00\x50\x23pixDeskew',0,b'\x00\x00\xB4\x23pixDestroy',0,b'\x00\x00\x0A\x23pixEndianByteSwapNew',0,b'\x00\x00\x00\x23pixFindPageForeground',0,b'\x00\x00\x7B\x23pixFindSkew',0,b'\x00\x00\x2F\x23pixGammaTRC',0,b'\x00\x00\x70\x23pixGetAverageMaskedRGB',0,b'\x00\x00\x36\x23pixGlobalNormRGB',0,b'\x00\x00\x12\x23pixInvert',0,b'\x00\x00\x54\x23pixMaskOverColorPixels',0,b'\x00\x00\x85\x23pixNumSignificantGrayColors',0,b'\x00\x00\x96\x23pixOtsuAdaptiveThreshold',0,b'\x00\x00\x3E\x23pixOtsuThreshOnBackgroundNorm',0,b'\x00\x00\x60\x23pixRead',0,b'\x00\x00\x50\x23pixRemoveColormap',0,b'\x00\x00\x12\x23pixRotate180',0,b'\x00\x00\x50\x23pixRotateOrth',0,b'\x00\x00\x4B\x23pixScale',0,b'\x00\x00\xA0\x23pixSerializeToMemory',0,b'\x00\x00\xA5\x23pixWriteImpliedFormat',0,b'\x00\x00\xAB\x23pixWriteMemPng',0), - _struct_unions = ((b'\x00\x00\x00\xBA\x00\x00\x00\x02Box',b'\x00\x00\x02\x11x',b'\x00\x00\x02\x11y',b'\x00\x00\x02\x11w',b'\x00\x00\x02\x11h',b'\x00\x00\xC7\x11refcount'),(b'\x00\x00\x00\xBB\x00\x00\x00\x02Pix',b'\x00\x00\xC7\x11w',b'\x00\x00\xC7\x11h',b'\x00\x00\xC7\x11d',b'\x00\x00\xC7\x11spp',b'\x00\x00\xC7\x11wpl',b'\x00\x00\xC7\x11refcount',b'\x00\x00\x02\x11xres',b'\x00\x00\x02\x11yres',b'\x00\x00\x02\x11informat',b'\x00\x00\x02\x11special',b'\x00\x00\xBD\x11text',b'\x00\x00\xC3\x11colormap',b'\x00\x00\xC6\x11data'),(b'\x00\x00\x00\xBC\x00\x00\x00\x02PixColormap',b'\x00\x00\xB8\x11array',b'\x00\x00\x02\x11depth',b'\x00\x00\x02\x11nalloc',b'\x00\x00\x02\x11n')), - _typenames = (b'\x00\x00\x00\xBABOX',b'\x00\x00\x00\xBBPIX',b'\x00\x00\x00\xBCPIXCMAP',b'\x00\x00\x00\x1Fl_float32',b'\x00\x00\x00\xBFl_float64',b'\x00\x00\x00\xC1l_int16',b'\x00\x00\x00\x02l_int32',b'\x00\x00\x00\xC0l_int64',b'\x00\x00\x00\xC2l_int8',b'\x00\x00\x00\xC9l_uint16',b'\x00\x00\x00\xC7l_uint32',b'\x00\x00\x00\xC8l_uint64',b'\x00\x00\x00\xC5l_uint8'), + _types = b'\x00\x00\x0F\x0D\x00\x00\xBB\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xBE\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\xBA\x03\x00\x00\x0F\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x02\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xC9\x03\x00\x00\x1C\x01\x00\x00\x00\x0F\x00\x00\xBD\x0D\x00\x00\x00\x0F\x00\x00\x49\x0D\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x1F\x03\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x49\x11\x00\x00\x49\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x49\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x01\x03\x00\x00\x9D\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\xC8\x03\x00\x00\x65\x03\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x08\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\xC6\x03\x00\x00\xA3\x11\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\xCC\x0D\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x00\xCC\x0D\x00\x00\x9D\x11\x00\x00\x00\x0F\x00\x00\xCC\x0D\x00\x00\xCC\x03\x00\x00\x00\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x09\x00\x00\xBE\x03\x00\x00\x02\x01\x00\x00\x0E\x01\x00\x00\x00\x0B\x00\x00\x01\x0B\x00\x00\x0B\x01\x00\x00\x05\x01\x00\x00\x03\x01\x00\x00\xBC\x03\x00\x00\xC7\x03\x00\x00\x04\x01\x00\x00\xC9\x03\x00\x00\x08\x01\x00\x00\x0C\x01\x00\x00\x06\x01\x00\x00\x00\x01', + _globals = (b'\xFF\xFF\xFF\x0BL_CLONE',2,b'\xFF\xFF\xFF\x0BL_COPY',1,b'\xFF\xFF\xFF\x0BL_COPY_CLONE',3,b'\xFF\xFF\xFF\x0BL_INSERT',0,b'\xFF\xFF\xFF\x0BL_NOCOPY',0,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_BASED_ON_SRC',4,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_BINARY',0,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_FULL_COLOR',2,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_GRAYSCALE',1,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_WITH_ALPHA',3,b'\x00\x00\xB1\x23boxDestroy',0,b'\x00\x00\x67\x23getLeptonicaVersion',0,b'\x00\x00\xB7\x23lept_free',0,b'\x00\x00\x69\x23makePixelSumTab8',0,b'\x00\x00\x16\x23pixAnd',0,b'\x00\x00\x23\x23pixBackgroundNorm',0,b'\x00\x00\x1B\x23pixCleanBackgroundToWhite',0,b'\x00\x00\x0D\x23pixClipRectangle',0,b'\x00\x00\x8D\x23pixColorFraction',0,b'\x00\x00\x59\x23pixColorMagnitude',0,b'\x00\x00\x0A\x23pixConvertRGBToLuminance',0,b'\x00\x00\x6B\x23pixCorrelationBinary',0,b'\x00\x00\x80\x23pixCountPixels',0,b'\x00\x00\x63\x23pixDeserializeFromMemory',0,b'\x00\x00\x50\x23pixDeskew',0,b'\x00\x00\xB4\x23pixDestroy',0,b'\x00\x00\x0A\x23pixEndianByteSwapNew',0,b'\x00\x00\x00\x23pixFindPageForeground',0,b'\x00\x00\x7B\x23pixFindSkew',0,b'\x00\x00\x2F\x23pixGammaTRC',0,b'\x00\x00\x70\x23pixGetAverageMaskedRGB',0,b'\x00\x00\x36\x23pixGlobalNormRGB',0,b'\x00\x00\x12\x23pixInvert',0,b'\x00\x00\x54\x23pixMaskOverColorPixels',0,b'\x00\x00\x85\x23pixNumSignificantGrayColors',0,b'\x00\x00\x96\x23pixOtsuAdaptiveThreshold',0,b'\x00\x00\x3E\x23pixOtsuThreshOnBackgroundNorm',0,b'\x00\x00\x60\x23pixRead',0,b'\x00\x00\x50\x23pixRemoveColormap',0,b'\x00\x00\x54\x23pixRemoveColormapGeneral',0,b'\x00\x00\x12\x23pixRotate180',0,b'\x00\x00\x50\x23pixRotateOrth',0,b'\x00\x00\x4B\x23pixScale',0,b'\x00\x00\xA0\x23pixSerializeToMemory',0,b'\x00\x00\xA5\x23pixWriteImpliedFormat',0,b'\x00\x00\xAB\x23pixWriteMemPng',0), + _struct_unions = ((b'\x00\x00\x00\xBA\x00\x00\x00\x02Box',b'\x00\x00\x02\x11x',b'\x00\x00\x02\x11y',b'\x00\x00\x02\x11w',b'\x00\x00\x02\x11h',b'\x00\x00\xC9\x11refcount'),(b'\x00\x00\x00\xBB\x00\x00\x00\x02Pix',b'\x00\x00\xC9\x11w',b'\x00\x00\xC9\x11h',b'\x00\x00\xC9\x11d',b'\x00\x00\xC9\x11spp',b'\x00\x00\xC9\x11wpl',b'\x00\x00\xC9\x11refcount',b'\x00\x00\x02\x11xres',b'\x00\x00\x02\x11yres',b'\x00\x00\x02\x11informat',b'\x00\x00\x02\x11special',b'\x00\x00\xBD\x11text',b'\x00\x00\xC5\x11colormap',b'\x00\x00\xC8\x11data'),(b'\x00\x00\x00\xBC\x00\x00\x00\x02PixColormap',b'\x00\x00\xB8\x11array',b'\x00\x00\x02\x11depth',b'\x00\x00\x02\x11nalloc',b'\x00\x00\x02\x11n')), + _enums = (b'\x00\x00\x00\xC0\x00\x00\x00\x16$1\x00REMOVE_CMAP_TO_BINARY,REMOVE_CMAP_TO_GRAYSCALE,REMOVE_CMAP_TO_FULL_COLOR,REMOVE_CMAP_WITH_ALPHA,REMOVE_CMAP_BASED_ON_SRC',b'\x00\x00\x00\xC1\x00\x00\x00\x16$2\x00L_NOCOPY,L_INSERT,L_COPY,L_CLONE,L_COPY_CLONE'), + _typenames = (b'\x00\x00\x00\xBABOX',b'\x00\x00\x00\xBBPIX',b'\x00\x00\x00\xBCPIXCMAP',b'\x00\x00\x00\x1Fl_float32',b'\x00\x00\x00\xBFl_float64',b'\x00\x00\x00\xC3l_int16',b'\x00\x00\x00\x02l_int32',b'\x00\x00\x00\xC2l_int64',b'\x00\x00\x00\xC4l_int8',b'\x00\x00\x00\xCBl_uint16',b'\x00\x00\x00\xC9l_uint32',b'\x00\x00\x00\xCAl_uint64',b'\x00\x00\x00\xC7l_uint8'), ) diff --git a/src/ocrmypdf/lib/compile_leptonica.py b/src/ocrmypdf/lib/compile_leptonica.py index 6da11266..5ffa7d87 100644 --- a/src/ocrmypdf/lib/compile_leptonica.py +++ b/src/ocrmypdf/lib/compile_leptonica.py @@ -72,6 +72,24 @@ struct Box }; typedef struct Box BOX; +enum { + REMOVE_CMAP_TO_BINARY = 0, /*!< remove colormap for conv to 1 bpp */ + REMOVE_CMAP_TO_GRAYSCALE = 1, /*!< remove colormap for conv to 8 bpp */ + REMOVE_CMAP_TO_FULL_COLOR = 2, /*!< remove colormap for conv to 32 bpp */ + REMOVE_CMAP_WITH_ALPHA = 3, /*!< remove colormap and alpha */ + REMOVE_CMAP_BASED_ON_SRC = 4 /*!< remove depending on src format */ +}; + +/*! Access and storage flags */ +enum { + L_NOCOPY = 0, /*!< do not copy the object; do not delete the ptr */ + L_INSERT = L_NOCOPY, /*!< stuff it in; do not copy or clone */ + L_COPY = 1, /*!< make/use a copy of the object */ + L_CLONE = 2, /*!< make/use clone (ref count) of the object */ + L_COPY_CLONE = 3 /*!< make a new array object (e.g., pixa) and fill */ + /*!< the array with clones (e.g., pix) */ +}; + """) ffibuilder.cdef(""" @@ -227,6 +245,11 @@ PIX * pixInvert(PIX * pixd, PIX * pixs); +PIX * +pixRemoveColormapGeneral(PIX *pixs, + l_int32 type, + l_int32 ifnocmap); + void boxDestroy(BOX **pbox); diff --git a/src/ocrmypdf/pdfa.py b/src/ocrmypdf/pdfa.py index 405a4d4a..93f11a22 100644 --- a/src/ocrmypdf/pdfa.py +++ b/src/ocrmypdf/pdfa.py @@ -125,31 +125,31 @@ def encode_pdf_date(d: datetime) -> str: s = d.strftime(pdfmark_date_fmt) tz = d.strftime('%z') - if tz == 'Z': + if tz == 'Z' or tz == '': + # Ghostscript <= 9.23 handles missing timezones incorrectly, so if + # timezone is missing, move it into GMT. + # https://bugs.ghostscript.com/show_bug.cgi?id=699182 s += "+00'00'" - elif tz != '': + else: sign, tz_hours, tz_mins = tz[0], tz[1:3], tz[3:5] s += "{}{}'{}'".format(sign, tz_hours, tz_mins) - else: - raise ValueError("Naive timezone not supported") return s def decode_pdf_date(s: str) -> datetime: - pdfmark_date_fmts = ( - r'%Y%m%d%H%M%S%z', # +0430 etc - r'%Y%m%d%H%M%S', # no time zone - r'%Y%m%d%H%M%SZ') # trailing Z - if s.startswith('D:'): s = s[2:] + + # Literal Z00'00', is incorrect but found in the wild, + # probably made by OS X Quartz -- standardize + if s.endswith("Z00'00'"): + s = s.replace("Z00'00'", '+0000') + elif s.endswith('Z'): + s = s.replace('Z', '+0000') + s = s.replace("'", "") # Remove apos from PDF time strings - for fmt in pdfmark_date_fmts: - try: - return datetime.strptime(s, fmt) - except ValueError: - continue - return None + + return datetime.strptime(s, r'%Y%m%d%H%M%S%z') def _get_pdfmark_dates(pdfmark): @@ -170,7 +170,12 @@ def _get_pdfmark_dates(pdfmark): date_str = pdfmark[key] if date_str.startswith('D:'): date_str = date_str[2:] - yield ' {} (D:{})'.format(key, date_str) + try: + yield ' {} (D:{})'.format( + key, + encode_pdf_date(decode_pdf_date(date_str))) + except ValueError: + yield ' {} null'.format(key) def _get_pdfa_def(icc_profile, icc_identifier, pdfmark): diff --git a/src/ocrmypdf/pipeline.py b/src/ocrmypdf/pipeline.py index a401b52a..3def7b25 100644 --- a/src/ocrmypdf/pipeline.py +++ b/src/ocrmypdf/pipeline.py @@ -232,7 +232,7 @@ def repair_and_parse_pdf( pdfinfo = PdfInfo(output_file) - if pdfinfo.has_userunit and options.output_type == 'pdfa': + if pdfinfo.has_userunit and options.output_type.startswith('pdfa'): log.error( "This input file uses a PDF feature that is not supported " "by Ghostscript, so you cannot use --output-type=pdfa for this " @@ -1067,6 +1067,7 @@ def _do_merge_ghostscript( context): options = context.get_options() input_pdfinfo = context.get_pdfinfo() + ghostscript.generate_pdfa( pdf_version=input_pdfinfo.min_version, pdf_pages=pdf_pages, @@ -1074,7 +1075,7 @@ def _do_merge_ghostscript( compression=options.pdfa_image_compression, log=log, threads=options.jobs or 1, - pdfa_part=('1' if options.output_type == 'pdfa-1' else '2')) + pdfa_part=options.output_type[-1]) # is pdfa-1, pdfa-2, or pdfa-3 if fitz: doc = fitz.Document(output_file + '_toc.pdf') doc.setToC(input_pdfinfo.table_of_contents) diff --git a/tests/resources/README.rst b/tests/resources/README.rst index 2b6e5683..c97c1061 100644 --- a/tests/resources/README.rst +++ b/tests/resources/README.rst @@ -147,6 +147,7 @@ Assemblies These test resources are assemblies or derivatives from other previously mentioned files, released under the same license terms as their input files. - baiona_gray.png (from baiona.png) +- baiona_colormapped.png (from baiona.png) - cardinal.pdf (four cardinal directions, baked-in rotated copies of LinnSequencer.jpg) - ccitt.pdf (LinnSequencer.jpg, converted to CCITT encoding) - encrypted_algo4.pdf (congress.jpg, encrypted with algorithm 4 - not supported by PyPDF2) diff --git a/tests/resources/baiona-colormapped.png b/tests/resources/baiona-colormapped.png new file mode 100644 index 00000000..5b60b913 Binary files /dev/null and b/tests/resources/baiona-colormapped.png differ diff --git a/tests/test_lept.py b/tests/test_lept.py new file mode 100644 index 00000000..be68ea98 --- /dev/null +++ b/tests/test_lept.py @@ -0,0 +1,30 @@ +# © 2018 James R. Barlow: github.com/jbarlow83 +# +# This file is part of OCRmyPDF. +# +# OCRmyPDF is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OCRmyPDF is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OCRmyPDF. If not, see . + + +import os +import shutil +import pytest +import sys +import ocrmypdf.leptonica as lept + + +def test_colormap_backgroundnorm(resources): + # Issue #262 - unclear how to reproduce exactly, so just ensure leptonica + # can handle that case + pix = lept.Pix.read(resources / 'baiona-colormapped.png') + pix.background_norm() diff --git a/tests/test_main.py b/tests/test_main.py index 77ed2ca9..ab035ab3 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -593,7 +593,7 @@ def test_tesseract_config_notfound(renderer, resources, outdir): '--pdf-renderer', renderer, '--tesseract-config', cfg_file) assert "Can't open" in err, "No error message about missing config file" - assert p.returncode == ExitCode.ok + assert p.returncode == ExitCode.ok, err @pytest.mark.parametrize('renderer', RENDERERS) @@ -822,15 +822,19 @@ def test_sidecar_nonempty(spoof_tesseract_cache, resources, outpdf): assert 'the' in ocr_text -def test_pdfa_1(spoof_tesseract_cache, resources, outpdf): +@pytest.mark.parametrize('pdfa_level', ['1', '2', '3']) +def test_pdfa_n(spoof_tesseract_cache, pdfa_level, resources, outpdf): + if pdfa_level == '3' and ghostscript.version() < '9.19': + pytest.xfail(reason='Ghostscript >= 9.19 required') + check_ocrmypdf( resources / 'ccitt.pdf', outpdf, - '--output-type', 'pdfa-1', + '--output-type', 'pdfa-' + pdfa_level, env=spoof_tesseract_cache ) pdfa_info = file_claims_pdfa(outpdf) - assert pdfa_info['conformance'] == 'PDF/A-1B' + assert pdfa_info['conformance'] == 'PDF/A-{}B'.format(pdfa_level) def test_bad_locale(): diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 300a70c7..23f47c40 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -73,11 +73,16 @@ def test_override_metadata(spoof_tesseract_noop, output_type, resources, assert p.returncode == ExitCode.ok, err - reader = pypdf.PdfFileReader(outpdf) + before = pypdf.PdfFileReader(str(input_file)) + after = pypdf.PdfFileReader(outpdf) - assert reader.documentInfo['/Title'] == german - assert reader.documentInfo['/Author'] == chinese - assert reader.documentInfo.get('/Keywords', '') == '' + assert after.documentInfo['/Title'] == german + assert after.documentInfo['/Author'] == chinese + assert after.documentInfo.get('/Keywords', '') == '' + + before_date = decode_pdf_date(before.documentInfo['/CreationDate']) + after_date = decode_pdf_date(after.documentInfo['/CreationDate']) + assert before_date == after_date pdfa_info = file_claims_pdfa(outpdf) assert pdfa_info['output'] == output_type diff --git a/tests/test_pageinfo.py b/tests/test_pageinfo.py index e421ed52..9893327f 100644 --- a/tests/test_pageinfo.py +++ b/tests/test_pageinfo.py @@ -26,7 +26,6 @@ import os import shutil import pytest import img2pdf -import pytest import sys import PyPDF2 as pypdf