From 06308a22cec463868aa82392f94a51d0bbb927e6 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sun, 30 Dec 2018 01:27:49 -0800 Subject: [PATCH] Reformat with black --- src/ocrmypdf/__init__.py | 16 +- src/ocrmypdf/__main__.py | 604 ++++++++++++-------- src/ocrmypdf/_jobcontext.py | 6 +- src/ocrmypdf/_pipeline.py | 526 +++++++++--------- src/ocrmypdf/_unicodefun.py | 15 +- src/ocrmypdf/_weave.py | 65 +-- src/ocrmypdf/exceptions.py | 14 +- src/ocrmypdf/exec/__init__.py | 34 +- src/ocrmypdf/exec/ghostscript.py | 151 ++--- src/ocrmypdf/exec/jbig2enc.py | 8 +- src/ocrmypdf/exec/pngquant.py | 8 +- src/ocrmypdf/exec/qpdf.py | 14 +- src/ocrmypdf/exec/tesseract.py | 128 ++--- src/ocrmypdf/exec/unpaper.py | 42 +- src/ocrmypdf/helpers.py | 38 +- src/ocrmypdf/hocrtransform.py | 174 +++--- src/ocrmypdf/leptonica.py | 243 +++++---- src/ocrmypdf/lib/_leptonica.py | 330 ++++++++++- src/ocrmypdf/lib/compile_leptonica.py | 12 +- src/ocrmypdf/optimize.py | 124 ++--- src/ocrmypdf/pdfa.py | 22 +- src/ocrmypdf/pdfinfo/__init__.py | 149 ++--- src/ocrmypdf/pdfinfo/ghosttext.py | 14 +- src/ocrmypdf/pdfinfo/layout.py | 105 +++- tests/conftest.py | 37 +- tests/spoof/gs_feature_elision.py | 1 + tests/spoof/gs_render_failure.py | 1 - tests/spoof/tesseract_badutf8.py | 7 +- tests/spoof/tesseract_big_image_error.py | 18 +- tests/spoof/tesseract_cache.py | 29 +- tests/spoof/tesseract_crash.py | 10 +- tests/spoof/tesseract_noop.py | 12 +- tests/spoof/unpaper_oldversion.py | 1 + tests/test_hocrtransform.py | 3 +- tests/test_lept.py | 3 +- tests/test_main.py | 667 ++++++++++++++--------- tests/test_metadata.py | 109 ++-- tests/test_optimize.py | 53 +- tests/test_pdfinfo.py | 22 +- tests/test_qpdf.py | 1 + tests/test_rotation.py | 130 +++-- tests/test_tess4.py | 48 +- tests/test_unpaper.py | 14 +- tests/test_userunit.py | 9 +- 44 files changed, 2459 insertions(+), 1558 deletions(-) diff --git a/src/ocrmypdf/__init__.py b/src/ocrmypdf/__init__.py index eff86707..a37d2658 100644 --- a/src/ocrmypdf/__init__.py +++ b/src/ocrmypdf/__init__.py @@ -25,10 +25,18 @@ __version__ = pkg_resources.get_distribution('ocrmypdf').version VERSION = __version__ from .exceptions import ( - ExitCode, BadArgsError, PdfMergeFailedError, MissingDependencyError, - UnsupportedImageFormatError, DpiError, OutputFileAccessError, - PriorOcrFoundError, InputFileError, SubprocessOutputError, - EncryptedPdfError, TesseractConfigError + ExitCode, + BadArgsError, + PdfMergeFailedError, + MissingDependencyError, + UnsupportedImageFormatError, + DpiError, + OutputFileAccessError, + PriorOcrFoundError, + InputFileError, + SubprocessOutputError, + EncryptedPdfError, + TesseractConfigError, ) from . import helpers diff --git a/src/ocrmypdf/__main__.py b/src/ocrmypdf/__main__.py index db5db2e5..841fbad3 100755 --- a/src/ocrmypdf/__main__.py +++ b/src/ocrmypdf/__main__.py @@ -35,13 +35,18 @@ import ruffus.proxy_logger as proxy_logger from ._jobcontext import JobContext, JobContextManager, cleanup_working_files from ._pipeline import build_pipeline from .pdfa import file_claims_pdfa -from .helpers import re_symlink, is_file_writable, \ - available_cpu_count +from .helpers import re_symlink, is_file_writable, available_cpu_count from .exec import tesseract, qpdf, ghostscript from . import PROGRAM_NAME, VERSION -from .exceptions import ExitCode, ExitCodeException, MissingDependencyError, \ - InputFileError, BadArgsError, OutputFileAccessError +from .exceptions import ( + ExitCode, + ExitCodeException, + MissingDependencyError, + InputFileError, + BadArgsError, + OutputFileAccessError, +) from . import exceptions as ocrmypdf_exceptions from ._unicodefun import verify_python3_env @@ -49,9 +54,8 @@ from ._unicodefun import verify_python3_env # ------------- # External dependencies -HOCR_OK_LANGS = frozenset([ - 'eng', 'deu', 'spa', 'ita', 'por' -]) +HOCR_OK_LANGS = frozenset(['eng', 'deu', 'spa', 'ita', 'por']) + def complain(message): print(*textwrap.wrap(message), file=sys.stderr) @@ -69,24 +73,26 @@ verify_python3_env() if not tesseract.v4: complain( "Please install tesseract 4.0.0 or newer " - "(currently installed version is {1})".format( - tesseract.version())) + "(currently installed version is {1})".format(tesseract.version()) + ) sys.exit(ExitCode.missing_dependency) # ------------- # Parser + def numeric(basetype, min_=None, max_=None): """Validator for numeric params""" min_ = basetype(min_) if min_ is not None else None max_ = basetype(max_) if max_ is not None else None + def _numeric(string): value = basetype(string) - if (min_ is not None and value < min_ - or max_ is not None and value > max_): + if min_ is not None and value < min_ or max_ is not None and value > max_: msg = "%r not in valid range %r" % (string, (min_, max_)) raise argparse.ArgumentTypeError(msg) return value + _numeric.__name__ = basetype.__name__ return _numeric @@ -142,259 +148,366 @@ ocrmypdf so it is already installed. Online documentation is located at: https://ocrmypdf.readthedocs.io/en/latest/introduction.html -""") +""", +) parser.add_argument( - 'input_file', metavar="input_pdf_or_image", + 'input_file', + metavar="input_pdf_or_image", help="PDF file containing the images to be OCRed (or '-' to read from " - "standard input)") + "standard input)", +) parser.add_argument( - 'output_file', metavar="output_pdf", + 'output_file', + metavar="output_pdf", help="Output searchable PDF file (or '-' to write to standard output). " - "Existing files will be ovewritten. If same as input file, the " - "input file will be updated only if processing is successful.") + "Existing files will be ovewritten. If same as input file, the " + "input file will be updated only if processing is successful.", +) parser.add_argument( - '-l', '--language', action='append', + '-l', + '--language', + action='append', help="Language(s) of the file to be OCRed (see tesseract --list-langs for " - "all language packs installed in your system). Use -l eng+deu for " - "multiple languages.") + "all language packs installed in your system). Use -l eng+deu for " + "multiple languages.", +) parser.add_argument( - '--image-dpi', metavar='DPI', type=int, - help="For input image instead of PDF, use this DPI instead of file's.") + '--image-dpi', + metavar='DPI', + type=int, + help="For input image instead of PDF, use this DPI instead of file's.", +) parser.add_argument( - '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'], + '--output-type', + choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'], default='pdfa', help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for " - "long term archiving (default, recommended) but may not suitable " - "for users who want their file altered as little as possible. 'pdfa' " - "also has problems with full Unicode text. 'pdf' attempts to " - "preserve file contents as much as possible. 'pdf-a1' creates a " - "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'. 'pdf-a3' creates a " - "PDF/A3-b file." - ) + "long term archiving (default, recommended) but may not suitable " + "for users who want their file altered as little as possible. 'pdfa' " + "also has problems with full Unicode text. 'pdf' attempts to " + "preserve file contents as much as possible. 'pdf-a1' creates a " + "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'. 'pdf-a3' creates a " + "PDF/A3-b file.", +) # Use null string '\0' as sentinel to indicate the user supplied no argument, # since that is the only invalid character for filepaths on all platforms # bool('\0') is True in Python parser.add_argument( - '--sidecar', nargs='?', const='\0', default=None, metavar='FILE', + '--sidecar', + nargs='?', + const='\0', + default=None, + metavar='FILE', help="Generate sidecar text files that contain the same text recognized " - "by Tesseract. This may be useful for building a OCR text database. " - "If FILE is omitted, the sidecar file be named {output_file}.txt " - "If FILE is set to '-', the sidecar is written to stdout (a " - "convenient way to preview OCR quality). The output file and sidecar " - "may not both use stdout at the same time.") + "by Tesseract. This may be useful for building a OCR text database. " + "If FILE is omitted, the sidecar file be named {output_file}.txt " + "If FILE is set to '-', the sidecar is written to stdout (a " + "convenient way to preview OCR quality). The output file and sidecar " + "may not both use stdout at the same time.", +) parser.add_argument( - '--version', action='version', version=VERSION, - help="Print program version and exit") + '--version', + action='version', + version=VERSION, + help="Print program version and exit", +) -jobcontrol = parser.add_argument_group( - "Job control options") +jobcontrol = parser.add_argument_group("Job control options") jobcontrol.add_argument( - '-j', '--jobs', metavar='N', type=numeric(int, 0, 256), - help="Use up to N CPU cores simultaneously (default: use all).") + '-j', + '--jobs', + metavar='N', + type=numeric(int, 0, 256), + help="Use up to N CPU cores simultaneously (default: use all).", +) jobcontrol.add_argument( - '-q', '--quiet', action='store_true', help="Suppress INFO messages") + '-q', '--quiet', action='store_true', help="Suppress INFO messages" +) jobcontrol.add_argument( - '-v', '--verbose', const="+", default=[], nargs='?', action="append", - help="Print more verbose messages for each additional verbose level") + '-v', + '--verbose', + const="+", + default=[], + nargs='?', + action="append", + help="Print more verbose messages for each additional verbose level", +) metadata = parser.add_argument_group( "Metadata options", - "Set output PDF/A metadata (default: copy input document's metadata)") + "Set output PDF/A metadata (default: copy input document's metadata)", +) metadata.add_argument( - '--title', type=str, - help="Set document title (place multiple words in quotes)") -metadata.add_argument( - '--author', type=str, - help="Set document author") -metadata.add_argument( - '--subject', type=str, - help="Set document subject description") -metadata.add_argument( - '--keywords', type=str, - help="Set document keywords") + '--title', type=str, help="Set document title (place multiple words in quotes)" +) +metadata.add_argument('--author', type=str, help="Set document author") +metadata.add_argument('--subject', type=str, help="Set document subject description") +metadata.add_argument('--keywords', type=str, help="Set document keywords") preprocessing = parser.add_argument_group( "Image preprocessing options", - "Options to improve the quality of the final PDF and OCR") + "Options to improve the quality of the final PDF and OCR", +) preprocessing.add_argument( - '-r', '--rotate-pages', action='store_true', - help="Automatically rotate pages based on detected text orientation") + '-r', + '--rotate-pages', + action='store_true', + help="Automatically rotate pages based on detected text orientation", +) preprocessing.add_argument( - '--remove-background', action='store_true', + '--remove-background', + action='store_true', help="Attempt to remove background from gray or color pages, setting it " - "to white ") + "to white ", +) preprocessing.add_argument( - '-d', '--deskew', action='store_true', - help="Deskew each page before performing OCR") + '-d', '--deskew', action='store_true', help="Deskew each page before performing OCR" +) preprocessing.add_argument( - '-c', '--clean', action='store_true', + '-c', + '--clean', + action='store_true', help="Clean pages from scanning artifacts before performing OCR, and send " - "the cleaned page to OCR, but do not include the cleaned page in " - "the output") + "the cleaned page to OCR, but do not include the cleaned page in " + "the output", +) preprocessing.add_argument( - '-i', '--clean-final', action='store_true', + '-i', + '--clean-final', + action='store_true', help="Clean page as above, and incorporate the cleaned image in the final " - "PDF. Might remove desired content.") + "PDF. Might remove desired content.", +) preprocessing.add_argument( - '--oversample', metavar='DPI', type=numeric(int, 0, 5000), default=0, + '--oversample', + metavar='DPI', + type=numeric(int, 0, 5000), + default=0, help="Oversample images to at least the specified DPI, to improve OCR " - "results slightly") + "results slightly", +) preprocessing.add_argument( - '--remove-vectors', action='store_true', + '--remove-vectors', + action='store_true', help="EXPERIMENTAL. Mask out any vector objects in the PDF so that they " - "will not be included in OCR. This can eliminate false characters.") + "will not be included in OCR. This can eliminate false characters.", +) preprocessing.add_argument( - '--mask-barcodes', action='store_true', + '--mask-barcodes', + action='store_true', help="EXPERIMENTAL. Mask out any barcodes that appear in the PDF so they are not " - "considered during OCR. Barcodes can introduce false characters into " - "OCR.") + "considered during OCR. Barcodes can introduce false characters into " + "OCR.", +) preprocessing.add_argument( - '--threshold', action='store_true', + '--threshold', + action='store_true', help="EXPERIMENTAL. Threshold image to 1bpp before sending it to Tesseract for OCR. Can " - "improve OCR quality compared to Tesseract's thresholder.") + "improve OCR quality compared to Tesseract's thresholder.", +) -ocrsettings = parser.add_argument_group( - "OCR options", - "Control how OCR is applied") +ocrsettings = parser.add_argument_group("OCR options", "Control how OCR is applied") ocrsettings.add_argument( - '-f', '--force-ocr', action='store_true', + '-f', + '--force-ocr', + action='store_true', help="Rasterize any text or vector objects on each page, apply OCR, and " - "save the rastered output (this rewrites the PDF)") + "save the rastered output (this rewrites the PDF)", +) ocrsettings.add_argument( - '-s', '--skip-text', action='store_true', + '-s', + '--skip-text', + action='store_true', help="Skip OCR on any pages that already contain text, but include the " - "page in final output; useful for PDFs that contain a mix of " - "images, text pages, and/or previously OCRed pages") + "page in final output; useful for PDFs that contain a mix of " + "images, text pages, and/or previously OCRed pages", +) ocrsettings.add_argument( - '--redo-ocr', action='store_true', + '--redo-ocr', + action='store_true', help="Attempt to detect and remove the hidden OCR layer from files that " - "were previously OCRed with OCRmyPDF or another program. Apply OCR " - "to text found in raster images. Existing visible text objects will " - "not be changed. If there is no existing OCR, OCR will be added.") + "were previously OCRed with OCRmyPDF or another program. Apply OCR " + "to text found in raster images. Existing visible text objects will " + "not be changed. If there is no existing OCR, OCR will be added.", +) ocrsettings.add_argument( - '--skip-big', type=numeric(float, 0, 5000), metavar='MPixels', + '--skip-big', + type=numeric(float, 0, 5000), + metavar='MPixels', help="Skip OCR on pages larger than the specified amount of megapixels, " - "but include skipped pages in final output") + "but include skipped pages in final output", +) optimizing = parser.add_argument_group( - "Optimization options", - "Control how the PDF is optimized after OCR" + "Optimization options", "Control how the PDF is optimized after OCR" ) optimizing.add_argument( - '-O', '--optimize', type=int, choices=range(0, 4), default=1, - help=("Control how PDF is optimized after processing:" + '-O', + '--optimize', + type=int, + choices=range(0, 4), + default=1, + help=( + "Control how PDF is optimized after processing:" "0 - do not optimize; " "1 - do safe, lossless optimizations (default); " "2 - do some lossy optimizations; " "3 - do aggressive lossy optimizations (including lossy JBIG2)" - ) + ), ) optimizing.add_argument( - '--jpeg-quality', type=numeric(int, 0, 100), default=0, metavar='Q', - help=("Adjust JPEG quality level for JPEG optimization. " - "100 is best quality and largest output size; " - "1 is lowest quality and smallest output; " - "0 uses the default." - ) + '--jpeg-quality', + type=numeric(int, 0, 100), + default=0, + metavar='Q', + help=( + "Adjust JPEG quality level for JPEG optimization. " + "100 is best quality and largest output size; " + "1 is lowest quality and smallest output; " + "0 uses the default." + ), ) optimizing.add_argument( - '--jpg-quality', type=numeric(int, 0, 100), default=0, metavar='Q', + '--jpg-quality', + type=numeric(int, 0, 100), + default=0, + metavar='Q', dest='jpeg_quality', - help=argparse.SUPPRESS # Alias for --jpeg-quality + help=argparse.SUPPRESS, # Alias for --jpeg-quality ) optimizing.add_argument( - '--png-quality', type=numeric(int, 0, 100), default=0, metavar='Q', - help=("Adjust PNG quality level to use when quantizing PNGs. " - "Values have same meaning as with --jpeg-quality" - ) + '--png-quality', + type=numeric(int, 0, 100), + default=0, + metavar='Q', + help=( + "Adjust PNG quality level to use when quantizing PNGs. " + "Values have same meaning as with --jpeg-quality" + ), ) optimizing.add_argument( - '--jbig2-lossy', action='store_true', - help=("Enable JBIG2 lossy mode (better compression, not suitable for some " - "use cases - see documentation)." - ) + '--jbig2-lossy', + action='store_true', + help=( + "Enable JBIG2 lossy mode (better compression, not suitable for some " + "use cases - see documentation)." + ), ) optimizing.add_argument( - '--jbig2-page-group-size', type=numeric(int, 1, 10000), default=0, + '--jbig2-page-group-size', + type=numeric(int, 1, 10000), + default=0, metavar='N', # Adjust number of pages to consider at once for JBIG2 compression - help=argparse.SUPPRESS + help=argparse.SUPPRESS, ) advanced = parser.add_argument_group( - "Advanced", - "Advanced options to control Tesseract's OCR behavior") + "Advanced", "Advanced options to control Tesseract's OCR behavior" +) advanced.add_argument( - '--max-image-mpixels', action='store', type=numeric(float, 0), + '--max-image-mpixels', + action='store', + type=numeric(float, 0), metavar='MPixels', help="Set maximum number of pixels to unpack before treating an image as a " - "decompression bomb", - default=128.0) + "decompression bomb", + default=128.0, +) advanced.add_argument( - '--tesseract-config', action='append', metavar='CFG', default=[], - help="Additional Tesseract configuration files -- see documentation") + '--tesseract-config', + action='append', + metavar='CFG', + default=[], + help="Additional Tesseract configuration files -- see documentation", +) advanced.add_argument( - '--tesseract-pagesegmode', action='store', type=int, metavar='PSM', + '--tesseract-pagesegmode', + action='store', + type=int, + metavar='PSM', choices=range(0, 14), - help="Set Tesseract page segmentation mode (see tesseract --help)") + help="Set Tesseract page segmentation mode (see tesseract --help)", +) advanced.add_argument( - '--tesseract-oem', action='store', type=int, metavar='MODE', + '--tesseract-oem', + action='store', + type=int, + metavar='MODE', choices=range(0, 4), - help=("Set Tesseract 4.0 OCR engine mode: " - "0 - original Tesseract only; " - "1 - neural nets LSTM only; " - "2 - Tesseract + LSTM; " - "3 - default.") - ) + help=( + "Set Tesseract 4.0 OCR engine mode: " + "0 - original Tesseract only; " + "1 - neural nets LSTM only; " + "2 - Tesseract + LSTM; " + "3 - default." + ), +) advanced.add_argument( '--pdf-renderer', - choices=['auto', 'hocr', 'sandwich'], default='auto', + choices=['auto', 'hocr', 'sandwich'], + default='auto', help="Choose OCR PDF renderer - the default option is to let OCRmyPDF " - "choose. See documentation for discussion." - ) + "choose. See documentation for discussion.", +) advanced.add_argument( - '--tesseract-timeout', default=180.0, type=numeric(float, 0), + '--tesseract-timeout', + default=180.0, + type=numeric(float, 0), metavar='SECONDS', help='Give up on OCR after the timeout, but copy the preprocessed page ' - 'into the final output') + 'into the final output', +) advanced.add_argument( - '--rotate-pages-threshold', default=14.0, type=numeric(float, 0, 1000), + '--rotate-pages-threshold', + default=14.0, + type=numeric(float, 0, 1000), metavar='CONFIDENCE', help="Only rotate pages when confidence is above this value (arbitrary " - "units reported by tesseract)") + "units reported by tesseract)", +) advanced.add_argument( - '--pdfa-image-compression', choices=['auto', 'jpeg', 'lossless'], + '--pdfa-image-compression', + choices=['auto', 'jpeg', 'lossless'], default='auto', help="Specify how to compress images in the output PDF/A. 'auto' lets " - "OCRmyPDF decide. 'jpeg' changes all grayscale and color images to " - "JPEG compression. 'lossless' uses PNG-style lossless compression " - "for all images. Monochrome images are always compressed using a " - "lossless codec. Compression settings " - "are applied to all pages, including those for which OCR was " - "skipped. Not supported for --output-type=pdf ; that setting " - "preserves the original compression of all images.") + "OCRmyPDF decide. 'jpeg' changes all grayscale and color images to " + "JPEG compression. 'lossless' uses PNG-style lossless compression " + "for all images. Monochrome images are always compressed using a " + "lossless codec. Compression settings " + "are applied to all pages, including those for which OCR was " + "skipped. Not supported for --output-type=pdf ; that setting " + "preserves the original compression of all images.", +) advanced.add_argument( - '--user-words', metavar='FILE', + '--user-words', + metavar='FILE', help="Specify the location of the Tesseract user words file. This is a " - "list of words Tesseract should consider while performing OCR in " - "addition to its standard language dictionaries. This can improve " - "OCR quality especially for specialized and technical documents.") + "list of words Tesseract should consider while performing OCR in " + "addition to its standard language dictionaries. This can improve " + "OCR quality especially for specialized and technical documents.", +) advanced.add_argument( - '--user-patterns', metavar='FILE', - help="Specify the location of the Tesseract user patterns file.") + '--user-patterns', + metavar='FILE', + help="Specify the location of the Tesseract user patterns file.", +) debugging = parser.add_argument_group( - "Debugging", - "Arguments to help with troubleshooting and debugging") + "Debugging", "Arguments to help with troubleshooting and debugging" +) debugging.add_argument( - '-k', '--keep-temporary-files', action='store_true', - help="Keep temporary files (helpful for debugging)") + '-k', + '--keep-temporary-files', + action='store_true', + help="Keep temporary files (helpful for debugging)", +) debugging.add_argument( - '--flowchart', type=str, - help="Generate the pipeline execution flowchart") + '--flowchart', type=str, help="Generate the pipeline execution flowchart" +) def check_options_languages(options, _log): @@ -409,8 +522,9 @@ def check_options_languages(options, _log): if not languages.issubset(tesseract.languages()): msg = ( "The installed version of tesseract does not have language " - "data for the following requested languages: \n") - for lang in (languages - tesseract.languages()): + "data for the following requested languages: \n" + ) + for lang in languages - tesseract.languages(): msg += lang + '\n' raise MissingDependencyError(msg) @@ -427,18 +541,18 @@ def check_options_output(options, log): msg = ( "The 'hocr' PDF renderer is known to cause problems with one " "or more of the languages in your document. Use " - "--pdf-renderer auto (the default) to avoid this issue.") + "--pdf-renderer auto (the default) to avoid this issue." + ) log.warning(msg) - if ghostscript.version() < '9.20' \ - and options.output_type != 'pdf' \ - and not is_latin: + if ghostscript.version() < '9.20' and options.output_type != 'pdf' and not is_latin: # https://bugs.ghostscript.com/show_bug.cgi?id=696874 # Ghostscript < 9.20 fails to encode multibyte characters properly msg = ( "The installed version of Ghostscript does not work correctly " "with the OCR languages you specified. Use --output-type pdf or " - "upgrade to Ghostscript 9.20 or later to avoid this issue.") + "upgrade to Ghostscript 9.20 or later to avoid this issue." + ) msg += "Found Ghostscript {}".format(ghostscript.version()) log.warning(msg) @@ -455,8 +569,14 @@ def check_options_output(options, log): ) lossless_reconstruction = False - if not any((options.deskew, options.clean_final, options.force_ocr, - options.remove_background)): + if not any( + ( + options.deskew, + options.clean_final, + options.force_ocr, + options.remove_background, + ) + ): lossless_reconstruction = True options.lossless_reconstruction = lossless_reconstruction @@ -464,7 +584,7 @@ def check_options_output(options, log): raise argparse.ArgumentError( None, "--redo-ocr is not currently compatible with --deskew, " - "--clean-final, and --remove-background" + "--clean-final, and --remove-background", ) @@ -473,8 +593,8 @@ def check_options_sidecar(options, log): if options.output_file == '-': raise argparse.ArgumentError( None, - "--sidecar filename must be specified when output file is " - "stdout.") + "--sidecar filename must be specified when output file is " "stdout.", + ) options.sidecar = options.output_file + '.txt' @@ -483,10 +603,12 @@ def _optional_program_required(name, version_fn, min_version, for_argument): if version_fn() < min_version: raise MissingDependencyError( "The installed '{}' is not supported. " - "Install version {} or newer.".format(name, min_version)) + "Install version {} or newer.".format(name, min_version) + ) except (FileNotFoundError, MissingDependencyError): raise MissingDependencyError( - "Install the '{}' program to use {}.".format(name, for_argument)) + "Install the '{}' program to use {}.".format(name, for_argument) + ) def _optional_program_recommended(name, version_fn, min_version, for_argument): @@ -494,7 +616,8 @@ def _optional_program_recommended(name, version_fn, min_version, for_argument): if version_fn() < min_version: raise MissingDependencyError( "The installed '{}' is not supported. " - "Install version {} or newer.".format(name, min_version)) + "Install version {} or newer.".format(name, min_version) + ) except (FileNotFoundError, MissingDependencyError): complain( "For best results, install the optional program '{}' to use the " @@ -505,6 +628,7 @@ def _optional_program_recommended(name, version_fn, min_version, for_argument): def check_options_preprocessing(options, log): if any((options.clean, options.clean_final)): from .exec import unpaper + _optional_program_required( 'unpaper', unpaper.version, '6.1', '--clean, --clean-final' ) @@ -512,27 +636,27 @@ def check_options_preprocessing(options, log): def check_options_ocr_behavior(options, log): exclusive_options = sum( - [(1 if opt else 0) - for opt in (options.force_ocr, options.skip_text, options.redo_ocr) + [ + (1 if opt else 0) + for opt in (options.force_ocr, options.skip_text, options.redo_ocr) ] ) if exclusive_options >= 2: raise argparse.ArgumentError( - None, - "Error: choose only one of --force-ocr, --skip-text, --redo-ocr.") + None, "Error: choose only one of --force-ocr, --skip-text, --redo-ocr." + ) def check_options_optimizing(options, log): if options.optimize >= 2: from .exec import pngquant, jbig2enc + _optional_program_required( 'pngquant', pngquant.version, '2.0.1', '--optimize {2,3}' ) if options.jbig2_lossy: - _optional_program_required( - 'jbig2', jbig2enc.version, '0.28', '--jbig2-lossy' - ) + _optional_program_required('jbig2', jbig2enc.version, '0.28', '--jbig2-lossy') elif options.optimize >= 2: # Although we use JBIG2 for optimize=1, don't nag about it unless the # user is asking for more optimization @@ -540,9 +664,9 @@ def check_options_optimizing(options, log): 'jbig2', jbig2enc.version, '0.28', '--optimize {2,3}' ) - if options.optimize == 0 and any([ - options.jbig2_lossy, options.png_quality, options.jpeg_quality - ]): + if options.optimize == 0 and any( + [options.jbig2_lossy, options.png_quality, options.jpeg_quality] + ): log.warning( "The arguments --jbig2-lossy, --png-quality, and --jpeg-quality " "will be ignored because --optimize=0." @@ -551,24 +675,23 @@ def check_options_optimizing(options, log): def check_options_advanced(options, log): if options.tesseract_oem and not tesseract.v4(): - log.warning( - "--tesseract-oem requires Tesseract 4.x -- argument ignored") - if options.pdfa_image_compression != 'auto' and \ - options.output_type.startswith('pdfa'): + log.warning("--tesseract-oem requires Tesseract 4.x -- argument ignored") + if options.pdfa_image_compression != 'auto' and options.output_type.startswith( + 'pdfa' + ): log.warning( "--pdfa-image-compression argument has no effect when " "--output-type is not 'pdfa', 'pdfa-1', or 'pdfa-2'" ) if tesseract.v4() and (options.user_words or options.user_patterns): - log.warning( - 'Tesseract 4.x ignores --user-words, so this has no effect') + log.warning('Tesseract 4.x ignores --user-words, so this has no effect') def check_options_metadata(options, log): import unicodedata - docinfo = [options.title, options.author, options.keywords, - options.subject] + + docinfo = [options.title, options.author, options.keywords, options.subject] for s in (m for m in docinfo if m): for c in s: if unicodedata.category(c) == 'Co' or ord(c) >= 0x10000: @@ -576,11 +699,12 @@ def check_options_metadata(options, log): "One of the metadata strings contains " "an unsupported Unicode character: '{}' (U+{})".format( c, hex(ord(c))[2:].upper() - )) + ) + ) def check_options_pillow(options, log): - PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1000000) + PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1_000_000) if PIL.Image.MAX_IMAGE_PIXELS == 0: PIL.Image.MAX_IMAGE_PIXELS = None @@ -686,8 +810,10 @@ def do_ruffus_exception(ruffus_five_tuple, options, log): log.error(exc_msg) elif exc_name == 'PIL.Image.DecompressionBombError': msg = cleanup_ruffus_error_message(exc_value) - msg += ("\nUse the --max-image-mpixels argument to set increase the " - "maximum number of megapixels to accept.") + msg += ( + "\nUse the --max-image-mpixels argument to set increase the " + "maximum number of megapixels to accept." + ) log.error(msg) exit_code = ExitCode.input_file @@ -753,8 +879,7 @@ def check_closed_streams(options): if sys.stdin is None: if options.input_file == '-': - print("Trying to read from stdin but stdin seems closed", - file=sys.stderr) + print("Trying to read from stdin but stdin seems closed", file=sys.stderr) return False sys.stdin = open(os.devnull, 'r') @@ -763,10 +888,15 @@ def check_closed_streams(options): # Can't replace stdout if the user is piping # If this case can even happen, it must be some kind of weird # stream. - print(textwrap.dedent("""\ + print( + textwrap.dedent( + """\ Output was set to stdout '-' but the stream attached to stdout does not support the flush() system call. This - will fail."""), file=sys.stderr) + will fail.""" + ), + file=sys.stderr, + ) return False sys.stdout = open(os.devnull, 'w') @@ -774,15 +904,12 @@ def check_closed_streams(options): def log_page_orientations(pdfinfo, _log): - direction = {0: 'n', 90: 'e', - 180: 's', 270: 'w'} + direction = {0: 'n', 90: 'e', 180: 's', 270: 'w'} orientations = [] for n, page in enumerate(pdfinfo): angle = page.rotation or 0 if angle != 0: - orientations.append('{0}{1}'.format( - n + 1, - direction.get(angle, ''))) + orientations.append('{0}{1}'.format(n + 1, direction.get(angle, ''))) if orientations: _log.info('Page orientations detected: ' + ' '.join(orientations)) @@ -799,12 +926,19 @@ def check_environ(options, _log): 'OCRMYPDF_TESSERACT', 'OCRMYPDF_QPDF', 'OCRMYPDF_GS', - 'OCRMYPDF_UNPAPER') + 'OCRMYPDF_UNPAPER', + ) for k in old_envvars: if k in os.environ: - _log.warning(textwrap.dedent("""\ + _log.warning( + textwrap.dedent( + """\ OCRmyPDF no longer uses the environment variable {}. - Change PATH to select alternate programs.""".format(k))) + Change PATH to select alternate programs.""".format( + k + ) + ) + ) def check_input_file(options, _log, start_input_file): @@ -813,6 +947,7 @@ def check_input_file(options, _log, start_input_file): _log.info('reading file from standard input') with open(start_input_file, 'wb') as stream_buffer: from shutil import copyfileobj + copyfileobj(sys.stdin.buffer, stream_buffer) else: try: @@ -825,15 +960,22 @@ def check_input_file(options, _log, start_input_file): def check_requested_output_file(options, _log): if options.output_file == '-': if sys.stdout.isatty(): - _log.error(textwrap.dedent("""\ + _log.error( + textwrap.dedent( + """\ Output was set to stdout '-' but it looks like stdout is connected to a terminal. Please redirect stdout to a - file.""")) + file.""" + ) + ) raise BadArgsError() elif not is_file_writable(options.output_file): _log.error( - "Output file location (" + options.output_file + ") " + - "is not a writable file.") + "Output file location (" + + options.output_file + + ") " + + "is not a writable file." + ) raise OutputFileAccessError() @@ -853,27 +995,33 @@ def report_output_file_size(options, _log, input_file, output_file): 'clean_final', 'remove_background', 'oversample', - 'force_ocr' + 'force_ocr', } for arg in image_preproc: attr = getattr(options, arg, None) if not attr: continue reasons.append( - "The argument --{} was issued, causing transcoding.".format( - arg.replace('_', '-'))) + "The argument --{} was issued, causing transcoding.".format( + arg.replace('_', '-') + ) + ) if reasons: - explanation = ( - "Possible reasons for this include:\n" + '\n'.join(reasons) + "\n") + explanation = "Possible reasons for this include:\n" + '\n'.join(reasons) + "\n" else: - explanation = ( - "No reason for this increase is known. Please report this issue.") + explanation = "No reason for this increase is known. Please report this issue." - _log.warning(textwrap.dedent("""\ + _log.warning( + textwrap.dedent( + """\ The output file size is {:.2f}× larger than the input file. {} - """.format(ratio, explanation))) + """.format( + ratio, explanation + ) + ) + ) def run_pipeline(args=None): @@ -888,7 +1036,8 @@ def run_pipeline(args=None): logger_args = {'verbose': options.verbose, 'quiet': options.quiet} _log, _log_mutex = proxy_logger.make_shared_logger_and_proxy( - logging_factory, __name__, logger_args) + logging_factory, __name__, logger_args + ) preamble(_log) check_options(options, _log) @@ -899,7 +1048,8 @@ def run_pipeline(args=None): complain( "You are using qpdf version {0} which has known issues including " "security vulnerabilities with certain malformed PDFs. Consider " - "upgrading to version 7.0.0 or newer.".format(qpdf.version())) + "upgrading to version 7.0.0 or newer.".format(qpdf.version()) + ) if ghostscript.version() == '9.24': complain( @@ -927,10 +1077,8 @@ def run_pipeline(args=None): try: work_folder = mkdtemp(prefix="com.github.ocrmypdf.") - options.history_file = os.path.join( - work_folder, 'ruffus_history.sqlite') - start_input_file = os.path.join( - work_folder, 'origin') + options.history_file = os.path.join(work_folder, 'ruffus_history.sqlite') + start_input_file = os.path.join(work_folder, 'origin') check_input_file(options, _log, start_input_file) check_requested_output_file(options, _log) @@ -983,12 +1131,12 @@ def run_pipeline(args=None): _log.warning('Output file: The generated PDF is INVALID') return ExitCode.invalid_output_pdf - report_output_file_size(options, _log, start_input_file, - options.output_file) + report_output_file_size(options, _log, start_input_file, options.output_file) pdfinfo = context.get_pdfinfo() if options.verbose: from pprint import pformat + _log.debug(pformat(pdfinfo)) log_page_orientations(pdfinfo, _log) diff --git a/src/ocrmypdf/_jobcontext.py b/src/ocrmypdf/_jobcontext.py index 7d21c213..95c92119 100644 --- a/src/ocrmypdf/_jobcontext.py +++ b/src/ocrmypdf/_jobcontext.py @@ -77,8 +77,10 @@ class JobContextManager(SyncManager): def cleanup_working_files(work_folder, options): if options.keep_temporary_files: - print("Temporary working files saved at:\n{0}".format(work_folder), - file=sys.stderr) + print( + "Temporary working files saved at:\n{0}".format(work_folder), + file=sys.stderr, + ) else: with suppress(FileNotFoundError): shutil.rmtree(work_folder) diff --git a/src/ocrmypdf/_pipeline.py b/src/ocrmypdf/_pipeline.py index b5685201..0e2c0eab 100644 --- a/src/ocrmypdf/_pipeline.py +++ b/src/ocrmypdf/_pipeline.py @@ -36,8 +36,13 @@ from .pdfinfo import PdfInfo, Colorspace from .pdfa import generate_pdfa_ps from .helpers import re_symlink, is_iterable_notstr, page_number, flatten_groups from .exec import ghostscript, tesseract -from .exceptions import UnsupportedImageFormatError, \ - DpiError, PriorOcrFoundError, InputFileError, EncryptedPdfError +from .exceptions import ( + UnsupportedImageFormatError, + DpiError, + PriorOcrFoundError, + InputFileError, + EncryptedPdfError, +) from . import leptonica from . import PROGRAM_NAME, VERSION from .optimize import optimize @@ -50,6 +55,7 @@ VECTOR_PAGE_DPI = 400 # The Pipeline # + def triage_image_file(input_file, output_file, log, options): try: log.info("Input file is not a PDF, checking if it is an image...") @@ -76,14 +82,16 @@ def triage_image_file(input_file, output_file, log, options): log.error( "Input file is an image, but the resolution (DPI) is " "not credible. Estimate the resolution at which the " - "image was scanned and specify it using --image-dpi.") + "image was scanned and specify it using --image-dpi." + ) raise DpiError() elif not options.image_dpi: log.info("Image size: (%d, %d)" % im.size) log.error( "Input file is an image, but has no resolution (DPI) " "in its metadata. Estimate the resolution at which " - "image was scanned and specify it using --image-dpi.") + "image was scanned and specify it using --image-dpi." + ) raise DpiError() if im.mode in ('RGBA', 'LA'): @@ -106,13 +114,12 @@ def triage_image_file(input_file, output_file, log, options): layout_fun = img2pdf.default_layout_fun if options.image_dpi: layout_fun = img2pdf.get_fixed_dpi_layout_fun( - (options.image_dpi, options.image_dpi)) + (options.image_dpi, options.image_dpi) + ) with open(output_file, 'wb') as outf: img2pdf.convert( - input_file, - layout_fun=layout_fun, - with_pdfrw=False, - outputstream=outf) + input_file, layout_fun=layout_fun, with_pdfrw=False, outputstream=outf + ) log.info("Successfully converted to PDF, processing...") except img2pdf.ImageOpenError as e: log.error(e) @@ -135,18 +142,16 @@ def _pdf_guess_version(input_file, search_window=1024): return '' -def triage( - input_file, - output_file, - log, - context): +def triage(input_file, output_file, log, context): options = context.get_options() try: if _pdf_guess_version(input_file): if options.image_dpi: - log.warning("Argument --image-dpi ignored because the " - "input file is a PDF, not an image.") + log.warning( + "Argument --image-dpi ignored because the " + "input file is a PDF, not an image." + ) re_symlink(input_file, output_file, log) return except EnvironmentError as e: @@ -156,11 +161,7 @@ def triage( triage_image_file(input_file, output_file, log, options) -def repair_and_parse_pdf( - input_file, - output_file, - log, - context): +def repair_and_parse_pdf(input_file, output_file, log, context): options = context.get_options() copyfile(input_file, output_file) @@ -169,7 +170,9 @@ def repair_and_parse_pdf( detailed_page_analysis = True try: - pdfinfo = PdfInfo(output_file, detailed_page_analysis=detailed_page_analysis, log=log) + pdfinfo = PdfInfo( + output_file, detailed_page_analysis=detailed_page_analysis, log=log + ) except pikepdf.PasswordError as e: raise EncryptedPdfError() except pikepdf.PdfError as e: @@ -225,12 +228,16 @@ def get_pageinfo(input_file, context): def get_page_dpi(pageinfo, options): "Get the DPI when nonsquare DPI is tolerable" - xres = max(pageinfo.xres or VECTOR_PAGE_DPI, - options.oversample or 0, - VECTOR_PAGE_DPI if pageinfo.has_vector else 0) - yres = max(pageinfo.yres or VECTOR_PAGE_DPI, - options.oversample or 0, - VECTOR_PAGE_DPI if pageinfo.has_vector else 0) + xres = max( + pageinfo.xres or VECTOR_PAGE_DPI, + options.oversample or 0, + VECTOR_PAGE_DPI if pageinfo.has_vector else 0, + ) + yres = max( + pageinfo.yres or VECTOR_PAGE_DPI, + options.oversample or 0, + VECTOR_PAGE_DPI if pageinfo.has_vector else 0, + ) return (float(xres), float(yres)) @@ -239,20 +246,26 @@ def get_page_square_dpi(pageinfo, options): xres = pageinfo.xres or 0 yres = pageinfo.yres or 0 userunit = pageinfo.userunit or 1 - return float(max( - (xres * userunit) or VECTOR_PAGE_DPI, - (yres * userunit) or VECTOR_PAGE_DPI, - VECTOR_PAGE_DPI if pageinfo.has_vector else 0, - options.oversample or 0)) + return float( + max( + (xres * userunit) or VECTOR_PAGE_DPI, + (yres * userunit) or VECTOR_PAGE_DPI, + VECTOR_PAGE_DPI if pageinfo.has_vector else 0, + options.oversample or 0, + ) + ) def get_canvas_square_dpi(pageinfo, options): """Get the DPI when we require xres == yres, in Postscript units""" - return float(max( - (pageinfo.xres) or VECTOR_PAGE_DPI, - (pageinfo.yres) or VECTOR_PAGE_DPI, - VECTOR_PAGE_DPI if pageinfo.has_vector else 0, - options.oversample or 0)) + return float( + max( + (pageinfo.xres) or VECTOR_PAGE_DPI, + (pageinfo.yres) or VECTOR_PAGE_DPI, + VECTOR_PAGE_DPI if pageinfo.has_vector else 0, + options.oversample or 0, + ) + ) def is_ocr_required(pageinfo, log, options): @@ -263,28 +276,26 @@ def is_ocr_required(pageinfo, log, options): msg = "{0:4d}: page already has text! – {1}" if not options.force_ocr and not (options.skip_text or options.redo_ocr): - log.error(msg.format(page, - "aborting (use --force-ocr to force OCR)")) + log.error(msg.format(page, "aborting (use --force-ocr to force OCR)")) raise PriorOcrFoundError() elif options.force_ocr: - log.info(msg.format(page, - "rasterizing text and running OCR anyway")) + log.info(msg.format(page, "rasterizing text and running OCR anyway")) ocr_required = True elif options.redo_ocr: if pageinfo.has_corrupt_text: - log.warning(msg.format( - page, - "some text on this page cannot be mapped to characters: " - "consider using --force-ocr instead") + log.warning( + msg.format( + page, + "some text on this page cannot be mapped to characters: " + "consider using --force-ocr instead", + ) ) raise PriorOcrFoundError() # Wrong error but will do for now else: - log.info(msg.format(page, - "redoing OCR")) + log.info(msg.format(page, "redoing OCR")) ocr_required = True elif options.skip_text: - log.info(msg.format(page, - "skipping all processing on this page")) + log.info(msg.format(page, "skipping all processing on this page")) ocr_required = False elif not pageinfo.images and not options.lossless_reconstruction: # We found a page with no images and no text. That means it may @@ -300,7 +311,9 @@ def is_ocr_required(pageinfo, log, options): "{0:4d}: page has no images - " "rasterizing at {1} DPI because " "--force-ocr --oversample was specified".format( - page, options.oversample)) + page, options.oversample + ) + ) elif options.force_ocr: # Warn the user they might not want to do this log.warning( @@ -308,31 +321,31 @@ def is_ocr_required(pageinfo, log, options): "all vector content will be " "rasterized at {1} DPI, losing some resolution and likely " "increasing file size. Use --oversample to adjust the " - "DPI.".format(page, VECTOR_PAGE_DPI)) + "DPI.".format(page, VECTOR_PAGE_DPI) + ) else: log.info( "{0:4d}: page has no images - " "skipping all processing on this page to avoid losing detail. " "Use --force-ocr if you wish to perform OCR on pages that " - "have vector content.".format(page)) + "have vector content.".format(page) + ) ocr_required = False if ocr_required and options.skip_big and pageinfo.images: pixel_count = pageinfo.width_pixels * pageinfo.height_pixels - if pixel_count > (options.skip_big * 1000000): + if pixel_count > (options.skip_big * 1_000_000): ocr_required = False log.warning( "{0:4d}: page too big, skipping OCR " "({1:.1f} MPixels > {2:.1f} MPixels --skip-big)".format( - page, pixel_count / 1000000, options.skip_big)) + page, pixel_count / 1_000_000, options.skip_big + ) + ) return ocr_required -def marker_pages( - input_files, - output_files, - log, - context): +def marker_pages(input_files, output_files, log, context): options = context.get_options() work_folder = context.get_work_folder() @@ -348,8 +361,7 @@ def marker_pages( # If no files were repaired the input will be empty if not input_file: - log.error("{0}: file not found or invalid argument".format( - options.input_file)) + log.error("{0}: file not found or invalid argument".format(options.input_file)) raise InputFileError() pdfinfo = context.get_pdfinfo() @@ -362,51 +374,46 @@ def marker_pages( page.symlink_to(input_file) # pylint: disable=E1101 -def ocr_or_skip( - input_files, - output_files, - log, - context): +def ocr_or_skip(input_files, output_files, log, context): options = context.get_options() work_folder = context.get_work_folder() - pdfinfo = context.get_pdfinfo() + pdfinfo = context.get_pdfinfo() for input_file in input_files: pageno = page_number(input_file) - 1 pageinfo = pdfinfo[pageno] - alt_suffix = \ - '.ocr.page.pdf' if is_ocr_required(pageinfo, log, options) \ + alt_suffix = ( + '.ocr.page.pdf' + if is_ocr_required(pageinfo, log, options) else '.skip.page.pdf' + ) re_symlink( input_file, - os.path.join( - work_folder, - os.path.basename(input_file)[0:6] + alt_suffix), - log) + os.path.join(work_folder, os.path.basename(input_file)[0:6] + alt_suffix), + log, + ) -def rasterize_preview( - input_file, - output_file, - log, - context): +def rasterize_preview(input_file, output_file, log, context): pageinfo = get_pageinfo(input_file, context) options = context.get_options() canvas_dpi = get_canvas_square_dpi(pageinfo, options) page_dpi = get_page_square_dpi(pageinfo, options) ghostscript.rasterize_pdf( - input_file, output_file, xres=canvas_dpi, yres=canvas_dpi, - raster_device='jpeggray', log=log, page_dpi=(page_dpi, page_dpi), - pageno=page_number(input_file)) - - -def orient_page( - infiles, + input_file, output_file, - log, - context): + xres=canvas_dpi, + yres=canvas_dpi, + raster_device='jpeggray', + log=log, + page_dpi=(page_dpi, page_dpi), + pageno=page_number(input_file), + ) + + +def orient_page(infiles, output_file, log, context): """ Work out orientation correct for each page. @@ -436,14 +443,10 @@ def orient_page( preview, engine_mode=options.tesseract_oem, timeout=options.tesseract_timeout, - log=log) + log=log, + ) - direction = { - 0: '⇧', - 90: '⇨', - 180: '⇩', - 270: '⇦' - } + direction = {0: '⇧', 90: '⇨', 180: '⇩', 270: '⇦'} pageno = page_number(page_pdf) - 1 pdfinfo = context.get_pdfinfo() @@ -467,17 +470,18 @@ def orient_page( facing = '' if existing_rotation != 0: - facing = 'with existing rotation {}, '.format(direction.get( - existing_rotation, '?')) - facing += 'page is facing {}'.format(direction.get( - orient_conf.angle, '?')) + facing = 'with existing rotation {}, '.format( + direction.get(existing_rotation, '?') + ) + facing += 'page is facing {}'.format(direction.get(orient_conf.angle, '?')) log.info( '{pagenum:4d}: {facing}, confidence {conf:.2f}{action}'.format( pagenum=page_number(preview), facing=facing, conf=orient_conf.confidence, - action=action) + action=action, + ) ) re_symlink(page_pdf, output_file, log) @@ -485,16 +489,13 @@ def orient_page( context.set_rotation(pageno, correction) -def rasterize_with_ghostscript( - input_file, - output_file, - log, - context): +def rasterize_with_ghostscript(input_file, output_file, log, context): options = context.get_options() pageinfo = get_pageinfo(input_file, context) colorspaces = ['pngmono', 'pnggray', 'png256', 'png16m'] device_idx = 0 + def at_least(cs): return max(device_idx, colorspaces.index(cs)) @@ -511,8 +512,7 @@ def rasterize_with_ghostscript( device = colorspaces[device_idx] - log.debug("Rasterize {0} with {1}".format( - os.path.basename(input_file), device)) + log.debug("Rasterize {0} with {1}".format(os.path.basename(input_file), device)) # Produce the page image with square resolution or else deskew and OCR # will not work properly. @@ -522,17 +522,20 @@ def rasterize_with_ghostscript( correction = context.get_rotation(page_number(input_file) - 1) ghostscript.rasterize_pdf( - input_file, output_file, xres=canvas_dpi, yres=canvas_dpi, - raster_device=device, log=log, page_dpi=(page_dpi, page_dpi), - pageno=page_number(input_file), rotation=correction, - filter_vector=options.remove_vectors) - - -def preprocess_remove_background( input_file, output_file, - log, - context): + xres=canvas_dpi, + yres=canvas_dpi, + raster_device=device, + log=log, + page_dpi=(page_dpi, page_dpi), + pageno=page_number(input_file), + rotation=correction, + filter_vector=options.remove_vectors, + ) + + +def preprocess_remove_background(input_file, output_file, log, context): options = context.get_options() if not options.remove_background: re_symlink(input_file, output_file, log) @@ -543,16 +546,13 @@ def preprocess_remove_background( if any(image.bpc > 1 for image in pageinfo.images): leptonica.remove_background(input_file, output_file) else: - log.info("{0:4d}: background removal skipped on mono page".format( - pageinfo.pageno)) + log.info( + "{0:4d}: background removal skipped on mono page".format(pageinfo.pageno) + ) re_symlink(input_file, output_file, log) -def preprocess_deskew( - input_file, - output_file, - log, - context): +def preprocess_deskew(input_file, output_file, log, context): options = context.get_options() if not options.deskew: re_symlink(input_file, output_file, log) @@ -564,28 +564,21 @@ def preprocess_deskew( leptonica.deskew(input_file, output_file, dpi) -def preprocess_clean( - input_file, - output_file, - log, - context): +def preprocess_clean(input_file, output_file, log, context): options = context.get_options() if not options.clean: re_symlink(input_file, output_file, log) return from .exec import unpaper + pageinfo = get_pageinfo(input_file, context) dpi = get_page_square_dpi(pageinfo, options) unpaper.clean(input_file, output_file, dpi, log) -def select_ocr_image( - infiles, - output_file, - log, - context): +def select_ocr_image(infiles, output_file, log, context): """Select the image we send for OCR. May not be the same as the display image depending on preprocessing. This image will never be shown to the user.""" @@ -598,8 +591,9 @@ def select_ocr_image( from PIL import ImageColor from PIL import ImageDraw from decimal import Decimal + white = ImageColor.getcolor('#ffffff', im.mode) - #pink = ImageColor.getcolor('#ff0080', im.mode) + # pink = ImageColor.getcolor('#ff0080', im.mode) draw = ImageDraw.ImageDraw(im) xres, yres = im.info['dpi'] @@ -618,14 +612,16 @@ def select_ocr_image( # be None) bbox = [float(v) for v in textarea] xscale, yscale = float(xres) / 72.0, float(yres) / 72.0 - pixcoords = [bbox[0] * xscale, - im.height - bbox[3] * yscale, - bbox[2] * xscale, - im.height - bbox[1] * yscale] + pixcoords = [ + bbox[0] * xscale, + im.height - bbox[3] * yscale, + bbox[2] * xscale, + im.height - bbox[1] * yscale, + ] pixcoords = [int(round(c)) for c in pixcoords] log.debug('blanking %r', pixcoords) draw.rectangle(pixcoords, fill=white) - #draw.rectangle(pixcoords, outline=pink) + # draw.rectangle(pixcoords, outline=pink) if options.mask_barcodes or options.threshold: pix = leptonica.Pix.frompil(im) @@ -645,11 +641,7 @@ def select_ocr_image( im.save(output_file, dpi=dpi) -def ocr_tesseract_hocr( - input_file, - output_files, - log, - context): +def ocr_tesseract_hocr(input_file, output_files, log, context): options = context.get_options() tesseract.generate_hocr( input_file=input_file, @@ -661,15 +653,11 @@ def ocr_tesseract_hocr( pagesegmode=options.tesseract_pagesegmode, user_words=options.user_words, user_patterns=options.user_patterns, - log=log - ) + log=log, + ) -def select_visible_page_image( - infiles, - output_file, - log, - context): +def select_visible_page_image(infiles, output_file, log, context): "Selects a whole page image that we can show the user (if necessary)" options = context.get_options() @@ -684,10 +672,8 @@ def select_visible_page_image( image = next(ii for ii in infiles if ii.endswith(image_suffix)) pageinfo = get_pageinfo(image, context) - if pageinfo.images and \ - all(im.enc == 'jpeg' for im in pageinfo.images): - log.debug('{:4d}: JPEG input -> JPEG output'.format( - page_number(image))) + if pageinfo.images and all(im.enc == 'jpeg' for im in pageinfo.images): + log.debug('{:4d}: JPEG input -> JPEG output'.format(page_number(image))) # If all images were JPEGs originally, produce a JPEG as output with Image.open(image) as im: # At this point the image should be a .png, but deskew, unpaper @@ -705,11 +691,7 @@ def select_visible_page_image( re_symlink(image, output_file, log) -def select_image_layer( - infiles, - output_file, - log, - context): +def select_image_layer(infiles, output_file, log, context): """Selects the image layer for the output page. If possible this is the orientation-corrected input page, or an image of the whole page converted to PDF.""" @@ -719,8 +701,11 @@ def select_image_layer( image = next(ii for ii in infiles if ii.endswith('.image')) if options.lossless_reconstruction: - log.debug("{:4d}: page eligible for lossless reconstruction".format( - page_number(page_pdf))) + log.debug( + "{:4d}: page eligible for lossless reconstruction".format( + page_number(page_pdf) + ) + ) re_symlink(page_pdf, output_file, log) # Still points to multipage return @@ -739,32 +724,28 @@ def select_image_layer( with open(image, 'rb') as imfile, open(output_file, 'wb') as pdf: log.debug('{:4d}: convert'.format(page_number(page_pdf))) img2pdf.convert( - imfile, with_pdfrw=False, - layout_fun=layout_fun, outputstream=pdf) + imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf + ) log.debug('{:4d}: convert done'.format(page_number(page_pdf))) -def render_hocr_page( - infiles, - output_file, - log, - context): +def render_hocr_page(infiles, output_file, log, context): options = context.get_options() hocr = next(ii for ii in infiles if ii.endswith('.hocr')) pageinfo = get_pageinfo(hocr, context) dpi = get_page_square_dpi(pageinfo, options) hocrtransform = HocrTransform(hocr, dpi) - hocrtransform.to_pdf(output_file, imageFileName=None, - showBoundingboxes=False, invisibleText=True, - interwordSpaces=True) + hocrtransform.to_pdf( + output_file, + imageFileName=None, + showBoundingboxes=False, + invisibleText=True, + interwordSpaces=True, + ) -def ocr_tesseract_textonly_pdf( - infiles, - outfiles, - log, - context): +def ocr_tesseract_textonly_pdf(infiles, outfiles, log, context): options = context.get_options() input_image = next((ii for ii in infiles if ii.endswith('.ocr.png')), '') if not input_image: @@ -786,7 +767,8 @@ def ocr_tesseract_textonly_pdf( pagesegmode=options.tesseract_pagesegmode, user_words=options.user_words, user_patterns=options.user_patterns, - log=log) + log=log, + ) def get_docinfo(base_pdf, options): @@ -797,8 +779,10 @@ def get_docinfo(base_pdf, options): except (KeyError, TypeError): return '' - pdfmark = {k: from_document_info(k) for k in - ('/Title', '/Author', '/Keywords', '/Subject', '/CreationDate')} + pdfmark = { + k: from_document_info(k) + for k in ('/Title', '/Author', '/Keywords', '/Subject', '/CreationDate') + } if options.title: pdfmark['/Title'] = options.title if options.author: @@ -814,9 +798,8 @@ def get_docinfo(base_pdf, options): renderer_tag = 'OCR' pdfmark['/Creator'] = '{0} {1} / Tesseract {2} {3}'.format( - PROGRAM_NAME, VERSION, - renderer_tag, - tesseract.version()) + PROGRAM_NAME, VERSION, renderer_tag, tesseract.version() + ) pdfmark['/Producer'] = 'pikepdf ' + pikepdf.__version__ if 'OCRMYPDF_CREATOR' in os.environ: pdfmark['/Creator'] = os.environ['OCRMYPDF_CREATOR'] @@ -827,22 +810,13 @@ def get_docinfo(base_pdf, options): return pdfmark -def generate_postscript_stub( - input_file, - output_file, - log, - context): +def generate_postscript_stub(input_file, output_file, log, context): options = context.get_options() pdf = pikepdf.open(input_file) generate_pdfa_ps(output_file) -def convert_to_pdfa( - input_files_groups, - output_file, - log, - context -): +def convert_to_pdfa(input_files_groups, output_file, log, context): options = context.get_options() input_pdfinfo = context.get_pdfinfo() @@ -850,9 +824,7 @@ def convert_to_pdfa( layers_file = next( (ii for ii in input_files if ii.endswith('layers.rendered.pdf')), None ) - ps = next( - (ii for ii in input_files if ii.endswith('.ps')), None - ) + ps = next((ii for ii in input_files if ii.endswith('.ps')), None) ghostscript.generate_pdfa( pdf_version=input_pdfinfo.min_version, pdf_pages=[layers_file, ps], @@ -860,15 +832,11 @@ def convert_to_pdfa( compression=options.pdfa_image_compression, log=log, threads=options.jobs or 1, - pdfa_part=options.output_type[-1] # is pdfa-1, pdfa-2, or pdfa-3 + pdfa_part=options.output_type[-1], # is pdfa-1, pdfa-2, or pdfa-3 ) -def metadata_fixup( - input_files_groups, - output_file, - log, - context): +def metadata_fixup(input_files_groups, output_file, log, context): options = context.get_options() input_files = list(f for f in flatten_groups(input_files_groups)) @@ -878,9 +846,7 @@ def metadata_fixup( layers_file = next( (ii for ii in input_files if ii.endswith('layers.rendered.pdf')), None ) - pdfa_file = next( - (ii for ii in input_files if ii.endswith('pdfa.pdf')), None - ) + pdfa_file = next((ii for ii in input_files if ii.endswith('pdfa.pdf')), None) original = pikepdf.open(original_file) docinfo = get_docinfo(original, options) @@ -903,27 +869,21 @@ def metadata_fixup( "PDF's XMP metadata." ) log.debug( - "The following metadata fields were not copied: %r", - not_copied + "The following metadata fields were not copied: %r", not_copied ) - pdf.save(output_file, compress_streams=True, - object_stream_mode=pikepdf.ObjectStreamMode.generate) - - -def optimize_pdf( - input_file, + pdf.save( output_file, - log, - context): + compress_streams=True, + object_stream_mode=pikepdf.ObjectStreamMode.generate, + ) + + +def optimize_pdf(input_file, output_file, log, context): optimize(input_file, output_file, log, context) -def merge_sidecars( - input_files_groups, - output_file, - log, - context): +def merge_sidecars(input_files_groups, output_file, log, context): pdfinfo = context.get_pdfinfo() txt_files = [None] * len(pdfinfo) @@ -950,8 +910,7 @@ def merge_sidecars( else: stream.write(txt) else: - stream.write('[OCR skipped on page {}]'.format( - page_num + 1)) + stream.write('[OCR skipped on page {}]'.format(page_num + 1)) if output_file == '-': write_pages(sys.stdout) @@ -961,11 +920,7 @@ def merge_sidecars( write_pages(out) -def copy_final( - input_files, - output_file, - log, - context): +def copy_final(input_files, output_file, log, context): input_file = next((ii for ii in input_files if ii.endswith('.pdf'))) log.debug('%s -> %s', input_file, output_file) with open(input_file, 'rb') as input_stream: @@ -989,7 +944,8 @@ def build_pipeline(options, work_folder, log, context): input=os.path.join(work_folder, 'origin'), filter=formatter('(?i)'), output=os.path.join(work_folder, 'origin.pdf'), - extras=[log, context]) + extras=[log, context], + ) task_repair_and_parse_pdf = main_pipeline.transform( task_func=repair_and_parse_pdf, @@ -997,21 +953,26 @@ def build_pipeline(options, work_folder, log, context): filter=suffix('.pdf'), output='.repaired.pdf', output_dir=work_folder, - extras=[log, context]) + extras=[log, context], + ) # Split (kwargs for split seems to be broken, so pass plain args) task_marker_pages = main_pipeline.split( marker_pages, task_repair_and_parse_pdf, os.path.join(work_folder, '*.marker.pdf'), - extras=[log, context]) + extras=[log, context], + ) task_ocr_or_skip = main_pipeline.split( ocr_or_skip, task_marker_pages, - [os.path.join(work_folder, '*.ocr.page.pdf'), - os.path.join(work_folder, '*.skip.page.pdf')], - extras=[log, context]) + [ + os.path.join(work_folder, '*.ocr.page.pdf'), + os.path.join(work_folder, '*.skip.page.pdf'), + ], + extras=[log, context], + ) # Rasterize preview task_rasterize_preview = main_pipeline.transform( @@ -1020,7 +981,8 @@ def build_pipeline(options, work_folder, log, context): filter=suffix('.page.pdf'), output='.preview.jpg', output_dir=work_folder, - extras=[log, context]) + extras=[log, context], + ) task_rasterize_preview.active_if(options.rotate_pages) # Orient @@ -1029,7 +991,8 @@ def build_pipeline(options, work_folder, log, context): input=[task_ocr_or_skip, task_rasterize_preview], filter=regex(r".*/(\d{6})(\.ocr|\.skip)(?:\.page\.pdf|\.preview\.jpg)"), output=os.path.join(work_folder, r'\1\2.oriented.pdf'), - extras=[log, context]) + extras=[log, context], + ) # Rasterize actual task_rasterize_with_ghostscript = main_pipeline.transform( @@ -1038,7 +1001,8 @@ def build_pipeline(options, work_folder, log, context): filter=suffix('.ocr.oriented.pdf'), output='.page.png', output_dir=work_folder, - extras=[log, context]) + extras=[log, context], + ) # Preprocessing subpipeline task_preprocess_remove_background = main_pipeline.transform( @@ -1046,28 +1010,32 @@ def build_pipeline(options, work_folder, log, context): input=task_rasterize_with_ghostscript, filter=suffix(".page.png"), output=".pp-background.png", - extras=[log, context]) + extras=[log, context], + ) task_preprocess_deskew = main_pipeline.transform( task_func=preprocess_deskew, input=task_preprocess_remove_background, filter=suffix(".pp-background.png"), output=".pp-deskew.png", - extras=[log, context]) + extras=[log, context], + ) task_preprocess_clean = main_pipeline.transform( task_func=preprocess_clean, input=task_preprocess_deskew, filter=suffix(".pp-deskew.png"), output=".pp-clean.png", - extras=[log, context]) + extras=[log, context], + ) task_select_ocr_image = main_pipeline.collate( task_func=select_ocr_image, input=[task_preprocess_clean], filter=regex(r".*/(\d{6})(?:\.page|\.pp-.*)\.png"), output=os.path.join(work_folder, r"\1.ocr.png"), - extras=[log, context]) + extras=[log, context], + ) # HOCR OCR task_ocr_tesseract_hocr = main_pipeline.transform( @@ -1075,19 +1043,23 @@ def build_pipeline(options, work_folder, log, context): input=task_select_ocr_image, filter=suffix(".ocr.png"), output=[".hocr", ".txt"], - extras=[log, context]) + extras=[log, context], + ) task_ocr_tesseract_hocr.graphviz(fillcolor='"#00cc66"') task_ocr_tesseract_hocr.active_if(options.pdf_renderer == 'hocr') task_select_visible_page_image = main_pipeline.collate( task_func=select_visible_page_image, - input=[task_rasterize_with_ghostscript, - task_preprocess_remove_background, - task_preprocess_deskew, - task_preprocess_clean], + input=[ + task_rasterize_with_ghostscript, + task_preprocess_remove_background, + task_preprocess_deskew, + task_preprocess_clean, + ], filter=regex(r".*/(\d{6})(?:\.page|\.pp-.*)\.png"), output=os.path.join(work_folder, r'\1.image'), - extras=[log, context]) + extras=[log, context], + ) task_select_visible_page_image.graphviz(shape='diamond') task_select_image_layer = main_pipeline.collate( @@ -1095,16 +1067,17 @@ def build_pipeline(options, work_folder, log, context): input=[task_select_visible_page_image, task_orient_page], filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.oriented\.pdf)"), output=os.path.join(work_folder, r'\1.image-layer.pdf'), - extras=[log, context]) - task_select_image_layer.graphviz( - fillcolor='"#00cc66"', shape='diamond') + extras=[log, context], + ) + task_select_image_layer.graphviz(fillcolor='"#00cc66"', shape='diamond') task_render_hocr_page = main_pipeline.transform( task_func=render_hocr_page, input=task_ocr_tesseract_hocr, filter=regex(r".*/(\d{6})(?:\.hocr)"), output=os.path.join(work_folder, r'\1.text.pdf'), - extras=[log, context]) + extras=[log, context], + ) task_render_hocr_page.graphviz(fillcolor='"#00cc66"') task_render_hocr_page.active_if(options.pdf_renderer == 'hocr') @@ -1113,22 +1086,29 @@ def build_pipeline(options, work_folder, log, context): task_func=ocr_tesseract_textonly_pdf, input=[task_select_ocr_image], filter=regex(r".*/(\d{6})(?:\.ocr.png)"), - output=[os.path.join(work_folder, r'\1.text.pdf'), - os.path.join(work_folder, r'\1.text.txt')], - extras=[log, context]) + output=[ + os.path.join(work_folder, r'\1.text.pdf'), + os.path.join(work_folder, r'\1.text.txt'), + ], + extras=[log, context], + ) task_ocr_tesseract_textonly_pdf.graphviz(fillcolor='"#ff69b4"') task_ocr_tesseract_textonly_pdf.active_if(options.pdf_renderer == 'sandwich') task_weave_layers = main_pipeline.collate( task_func=weave_layers, - input=[task_repair_and_parse_pdf, - task_render_hocr_page, - task_ocr_tesseract_textonly_pdf, - task_select_image_layer], + input=[ + task_repair_and_parse_pdf, + task_render_hocr_page, + task_ocr_tesseract_textonly_pdf, + task_select_image_layer, + ], filter=regex( - r".*/((?:\d{6}(?:\.text\.pdf|\.image-layer\.pdf))|(?:origin\.repaired\.pdf))"), + r".*/((?:\d{6}(?:\.text\.pdf|\.image-layer\.pdf))|(?:origin\.repaired\.pdf))" + ), output=os.path.join(work_folder, r'layers.rendered.pdf'), - extras=[log, context]) + extras=[log, context], + ) task_weave_layers.graphviz(fillcolor='"#00cc66"') # PDF/A pdfmark @@ -1137,34 +1117,32 @@ def build_pipeline(options, work_folder, log, context): input=task_repair_and_parse_pdf, filter=formatter(r'\.repaired\.pdf'), output=os.path.join(work_folder, 'pdfa.ps'), - extras=[log, context]) + extras=[log, context], + ) task_generate_postscript_stub.active_if(options.output_type.startswith('pdfa')) # PDF/A conversion task_convert_to_pdfa = main_pipeline.merge( task_func=convert_to_pdfa, - input=[task_generate_postscript_stub, - task_weave_layers], + input=[task_generate_postscript_stub, task_weave_layers], output=os.path.join(work_folder, 'pdfa.pdf'), - extras=[log, context] + extras=[log, context], ) task_convert_to_pdfa.active_if(options.output_type.startswith('pdfa')) task_metadata_fixup = main_pipeline.merge( task_func=metadata_fixup, - input=[task_repair_and_parse_pdf, - task_weave_layers, - task_convert_to_pdfa], + input=[task_repair_and_parse_pdf, task_weave_layers, task_convert_to_pdfa], output=os.path.join(work_folder, 'metafix.pdf'), - extras=[log, context] + extras=[log, context], ) task_merge_sidecars = main_pipeline.merge( task_func=merge_sidecars, - input=[task_ocr_tesseract_hocr, - task_ocr_tesseract_textonly_pdf], + input=[task_ocr_tesseract_hocr, task_ocr_tesseract_textonly_pdf], output=options.sidecar, - extras=[log, context]) + extras=[log, context], + ) task_merge_sidecars.active_if(options.sidecar) # Optimize @@ -1174,11 +1152,13 @@ def build_pipeline(options, work_folder, log, context): filter=suffix('.pdf'), output='.optimized.pdf', output_dir=work_folder, - extras=[log, context]) + extras=[log, context], + ) # Finalize main_pipeline.merge( task_func=copy_final, input=[task_optimize_pdf], output=options.output_file, - extras=[log, context]) + extras=[log, context], + ) diff --git a/src/ocrmypdf/_unicodefun.py b/src/ocrmypdf/_unicodefun.py index 6834884d..c6b0ad16 100644 --- a/src/ocrmypdf/_unicodefun.py +++ b/src/ocrmypdf/_unicodefun.py @@ -47,6 +47,7 @@ def verify_python3_env(): # pragma: no cover try: import locale + fs_enc = codecs.lookup(locale.getpreferredencoding()).name except Exception: fs_enc = 'ascii' @@ -56,8 +57,10 @@ def verify_python3_env(): # pragma: no cover extra = '' if os.name == 'posix': import subprocess - rv = subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate()[0] + + rv = subprocess.Popen( + ['locale', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ).communicate()[0] good_locales = set() has_c_utf8 = False @@ -108,6 +111,8 @@ def verify_python3_env(): # pragma: no cover 'is not supported' ) % bad_locale - raise RuntimeError('ocrmypdf will abort further execution because Python 3 ' - 'was configured to use ASCII as encoding for the ' - 'environment.' + extra) + raise RuntimeError( + 'ocrmypdf will abort further execution because Python 3 ' + 'was configured to use ASCII as encoding for the ' + 'environment.' + extra + ) diff --git a/src/ocrmypdf/_weave.py b/src/ocrmypdf/_weave.py index 995878b6..e4aefca7 100644 --- a/src/ocrmypdf/_weave.py +++ b/src/ocrmypdf/_weave.py @@ -89,8 +89,8 @@ def strip_invisible_text(pdf, page, log): def _weave_layers_graft( - *, pdf_base, page_num, text, font, font_key, procset, rotation, - strip_old_text, log): + *, pdf_base, page_num, text, font, font_key, procset, rotation, strip_old_text, log +): """Insert the text layer from text page 0 on to pdf_base at page_num""" log.debug("Grafting") @@ -108,7 +108,7 @@ def _weave_layers_graft( stream = bytearray(pdf_text_contents) pattern = b'/Im1 Do' idx = stream.find(pattern) - stream[idx:(idx + len(pattern))] = b' ' * len(pattern) + stream[idx : (idx + len(pattern))] = b' ' * len(pattern) pdf_text_contents = bytes(stream) base_page = pdf_base.pages.p(page_num) @@ -117,12 +117,10 @@ def _weave_layers_graft( # content may have a rotation applied. Wrap the text stream with a rotation # so it will be oriented the same way as the rest of the page content. # (Previous versions OCRmyPDF rotated the content layer to match the text.) - mediabox = [float(pdf_text.pages[0].MediaBox[v]) - for v in range(4)] + mediabox = [float(pdf_text.pages[0].MediaBox[v]) for v in range(4)] wt, ht = mediabox[2] - mediabox[0], mediabox[3] - mediabox[1] - mediabox = [float(base_page.MediaBox[v]) - for v in range(4)] + mediabox = [float(base_page.MediaBox[v]) for v in range(4)] wp, hp = mediabox[2] - mediabox[0], mediabox[3] - mediabox[1] translate = pikepdf.PdfMatrix().translated(-wt / 2, -ht / 2) @@ -147,11 +145,7 @@ def _weave_layers_graft( # for a size different between initial and text PDF, then untranslate ctm = translate @ rotate @ scale @ untranslate - pdf_text_contents = ( - b'q %s cm\n' % ctm.encode() + - pdf_text_contents + - b'\nQ\n' - ) + pdf_text_contents = b'q %s cm\n' % ctm.encode() + pdf_text_contents + b'\nQ\n' new_text_layer = pikepdf.Stream(pdf_base, pdf_text_contents) @@ -254,8 +248,7 @@ def _fix_toc(pdf_base, pageref_remap, log): if not isinstance(dest_node, pikepdf.Array): return pageref = dest_node[0] - if pageref['/Type'] == '/Page' and \ - pageref.objgen in pageref_remap: + if pageref['/Type'] == '/Page' and pageref.objgen in pageref_remap: new_objgen = pageref_remap[pageref.objgen] dest_node[0] = pdf_base.get_object(new_objgen) @@ -278,11 +271,7 @@ def _fix_toc(pdf_base, pageref_remap, log): _traverse_toc(pdf_base, visit_remap_dest, log) -def weave_layers( - infiles, - output_file, - log, - context): +def weave_layers(infiles, output_file, log, context): """Apply text layer and/or image layer changes to baseline file This is where the magic happens. infiles will be the main PDF to modify, @@ -313,6 +302,7 @@ def weave_layers( return page_number(key) except ValueError: return -1 + flat_inputs = sorted(flatten_groups(infiles), key=input_sorter) groups = groupby(flat_inputs, key=input_sorter) @@ -333,7 +323,8 @@ def weave_layers( _traverse_toc(pdf_base, None, log) procset = pdf_base.make_indirect( - pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]')) + pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]') + ) # Iterate rest for page_num, layers in groups: @@ -341,12 +332,8 @@ def weave_layers( log.debug(page_num) log.debug(layers) - text = next( - (ii for ii in layers if ii.endswith('.text.pdf')), None - ) - image = next( - (ii for ii in layers if ii.endswith('.image-layer.pdf')), None - ) + text = next((ii for ii in layers if ii.endswith('.text.pdf')), None) + image = next((ii for ii in layers if ii.endswith('.image-layer.pdf')), None) if text and not font: font, font_key = _find_font(text, pdf_base) @@ -378,23 +365,30 @@ def weave_layers( content_rotation = autorotate_correction text_rotation = autorotate_correction text_misaligned = (text_rotation - content_rotation) % 360 - log.debug('%r', [ - text_rotation, autorotate_correction, text_misaligned, - content_rotation] + log.debug( + '%r', + [text_rotation, autorotate_correction, text_misaligned, content_rotation], ) if text and font: # Graft the text layer onto this page, whether new or old strip_old = context.get_options().redo_ocr _weave_layers_graft( - pdf_base=pdf_base, page_num=page_num, text=text, font=font, - font_key=font_key, rotation=text_misaligned, procset=procset, - strip_old_text=strip_old, log=log + pdf_base=pdf_base, + page_num=page_num, + text=text, + font=font, + font_key=font_key, + rotation=text_misaligned, + procset=procset, + strip_old_text=strip_old, + log=log, ) # Correct the rotation if applicable - pdf_base.pages[page_num - 1].Rotate = \ - (content_rotation - autorotate_correction) % 360 + pdf_base.pages[page_num - 1].Rotate = ( + content_rotation - autorotate_correction + ) % 360 if len(keep_open) > 100: # qpdf limitations require us to keep files open when we intend @@ -404,7 +398,8 @@ def weave_layers( # even if page 1 doesn't use it, so we have a way to get it back. page0 = pdf_base.pages[0] _update_page_resources( - page=page0, font=font, font_key=font_key, procset=procset) + page=page0, font=font, font_key=font_key, procset=procset + ) interim = output_file + '_working{}.pdf'.format(page_num) pdf_base.save(interim) del pdf_base diff --git a/src/ocrmypdf/exceptions.py b/src/ocrmypdf/exceptions.py index 199c21ea..a2df1963 100644 --- a/src/ocrmypdf/exceptions.py +++ b/src/ocrmypdf/exceptions.py @@ -19,6 +19,7 @@ from enum import IntEnum from textwrap import dedent + class ExitCode(IntEnum): ok = 0 bad_args = 1 @@ -52,7 +53,8 @@ class BadArgsError(ExitCodeException): class PdfMergeFailedError(ExitCodeException): exit_code = ExitCode.input_file - message = dedent('''\ + message = dedent( + '''\ Failed to merge PDF image layer with OCR layer Usually this happens because the input PDF file is malformed and @@ -60,7 +62,9 @@ class PdfMergeFailedError(ExitCodeException): Try using ocrmypdf --pdf-renderer sandwich [..other args..] - ''') + ''' + ) + class MissingDependencyError(ExitCodeException): exit_code = ExitCode.missing_dependency @@ -92,7 +96,8 @@ class SubprocessOutputError(ExitCodeException): class EncryptedPdfError(ExitCodeException): exit_code = ExitCode.encrypted_pdf - message = dedent('''\ + message = dedent( + '''\ Input PDF is encrypted. The encryption must be removed to perform OCR. @@ -101,7 +106,8 @@ class EncryptedPdfError(ExitCodeException): You can remove the encryption using qpdf --decrypt [--password=[password]] infilename - ''') + ''' + ) class TesseractConfigError(ExitCodeException): diff --git a/src/ocrmypdf/exec/__init__.py b/src/ocrmypdf/exec/__init__.py index 5fd2195c..4c6fd1eb 100644 --- a/src/ocrmypdf/exec/__init__.py +++ b/src/ocrmypdf/exec/__init__.py @@ -24,36 +24,40 @@ from subprocess import run, STDOUT, PIPE, CalledProcessError from ..exceptions import MissingDependencyError -def get_version(program, *, - version_arg='--version', regex=r'(\d+(\.\d+)*)'): +def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'): "Get the version of the specified program" - args_prog = [ - program, - version_arg - ] + args_prog = [program, version_arg] try: proc = run( - args_prog, close_fds=True, universal_newlines=True, - stdout=PIPE, stderr=STDOUT, check=True) + args_prog, + close_fds=True, + universal_newlines=True, + stdout=PIPE, + stderr=STDOUT, + check=True, + ) output = proc.stdout except FileNotFoundError as e: raise MissingDependencyError( - "Could not find program '{}' on the PATH".format( - program)) from e + "Could not find program '{}' on the PATH".format(program) + ) from e except CalledProcessError as e: if e.returncode < 0: raise MissingDependencyError( "Ran program '{}' but it exited with an error:\n{}".format( - program, e.output)) from e + program, e.output + ) + ) from e raise MissingDependencyError( - "Could not find program '{}' on the PATH".format( - program)) from e + "Could not find program '{}' on the PATH".format(program) + ) from e try: version = re.match(regex, output.strip()).group(1) except AttributeError as e: raise MissingDependencyError( - ("The program '{}' did not report its version. " - "Message was:\n{}").format(program, output) + ("The program '{}' did not report its version. " "Message was:\n{}").format( + program, output + ) ) return version diff --git a/src/ocrmypdf/exec/ghostscript.py b/src/ocrmypdf/exec/ghostscript.py index 5263bbe4..a4c67fb3 100644 --- a/src/ocrmypdf/exec/ghostscript.py +++ b/src/ocrmypdf/exec/ghostscript.py @@ -69,39 +69,46 @@ def extract_text(input_file, pageno=1): """ if pageno is not None: - pages = [ - '-dFirstPage=%i' % pageno, - '-dLastPage=%i' % pageno - ] + pages = ['-dFirstPage=%i' % pageno, '-dLastPage=%i' % pageno] else: pages = [] - args_gs = [ - 'gs', - '-dQUIET', - '-dSAFER', - '-dBATCH', - '-dNOPAUSE', - '-sDEVICE=txtwrite', - '-dTextFormat=0', - ] + pages + [ - '-o', '-', - fspath(input_file) - ] + args_gs = ( + [ + 'gs', + '-dQUIET', + '-dSAFER', + '-dBATCH', + '-dNOPAUSE', + '-sDEVICE=txtwrite', + '-dTextFormat=0', + ] + + pages + + ['-o', '-', fspath(input_file)] + ) p = run(args_gs, stdout=PIPE, stderr=PIPE) if p.returncode != 0: raise SubprocessOutputError( - 'Ghostscript text extraction failed\n%s\n%s\n%s' % ( - input_file, p.stdout.decode(), p.stderr.decode() - ) + 'Ghostscript text extraction failed\n%s\n%s\n%s' + % (input_file, p.stdout.decode(), p.stderr.decode()) ) return p.stdout -def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log, - pageno=1, page_dpi=None, rotation=None, filter_vector=False): +def rasterize_pdf( + input_file, + output_file, + xres, + yres, + raster_device, + log, + pageno=1, + page_dpi=None, + rotation=None, + filter_vector=False, +): """Rasterize one page of a PDF at resolution (xres, yres) in canvas units. The image is sized to match the integer pixels dimensions implied by @@ -126,26 +133,30 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log, page_dpi = res with NamedTemporaryFile(delete=True) as tmp: - args_gs = [ - 'gs', - '-dQUIET', - '-dSAFER', - '-dBATCH', - '-dNOPAUSE', - '-sDEVICE=%s' % raster_device, - '-dFirstPage=%i' % pageno, - '-dLastPage=%i' % pageno, - '-r{0}x{1}'.format(str(int_res[0]), str(int_res[1])), - ] + (['-dFILTERVECTOR'] if filter_vector else []) + [ - '-o', tmp.name, - '-dAutoRotatePages=/None', # Probably has no effect on raster - '-f', - fspath(input_file) - ] + args_gs = ( + [ + 'gs', + '-dQUIET', + '-dSAFER', + '-dBATCH', + '-dNOPAUSE', + '-sDEVICE=%s' % raster_device, + '-dFirstPage=%i' % pageno, + '-dLastPage=%i' % pageno, + '-r{0}x{1}'.format(str(int_res[0]), str(int_res[1])), + ] + + (['-dFILTERVECTOR'] if filter_vector else []) + + [ + '-o', + tmp.name, + '-dAutoRotatePages=/None', # Probably has no effect on raster + '-f', + fspath(input_file), + ] + ) log.debug(args_gs) - p = run(args_gs, stdout=PIPE, stderr=STDOUT, - universal_newlines=True) + p = run(args_gs, stdout=PIPE, stderr=STDOUT, universal_newlines=True) if _gs_error_reported(p.stdout): log.error(p.stdout) else: @@ -162,12 +173,16 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log, tmp.seek(0) with Image.open(tmp) as im: - expected_size = round(im.size[0] / int_res[0] * res[0]), \ - round(im.size[1] / int_res[1] * res[1]) + expected_size = ( + round(im.size[0] / int_res[0] * res[0]), + round(im.size[1] / int_res[1] * res[1]), + ) if expected_size != im.size or page_dpi != (xres, yres): log.debug( "Ghostscript: resize output image {} -> {}".format( - im.size, expected_size)) + im.size, expected_size + ) + ) im = im.resize(expected_size) if rotation is not None: @@ -185,8 +200,15 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log, im.save(fspath(output_file), dpi=page_dpi) -def generate_pdfa(pdf_pages, output_file, compression, log, - threads=1, pdf_version='1.5', pdfa_part='2'): +def generate_pdfa( + pdf_pages, + output_file, + compression, + log, + threads=1, + pdf_version='1.5', + pdfa_part='2', +): """Generate a PDF/A. The pdf_pages, a list files, will be merged into output_file. One or more @@ -240,26 +262,29 @@ def generate_pdfa(pdf_pages, output_file, compression, log, # nb no need to specify ProcessColorModel when ColorConversionStrategy # is set; see: # https://bugs.ghostscript.com/show_bug.cgi?id=699392 - args_gs = [ - "gs", - "-dQUIET", - "-dBATCH", - "-dNOPAUSE", - "-dCompatibilityLevel=" + str(pdf_version), - "-dNumRenderingThreads=" + str(threads), - "-sDEVICE=pdfwrite", - "-dAutoRotatePages=/None", - "-sColorConversionStrategy=" + strategy - ] + compression_args + [ - "-dJPEGQ=95", - "-dPDFA=" + pdfa_part, - "-dPDFACompatibilityPolicy=1", - "-sOutputFile=" + gs_pdf.name, - ] + args_gs = ( + [ + "gs", + "-dQUIET", + "-dBATCH", + "-dNOPAUSE", + "-dCompatibilityLevel=" + str(pdf_version), + "-dNumRenderingThreads=" + str(threads), + "-sDEVICE=pdfwrite", + "-dAutoRotatePages=/None", + "-sColorConversionStrategy=" + strategy, + ] + + compression_args + + [ + "-dJPEGQ=95", + "-dPDFA=" + pdfa_part, + "-dPDFACompatibilityPolicy=1", + "-sOutputFile=" + gs_pdf.name, + ] + ) args_gs.extend(fspath(s) for s in pdf_pages) # Stringify Path objs log.debug(args_gs) - p = run(args_gs, stdout=PIPE, stderr=STDOUT, - universal_newlines=True) + p = run(args_gs, stdout=PIPE, stderr=STDOUT, universal_newlines=True) if _gs_error_reported(p.stdout): log.error(p.stdout) @@ -270,7 +295,7 @@ def generate_pdfa(pdf_pages, output_file, compression, log, log.debug( "Ghostscript had to remove PDF 'overprinting' from the " "input file to complete PDF/A conversion. " - ) + ) else: log.debug(p.stdout) diff --git a/src/ocrmypdf/exec/jbig2enc.py b/src/ocrmypdf/exec/jbig2enc.py index 55d17ac9..f9e23645 100644 --- a/src/ocrmypdf/exec/jbig2enc.py +++ b/src/ocrmypdf/exec/jbig2enc.py @@ -42,7 +42,7 @@ def convert_group(*, cwd, infiles, out_prefix): out_prefix, '-s', # symbol mode (lossy) # '-r', # refinement mode (lossless symbol mode, currently disabled in - # jbig2) + # jbig2) '-p', ] args.extend(infiles) @@ -52,11 +52,7 @@ def convert_group(*, cwd, infiles, out_prefix): def convert_single(*, cwd, infile, outfile): - args = [ - 'jbig2', - '-p', - infile - ] + args = ['jbig2', '-p', infile] with open(outfile, 'wb') as fstdout: proc = run(args, cwd=cwd, stdout=fstdout, stderr=PIPE) proc.check_returncode() diff --git a/src/ocrmypdf/exec/pngquant.py b/src/ocrmypdf/exec/pngquant.py index 0440ce30..cd6a3844 100644 --- a/src/ocrmypdf/exec/pngquant.py +++ b/src/ocrmypdf/exec/pngquant.py @@ -40,10 +40,12 @@ def quantize(input_file, output_file, quality_min, quality_max): 'pngquant', '--force', '--skip-if-larger', - '--output', output_file, - '--quality', '{}-{}'.format(quality_min, quality_max), + '--output', + output_file, + '--quality', + '{}-{}'.format(quality_min, quality_max), '--', - input_file + input_file, ] proc = run(args) proc.check_returncode() diff --git a/src/ocrmypdf/exec/qpdf.py b/src/ocrmypdf/exec/qpdf.py index a703dc56..38d0202a 100644 --- a/src/ocrmypdf/exec/qpdf.py +++ b/src/ocrmypdf/exec/qpdf.py @@ -18,7 +18,7 @@ from subprocess import CalledProcessError, STDOUT, PIPE, run from functools import lru_cache -from . import get_version +from . import get_version from os import fspath @@ -28,22 +28,16 @@ def version(): def check(input_file, log=None): - args_qpdf = [ - 'qpdf', - '--check', - fspath(input_file) - ] + args_qpdf = ['qpdf', '--check', fspath(input_file)] if log is None: import logging as log try: - run(args_qpdf, stderr=STDOUT, stdout=PIPE, universal_newlines=True, - check=True) + run(args_qpdf, stderr=STDOUT, stdout=PIPE, universal_newlines=True, check=True) except CalledProcessError as e: if e.returncode == 2: - log.error("%s: not a valid PDF, and could not repair it.", - input_file) + log.error("%s: not a valid PDF, and could not repair it.", input_file) log.error("Details:") log.error(e.output) elif e.returncode == 3: diff --git a/src/ocrmypdf/exec/tesseract.py b/src/ocrmypdf/exec/tesseract.py index 091eb95c..8e184eb5 100644 --- a/src/ocrmypdf/exec/tesseract.py +++ b/src/ocrmypdf/exec/tesseract.py @@ -21,7 +21,14 @@ import shutil from functools import lru_cache from collections import namedtuple from textwrap import dedent -from subprocess import CalledProcessError, TimeoutExpired, check_output, STDOUT, run, PIPE +from subprocess import ( + CalledProcessError, + TimeoutExpired, + check_output, + STDOUT, + run, + PIPE, +) from contextlib import suppress from os import fspath @@ -29,9 +36,7 @@ from ..exceptions import MissingDependencyError, TesseractConfigError from ..helpers import page_number from . import get_version -OrientationConfidence = namedtuple( - 'OrientationConfidence', - ('angle', 'confidence')) +OrientationConfidence = namedtuple('OrientationConfidence', ('angle', 'confidence')) HOCR_TEMPLATE = """ " - return s.format(self._cdata.w, self._cdata.h, self._cdata.d, - int(ffi.cast('intptr_t', self._cdata)), - '(colormapped)' if self._cdata.colormap else '') + return s.format( + self._cdata.w, + self._cdata.h, + self._cdata.d, + int(ffi.cast('intptr_t', self._cdata)), + '(colormapped)' if self._cdata.colormap else '', + ) else: return "" @@ -289,8 +293,7 @@ class Pix(LeptonicaObject): with _LeptonicaErrorTrap(): return cls(lept.pixRead(os.fsencode(filename))) - def write_implied_format( - self, path, jpeg_quality=0, jpeg_progressive=0): + def write_implied_format(self, path, jpeg_quality=0, jpeg_progressive=0): """Write pix to the filename, with the extension indicating format. jpeg_quality -- quality (iff JPEG; 1 - 100, 0 for default) @@ -299,8 +302,8 @@ class Pix(LeptonicaObject): filename = fspath(path) with _LeptonicaErrorTrap(): lept.pixWriteImpliedFormat( - os.fsencode(filename), - self._cdata, jpeg_quality, jpeg_progressive) + os.fsencode(filename), self._cdata, jpeg_quality, jpeg_progressive + ) @classmethod def frompil(self, pillow_image): @@ -401,11 +404,13 @@ class Pix(LeptonicaObject): """ with _LeptonicaErrorTrap(): - return Pix(lept.pixRemoveColormapGeneral( - self._cdata, removal_type, lept.L_COPY)) + return Pix( + lept.pixRemoveColormapGeneral(self._cdata, removal_type, lept.L_COPY) + ) def otsu_adaptive_threshold( - self, tile_size=(300, 300), kernel_size=(4, 4), scorefract=0.1): + self, tile_size=(300, 300), kernel_size=(4, 4), scorefract=0.1 + ): with _LeptonicaErrorTrap(): sx, sy = tile_size smoothx, smoothy = kernel_size @@ -413,20 +418,23 @@ class Pix(LeptonicaObject): pix = Pix(lept.pixConvertTo8(self._cdata, 0)) result = lept.pixOtsuAdaptiveThreshold( - pix._cdata, - sx, sy, - smoothx, smoothy, - scorefract, - ffi.NULL, - p_pix) + pix._cdata, sx, sy, smoothx, smoothy, scorefract, ffi.NULL, p_pix + ) if result == 0: return Pix(p_pix[0]) else: return None def otsu_threshold_on_background_norm( - self, mask=None, tile_size=(10, 15), thresh=100, mincount=50, - bgval=255, kernel_size=(2, 2), scorefract=0.1): + self, + mask=None, + tile_size=(10, 15), + thresh=100, + mincount=50, + bgval=255, + kernel_size=(2, 2), + scorefract=0.1, + ): with _LeptonicaErrorTrap(): sx, sy = tile_size smoothx, smoothy = kernel_size @@ -438,17 +446,27 @@ class Pix(LeptonicaObject): thresh_pix = lept.pixOtsuThreshOnBackgroundNorm( pix._cdata, mask, - sx, sy, - thresh, mincount, bgval, - smoothx, smoothy, + sx, + sy, + thresh, + mincount, + bgval, + smoothx, + smoothy, scorefract, - ffi.NULL + ffi.NULL, ) return Pix(thresh_pix) def masked_threshold_on_background_norm( - self, mask=None, tile_size=(10, 15), thresh=100, mincount=50, - kernel_size=(2, 2), scorefract=0.1): + self, + mask=None, + tile_size=(10, 15), + thresh=100, + mincount=50, + kernel_size=(2, 2), + scorefract=0.1, + ): with _LeptonicaErrorTrap(): sx, sy = tile_size smoothx, smoothy = kernel_size @@ -460,74 +478,91 @@ class Pix(LeptonicaObject): thresh_pix = lept.pixMaskedThreshOnBackgroundNorm( pix._cdata, mask, - sx, sy, - thresh, mincount, - smoothx, smoothy, + sx, + sy, + thresh, + mincount, + smoothx, + smoothy, scorefract, - ffi.NULL + ffi.NULL, ) return Pix(thresh_pix) def crop_to_foreground( - self, threshold=128, mindist=70, erasedist=30, pagenum=0, - showmorph=0, display=0, pdfdir=ffi.NULL): + self, + threshold=128, + mindist=70, + erasedist=30, + pagenum=0, + showmorph=0, + display=0, + pdfdir=ffi.NULL, + ): with _LeptonicaErrorTrap(): - cropbox = Box(lept.pixFindPageForeground( - self._cdata, - threshold, - mindist, - erasedist, - pagenum, - showmorph, - display, - pdfdir)) + cropbox = Box( + lept.pixFindPageForeground( + self._cdata, + threshold, + mindist, + erasedist, + pagenum, + showmorph, + display, + pdfdir, + ) + ) - cropped_pix = lept.pixClipRectangle( - self._cdata, - cropbox._cdata, - ffi.NULL) + cropped_pix = lept.pixClipRectangle(self._cdata, cropbox._cdata, ffi.NULL) return Pix(cropped_pix) def clean_background_to_white( - self, mask=None, grayscale=None, gamma=1.0, black=0, white=255): + self, mask=None, grayscale=None, gamma=1.0, black=0, white=255 + ): with _LeptonicaErrorTrap(): - return Pix(lept.pixCleanBackgroundToWhite( - self._cdata, - mask or ffi.NULL, - grayscale or ffi.NULL, - gamma, - black, - white)) + return Pix( + lept.pixCleanBackgroundToWhite( + self._cdata, + mask or ffi.NULL, + grayscale or ffi.NULL, + gamma, + black, + white, + ) + ) def gamma_trc(self, gamma=1.0, minval=0, maxval=255): with _LeptonicaErrorTrap(): - return Pix(lept.pixGammaTRC( - ffi.NULL, - self._cdata, - gamma, - minval, - maxval - )) + return Pix(lept.pixGammaTRC(ffi.NULL, self._cdata, gamma, minval, maxval)) def background_norm( - self, mask=None, grayscale=None, tile_size=(10, 15), fg_threshold=60, - min_count=40, bg_val=200, smooth_kernel=(2, 1)): + self, + mask=None, + grayscale=None, + tile_size=(10, 15), + fg_threshold=60, + min_count=40, + bg_val=200, + smooth_kernel=(2, 1), + ): # Background norm doesn't work on color mapped Pix, so remove colormap target_pix = self.remove_colormap(lept.REMOVE_CMAP_BASED_ON_SRC) with _LeptonicaErrorTrap(): - return Pix(lept.pixBackgroundNorm( - target_pix._cdata, - mask or ffi.NULL, - grayscale or ffi.NULL, - tile_size[0], - tile_size[1], - fg_threshold, - min_count, - bg_val, - smooth_kernel[0], - smooth_kernel[1] - )) + return Pix( + lept.pixBackgroundNorm( + target_pix._cdata, + mask or ffi.NULL, + grayscale or ffi.NULL, + tile_size[0], + tile_size[1], + fg_threshold, + min_count, + bg_val, + smooth_kernel[0], + smooth_kernel[1], + ) + ) @staticmethod @lru_cache(maxsize=1) @@ -544,8 +579,7 @@ class Pix(LeptonicaObject): raise LeptonicaError("Leptonica version is too old") correlation = ffi.new('float *', 0.0) - result = lept.pixCorrelationBinary(pix1._cdata, pix2._cdata, - correlation) + result = lept.pixCorrelationBinary(pix1._cdata, pix2._cdata, correlation) if result != 0: raise LeptonicaError("Correlation failed") return correlation[0] @@ -553,8 +587,7 @@ class Pix(LeptonicaObject): def generate_pdf_ci_data(self, type_, quality): "Convert to PDF data, with transcoding" p_compdata = ffi.new('L_COMP_DATA **') - result = lept.pixGenerateCIData(self._cdata, type_, quality, 0, - p_compdata) + result = lept.pixGenerateCIData(self._cdata, type_, quality, 0, p_compdata) if result != 0: raise LeptonicaError("Generate PDF data failed") return CompressedData(p_compdata[0]) @@ -569,13 +602,15 @@ class Pix(LeptonicaObject): pixa_candidates = PixArray(lept.pixExtractBarcodes(pix._cdata, 0)) if not pixa_candidates: return - sarray = StringArray(lept.pixReadBarcodes( - pixa_candidates._cdata, - lept.L_BF_ANY, - lept.L_USE_WIDTHS, - ffi.NULL, - 0 - )) + sarray = StringArray( + lept.pixReadBarcodes( + pixa_candidates._cdata, + lept.L_BF_ANY, + lept.L_USE_WIDTHS, + ffi.NULL, + 0, + ) + ) except (LeptonicaError, ValueError, IndexError) as e: return finally: @@ -635,7 +670,8 @@ class CompressedData(LeptonicaObject): p_compdata = ffi.new('L_COMP_DATA **') result = lept.l_generateCIDataForPdf( - os.fsencode(filename), ffi.NULL, jpeg_quality, p_compdata) + os.fsencode(filename), ffi.NULL, jpeg_quality, p_compdata + ) if result != 0: raise LeptonicaError("CompressedData.open") return CompressedData(p_compdata[0]) @@ -689,7 +725,8 @@ class Box(LeptonicaObject): def __repr__(self): if self._cdata: return ''.format( - self.x, self.y, self.w, self.h) + self.x, self.y, self.w, self.h + ) return '' @property @@ -808,15 +845,22 @@ def deskew(infile, outfile, dpi): raise LeptonicaIOError("Failed to open destination file: %s" % outfile) -def remove_background(infile, outfile, tile_size=(40, 60), gamma=1.0, - black_threshold=70, white_threshold=190): +def remove_background( + infile, + outfile, + tile_size=(40, 60), + gamma=1.0, + black_threshold=70, + white_threshold=190, +): try: pix = Pix.open(infile) except LeptonicaIOError: raise LeptonicaIOError("Failed to open file: %s" % infile) pix = pix.background_norm(tile_size=tile_size).gamma_trc( - gamma, black_threshold, white_threshold) + gamma, black_threshold, white_threshold + ) try: pix.write_implied_format(outfile) @@ -825,15 +869,22 @@ def remove_background(infile, outfile, tile_size=(40, 60), gamma=1.0, if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="Python wrapper to access Leptonica") + parser = argparse.ArgumentParser(description="Python wrapper to access Leptonica") - subparsers = parser.add_subparsers(title='commands', - description='supported operations') + subparsers = parser.add_subparsers( + title='commands', description='supported operations' + ) parser_deskew = subparsers.add_parser('deskew') - parser_deskew.add_argument('-r', '--dpi', dest='dpi', action='store', - type=int, default=300, help='input resolution') + parser_deskew.add_argument( + '-r', + '--dpi', + dest='dpi', + action='store', + type=int, + default=300, + help='input resolution', + ) parser_deskew.add_argument('infile', help='image to deskew') parser_deskew.add_argument('outfile', help='deskewed output image') parser_deskew.set_defaults(func=deskew) diff --git a/src/ocrmypdf/lib/_leptonica.py b/src/ocrmypdf/lib/_leptonica.py index 17a2f757..d152a7d4 100644 --- a/src/ocrmypdf/lib/_leptonica.py +++ b/src/ocrmypdf/lib/_leptonica.py @@ -1,11 +1,327 @@ # auto-generated file import _cffi_backend -ffi = _cffi_backend.FFI('ocrmypdf.lib._leptonica', - _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x01\x33\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x34\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x37\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x3F\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x38\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x04\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x09\x03\x00\x00\x1A\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x01\x11\x00\x00\x01\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x01\x3C\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x05\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x60\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x13\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x01\x4E\x03\x00\x00\x1C\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x01\x50\x03\x00\x00\x1C\x01\x00\x00\x00\x0F\x00\x00\x13\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x3A\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x3A\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x3A\x0D\x00\x00\x13\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x9C\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x45\x0D\x00\x00\x10\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x00\x45\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x3E\x0D\x00\x00\x45\x11\x00\x00\x00\x0F\x00\x01\x3E\x0D\x00\x00\x00\x0F\x00\x00\x60\x0D\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x04\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x32\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x60\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xC3\x11\x00\x00\xC3\x11\x00\x00\xC3\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\xC3\x11\x00\x00\xC3\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x60\x11\x00\x00\x60\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x60\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x35\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xC3\x11\x00\x00\xC3\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x01\x4F\x03\x00\x00\x8E\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x10\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\xEC\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x10\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x4D\x03\x00\x01\x04\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x23\x11\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x04\x03\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\xEC\x11\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x1A\x11\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x13\x03\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x9C\x11\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x45\x03\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x01\x53\x03\x00\x00\x00\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x01\x36\x03\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x05\x09\x00\x00\x04\x09\x00\x01\x3B\x03\x00\x00\x06\x09\x00\x00\x07\x09\x00\x01\x3E\x03\x00\x01\x3F\x03\x00\x00\x02\x01\x00\x00\x0E\x01\x00\x00\x00\x0B\x00\x00\x01\x0B\x00\x00\x02\x0B\x00\x00\x03\x0B\x00\x00\x04\x0B\x00\x00\x05\x0B\x00\x00\x06\x0B\x00\x00\x60\x03\x00\x00\x0B\x01\x00\x00\x05\x01\x00\x00\x03\x01\x00\x01\x39\x03\x00\x01\x4E\x03\x00\x00\x04\x01\x00\x01\x50\x03\x00\x00\x08\x01\x00\x00\x0C\x01\x00\x00\x06\x01\x00\x00\x00\x01', - _globals = (b'\xFF\xFF\xFF\x0BL_BF_ANY',1,b'\xFF\xFF\xFF\x0BL_BF_CODABAR',9,b'\xFF\xFF\xFF\x0BL_BF_CODE128',2,b'\xFF\xFF\xFF\x0BL_BF_CODE2OF5',5,b'\xFF\xFF\xFF\x0BL_BF_CODE39',7,b'\xFF\xFF\xFF\x0BL_BF_CODE93',8,b'\xFF\xFF\xFF\x0BL_BF_CODEI2OF5',6,b'\xFF\xFF\xFF\x0BL_BF_EAN13',4,b'\xFF\xFF\xFF\x0BL_BF_EAN8',3,b'\xFF\xFF\xFF\x0BL_BF_UNKNOWN',0,b'\xFF\xFF\xFF\x0BL_BF_UPCA',10,b'\xFF\xFF\xFF\x0BL_CLONE',2,b'\xFF\xFF\xFF\x0BL_COPY',1,b'\xFF\xFF\xFF\x0BL_COPY_CLONE',3,b'\xFF\xFF\xFF\x0BL_DEFAULT_ENCODE',0,b'\xFF\xFF\xFF\x0BL_FLATE_ENCODE',3,b'\xFF\xFF\xFF\x0BL_G4_ENCODE',2,b'\xFF\xFF\xFF\x0BL_INSERT',0,b'\xFF\xFF\xFF\x0BL_JP2K_ENCODE',4,b'\xFF\xFF\xFF\x0BL_JPEG_ENCODE',1,b'\xFF\xFF\xFF\x0BL_NOCOPY',0,b'\xFF\xFF\xFF\x0BL_SEVERITY_ALL',1,b'\xFF\xFF\xFF\x0BL_SEVERITY_DEBUG',2,b'\xFF\xFF\xFF\x0BL_SEVERITY_ERROR',5,b'\xFF\xFF\xFF\x0BL_SEVERITY_EXTERNAL',0,b'\xFF\xFF\xFF\x0BL_SEVERITY_INFO',3,b'\xFF\xFF\xFF\x0BL_SEVERITY_NONE',6,b'\xFF\xFF\xFF\x0BL_SEVERITY_WARNING',4,b'\xFF\xFF\xFF\x0BL_USE_WIDTHS',1,b'\xFF\xFF\xFF\x0BL_USE_WINDOWS',2,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_BASED_ON_SRC',4,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_BINARY',0,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_FULL_COLOR',2,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_GRAYSCALE',1,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_WITH_ALPHA',3,b'\xFF\xFF\xFF\x0BSEL_DONT_CARE',0,b'\xFF\xFF\xFF\x0BSEL_HIT',1,b'\xFF\xFF\xFF\x0BSEL_MISS',2,b'\x00\x00\x00\x23boxClone',0,b'\x00\x01\x1B\x23boxDestroy',0,b'\x00\x01\x1E\x23boxaDestroy',0,b'\x00\x00\x03\x23boxaGetBox',0,b'\x00\x00\xB6\x23getLeptonicaVersion',0,b'\x00\x01\x21\x23l_CIDataDestroy',0,b'\x00\x01\x06\x23l_generateCIDataForPdf',0,b'\x00\x01\x30\x23lept_free',0,b'\x00\x00\xB8\x23makePixelSumTab8',0,b'\x00\x00\x29\x23pixAnd',0,b'\x00\x00\x36\x23pixBackgroundNorm',0,b'\x00\x00\x2E\x23pixCleanBackgroundToWhite',0,b'\x00\x00\x20\x23pixClipRectangle',0,b'\x00\x00\xEE\x23pixColorFraction',0,b'\x00\x00\x7D\x23pixColorMagnitude',0,b'\x00\x00\x1D\x23pixConvertRGBToLuminance',0,b'\x00\x00\x74\x23pixConvertTo8',0,b'\x00\x00\xC0\x23pixCorrelationBinary',0,b'\x00\x00\xDA\x23pixCountPixels',0,b'\x00\x00\x90\x23pixDeserializeFromMemory',0,b'\x00\x00\x74\x23pixDeskew',0,b'\x00\x01\x24\x23pixDestroy',0,b'\x00\x00\x42\x23pixDilate',0,b'\x00\x00\x1D\x23pixEndianByteSwapNew',0,b'\x00\x00\xC5\x23pixEqual',0,b'\x00\x00\x42\x23pixErode',0,b'\x00\x00\x94\x23pixExtractBarcodes',0,b'\x00\x00\x08\x23pixFindPageForeground',0,b'\x00\x00\xD5\x23pixFindSkew',0,b'\x00\x00\x47\x23pixGammaTRC',0,b'\x00\x00\xE7\x23pixGenerateCIData',0,b'\x00\x00\xCA\x23pixGetAverageMaskedRGB',0,b'\x00\x00\x4E\x23pixGlobalNormRGB',0,b'\x00\x00\x42\x23pixHMT',0,b'\x00\x00\x25\x23pixInvert',0,b'\x00\x00\x17\x23pixLocateBarcodes',0,b'\x00\x00\x78\x23pixMaskOverColorPixels',0,b'\x00\x00\x56\x23pixMaskedThreshOnBackgroundNorm',0,b'\x00\x00\xDF\x23pixNumSignificantGrayColors',0,b'\x00\x00\xF7\x23pixOtsuAdaptiveThreshold',0,b'\x00\x00\x62\x23pixOtsuThreshOnBackgroundNorm',0,b'\x00\x00\x98\x23pixProcessBarcodes',0,b'\x00\x00\x89\x23pixRead',0,b'\x00\x00\x9F\x23pixReadBarcodes',0,b'\x00\x00\x8C\x23pixReadMem',0,b'\x00\x00\x74\x23pixRemoveColormap',0,b'\x00\x00\x78\x23pixRemoveColormapGeneral',0,b'\x00\x00\xBA\x23pixRenderBoxa',0,b'\x00\x00\x25\x23pixRotate180',0,b'\x00\x00\x74\x23pixRotateOrth',0,b'\x00\x00\x6F\x23pixScale',0,b'\x00\x01\x01\x23pixSerializeToMemory',0,b'\x00\x00\x29\x23pixSubtract',0,b'\x00\x01\x0C\x23pixWriteImpliedFormat',0,b'\x00\x01\x15\x23pixWriteMemPng',0,b'\x00\x01\x27\x23pixaDestroy',0,b'\x00\x00\x12\x23pixaGetBox',0,b'\x00\x00\x84\x23pixaGetPix',0,b'\x00\x01\x2A\x23sarrayDestroy',0,b'\x00\x00\xAC\x23selCreateBrick',0,b'\x00\x00\xA6\x23selCreateFromString',0,b'\x00\x01\x2D\x23selDestroy',0,b'\x00\x00\xB3\x23selPrintToString',0,b'\x00\x01\x12\x23setMsgSeverity',0), - _struct_unions = ((b'\x00\x00\x01\x33\x00\x00\x00\x02Box',b'\x00\x00\x05\x11x',b'\x00\x00\x05\x11y',b'\x00\x00\x05\x11w',b'\x00\x00\x05\x11h',b'\x00\x01\x50\x11refcount'),(b'\x00\x00\x01\x34\x00\x00\x00\x02Boxa',b'\x00\x00\x05\x11n',b'\x00\x00\x05\x11nalloc',b'\x00\x01\x50\x11refcount',b'\x00\x00\x23\x11box'),(b'\x00\x00\x01\x36\x00\x00\x00\x02L_Compressed_Data',b'\x00\x00\x05\x11type',b'\x00\x01\x4D\x11datacomp',b'\x00\x00\x8E\x11nbytescomp',b'\x00\x01\x3E\x11data85',b'\x00\x00\x8E\x11nbytes85',b'\x00\x01\x3E\x11cmapdata85',b'\x00\x01\x3E\x11cmapdatahex',b'\x00\x00\x05\x11ncolors',b'\x00\x00\x05\x11w',b'\x00\x00\x05\x11h',b'\x00\x00\x05\x11bps',b'\x00\x00\x05\x11spp',b'\x00\x00\x05\x11minisblack',b'\x00\x00\x05\x11predictor',b'\x00\x00\x8E\x11nbytes',b'\x00\x00\x05\x11res'),(b'\x00\x00\x01\x37\x00\x00\x00\x02Pix',b'\x00\x01\x50\x11w',b'\x00\x01\x50\x11h',b'\x00\x01\x50\x11d',b'\x00\x01\x50\x11spp',b'\x00\x01\x50\x11wpl',b'\x00\x01\x50\x11refcount',b'\x00\x00\x05\x11xres',b'\x00\x00\x05\x11yres',b'\x00\x00\x05\x11informat',b'\x00\x00\x05\x11special',b'\x00\x01\x3E\x11text',b'\x00\x01\x4C\x11colormap',b'\x00\x01\x4F\x11data'),(b'\x00\x00\x01\x39\x00\x00\x00\x02PixColormap',b'\x00\x01\x31\x11array',b'\x00\x00\x05\x11depth',b'\x00\x00\x05\x11nalloc',b'\x00\x00\x05\x11n'),(b'\x00\x00\x01\x38\x00\x00\x00\x02Pixa',b'\x00\x00\x05\x11n',b'\x00\x00\x05\x11nalloc',b'\x00\x01\x50\x11refcount',b'\x00\x00\x1A\x11pix',b'\x00\x00\x04\x11boxa'),(b'\x00\x00\x01\x3B\x00\x00\x00\x02Sarray',b'\x00\x00\x05\x11nalloc',b'\x00\x00\x05\x11n',b'\x00\x00\x05\x11refcount',b'\x00\x01\x3D\x11array'),(b'\x00\x00\x01\x3C\x00\x00\x00\x02Sel',b'\x00\x00\x05\x11sy',b'\x00\x00\x05\x11sx',b'\x00\x00\x05\x11cy',b'\x00\x00\x05\x11cx',b'\x00\x01\x48\x11data',b'\x00\x01\x3E\x11name')), - _enums = (b'\x00\x00\x01\x41\x00\x00\x00\x16$1\x00L_DEFAULT_ENCODE,L_JPEG_ENCODE,L_G4_ENCODE,L_FLATE_ENCODE,L_JP2K_ENCODE',b'\x00\x00\x01\x42\x00\x00\x00\x16$2\x00REMOVE_CMAP_TO_BINARY,REMOVE_CMAP_TO_GRAYSCALE,REMOVE_CMAP_TO_FULL_COLOR,REMOVE_CMAP_WITH_ALPHA,REMOVE_CMAP_BASED_ON_SRC',b'\x00\x00\x01\x43\x00\x00\x00\x16$3\x00L_NOCOPY,L_INSERT,L_COPY,L_CLONE,L_COPY_CLONE',b'\x00\x00\x01\x44\x00\x00\x00\x16$4\x00L_USE_WIDTHS,L_USE_WINDOWS',b'\x00\x00\x01\x45\x00\x00\x00\x16$5\x00L_BF_UNKNOWN,L_BF_ANY,L_BF_CODE128,L_BF_EAN8,L_BF_EAN13,L_BF_CODE2OF5,L_BF_CODEI2OF5,L_BF_CODE39,L_BF_CODE93,L_BF_CODABAR,L_BF_UPCA',b'\x00\x00\x01\x46\x00\x00\x00\x16$6\x00L_SEVERITY_EXTERNAL,L_SEVERITY_ALL,L_SEVERITY_DEBUG,L_SEVERITY_INFO,L_SEVERITY_WARNING,L_SEVERITY_ERROR,L_SEVERITY_NONE',b'\x00\x00\x01\x47\x00\x00\x00\x16$7\x00SEL_DONT_CARE,SEL_HIT,SEL_MISS'), - _typenames = (b'\x00\x00\x01\x33BOX',b'\x00\x00\x01\x34BOXA',b'\x00\x00\x01\x36L_COMP_DATA',b'\x00\x00\x01\x37PIX',b'\x00\x00\x01\x38PIXA',b'\x00\x00\x01\x39PIXCMAP',b'\x00\x00\x01\x3BSARRAY',b'\x00\x00\x01\x3CSEL',b'\x00\x00\x00\x32l_float32',b'\x00\x00\x01\x40l_float64',b'\x00\x00\x01\x4Al_int16',b'\x00\x00\x00\x05l_int32',b'\x00\x00\x01\x49l_int64',b'\x00\x00\x01\x4Bl_int8',b'\x00\x00\x00\x05l_ok',b'\x00\x00\x01\x52l_uint16',b'\x00\x00\x01\x50l_uint32',b'\x00\x00\x01\x51l_uint64',b'\x00\x00\x01\x4El_uint8'), +ffi = _cffi_backend.FFI( + 'ocrmypdf.lib._leptonica', + _version=0x2601, + _types=b'\x00\x00\x01\x0D\x00\x01\x33\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x34\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x37\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x3F\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x38\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x04\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x09\x03\x00\x00\x1A\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x01\x11\x00\x00\x01\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x01\x3C\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x05\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x60\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x13\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x01\x4E\x03\x00\x00\x1C\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x01\x50\x03\x00\x00\x1C\x01\x00\x00\x00\x0F\x00\x00\x13\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x3A\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x3A\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x3A\x0D\x00\x00\x13\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x9C\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x45\x0D\x00\x00\x10\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x00\x45\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x3E\x0D\x00\x00\x45\x11\x00\x00\x00\x0F\x00\x01\x3E\x0D\x00\x00\x00\x0F\x00\x00\x60\x0D\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x04\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x32\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x60\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xC3\x11\x00\x00\xC3\x11\x00\x00\xC3\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\xC3\x11\x00\x00\xC3\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x60\x11\x00\x00\x60\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x60\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x35\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xC3\x11\x00\x00\xC3\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x01\x4F\x03\x00\x00\x8E\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x10\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\xEC\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x10\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x4D\x03\x00\x01\x04\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x23\x11\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x04\x03\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\xEC\x11\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x1A\x11\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x13\x03\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x9C\x11\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x45\x03\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x01\x53\x03\x00\x00\x00\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x01\x36\x03\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x05\x09\x00\x00\x04\x09\x00\x01\x3B\x03\x00\x00\x06\x09\x00\x00\x07\x09\x00\x01\x3E\x03\x00\x01\x3F\x03\x00\x00\x02\x01\x00\x00\x0E\x01\x00\x00\x00\x0B\x00\x00\x01\x0B\x00\x00\x02\x0B\x00\x00\x03\x0B\x00\x00\x04\x0B\x00\x00\x05\x0B\x00\x00\x06\x0B\x00\x00\x60\x03\x00\x00\x0B\x01\x00\x00\x05\x01\x00\x00\x03\x01\x00\x01\x39\x03\x00\x01\x4E\x03\x00\x00\x04\x01\x00\x01\x50\x03\x00\x00\x08\x01\x00\x00\x0C\x01\x00\x00\x06\x01\x00\x00\x00\x01', + _globals=( + b'\xFF\xFF\xFF\x0BL_BF_ANY', + 1, + b'\xFF\xFF\xFF\x0BL_BF_CODABAR', + 9, + b'\xFF\xFF\xFF\x0BL_BF_CODE128', + 2, + b'\xFF\xFF\xFF\x0BL_BF_CODE2OF5', + 5, + b'\xFF\xFF\xFF\x0BL_BF_CODE39', + 7, + b'\xFF\xFF\xFF\x0BL_BF_CODE93', + 8, + b'\xFF\xFF\xFF\x0BL_BF_CODEI2OF5', + 6, + b'\xFF\xFF\xFF\x0BL_BF_EAN13', + 4, + b'\xFF\xFF\xFF\x0BL_BF_EAN8', + 3, + b'\xFF\xFF\xFF\x0BL_BF_UNKNOWN', + 0, + b'\xFF\xFF\xFF\x0BL_BF_UPCA', + 10, + b'\xFF\xFF\xFF\x0BL_CLONE', + 2, + b'\xFF\xFF\xFF\x0BL_COPY', + 1, + b'\xFF\xFF\xFF\x0BL_COPY_CLONE', + 3, + b'\xFF\xFF\xFF\x0BL_DEFAULT_ENCODE', + 0, + b'\xFF\xFF\xFF\x0BL_FLATE_ENCODE', + 3, + b'\xFF\xFF\xFF\x0BL_G4_ENCODE', + 2, + b'\xFF\xFF\xFF\x0BL_INSERT', + 0, + b'\xFF\xFF\xFF\x0BL_JP2K_ENCODE', + 4, + b'\xFF\xFF\xFF\x0BL_JPEG_ENCODE', + 1, + b'\xFF\xFF\xFF\x0BL_NOCOPY', + 0, + b'\xFF\xFF\xFF\x0BL_SEVERITY_ALL', + 1, + b'\xFF\xFF\xFF\x0BL_SEVERITY_DEBUG', + 2, + b'\xFF\xFF\xFF\x0BL_SEVERITY_ERROR', + 5, + b'\xFF\xFF\xFF\x0BL_SEVERITY_EXTERNAL', + 0, + b'\xFF\xFF\xFF\x0BL_SEVERITY_INFO', + 3, + b'\xFF\xFF\xFF\x0BL_SEVERITY_NONE', + 6, + b'\xFF\xFF\xFF\x0BL_SEVERITY_WARNING', + 4, + b'\xFF\xFF\xFF\x0BL_USE_WIDTHS', + 1, + b'\xFF\xFF\xFF\x0BL_USE_WINDOWS', + 2, + b'\xFF\xFF\xFF\x0BREMOVE_CMAP_BASED_ON_SRC', + 4, + b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_BINARY', + 0, + b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_FULL_COLOR', + 2, + b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_GRAYSCALE', + 1, + b'\xFF\xFF\xFF\x0BREMOVE_CMAP_WITH_ALPHA', + 3, + b'\xFF\xFF\xFF\x0BSEL_DONT_CARE', + 0, + b'\xFF\xFF\xFF\x0BSEL_HIT', + 1, + b'\xFF\xFF\xFF\x0BSEL_MISS', + 2, + b'\x00\x00\x00\x23boxClone', + 0, + b'\x00\x01\x1B\x23boxDestroy', + 0, + b'\x00\x01\x1E\x23boxaDestroy', + 0, + b'\x00\x00\x03\x23boxaGetBox', + 0, + b'\x00\x00\xB6\x23getLeptonicaVersion', + 0, + b'\x00\x01\x21\x23l_CIDataDestroy', + 0, + b'\x00\x01\x06\x23l_generateCIDataForPdf', + 0, + b'\x00\x01\x30\x23lept_free', + 0, + b'\x00\x00\xB8\x23makePixelSumTab8', + 0, + b'\x00\x00\x29\x23pixAnd', + 0, + b'\x00\x00\x36\x23pixBackgroundNorm', + 0, + b'\x00\x00\x2E\x23pixCleanBackgroundToWhite', + 0, + b'\x00\x00\x20\x23pixClipRectangle', + 0, + b'\x00\x00\xEE\x23pixColorFraction', + 0, + b'\x00\x00\x7D\x23pixColorMagnitude', + 0, + b'\x00\x00\x1D\x23pixConvertRGBToLuminance', + 0, + b'\x00\x00\x74\x23pixConvertTo8', + 0, + b'\x00\x00\xC0\x23pixCorrelationBinary', + 0, + b'\x00\x00\xDA\x23pixCountPixels', + 0, + b'\x00\x00\x90\x23pixDeserializeFromMemory', + 0, + b'\x00\x00\x74\x23pixDeskew', + 0, + b'\x00\x01\x24\x23pixDestroy', + 0, + b'\x00\x00\x42\x23pixDilate', + 0, + b'\x00\x00\x1D\x23pixEndianByteSwapNew', + 0, + b'\x00\x00\xC5\x23pixEqual', + 0, + b'\x00\x00\x42\x23pixErode', + 0, + b'\x00\x00\x94\x23pixExtractBarcodes', + 0, + b'\x00\x00\x08\x23pixFindPageForeground', + 0, + b'\x00\x00\xD5\x23pixFindSkew', + 0, + b'\x00\x00\x47\x23pixGammaTRC', + 0, + b'\x00\x00\xE7\x23pixGenerateCIData', + 0, + b'\x00\x00\xCA\x23pixGetAverageMaskedRGB', + 0, + b'\x00\x00\x4E\x23pixGlobalNormRGB', + 0, + b'\x00\x00\x42\x23pixHMT', + 0, + b'\x00\x00\x25\x23pixInvert', + 0, + b'\x00\x00\x17\x23pixLocateBarcodes', + 0, + b'\x00\x00\x78\x23pixMaskOverColorPixels', + 0, + b'\x00\x00\x56\x23pixMaskedThreshOnBackgroundNorm', + 0, + b'\x00\x00\xDF\x23pixNumSignificantGrayColors', + 0, + b'\x00\x00\xF7\x23pixOtsuAdaptiveThreshold', + 0, + b'\x00\x00\x62\x23pixOtsuThreshOnBackgroundNorm', + 0, + b'\x00\x00\x98\x23pixProcessBarcodes', + 0, + b'\x00\x00\x89\x23pixRead', + 0, + b'\x00\x00\x9F\x23pixReadBarcodes', + 0, + b'\x00\x00\x8C\x23pixReadMem', + 0, + b'\x00\x00\x74\x23pixRemoveColormap', + 0, + b'\x00\x00\x78\x23pixRemoveColormapGeneral', + 0, + b'\x00\x00\xBA\x23pixRenderBoxa', + 0, + b'\x00\x00\x25\x23pixRotate180', + 0, + b'\x00\x00\x74\x23pixRotateOrth', + 0, + b'\x00\x00\x6F\x23pixScale', + 0, + b'\x00\x01\x01\x23pixSerializeToMemory', + 0, + b'\x00\x00\x29\x23pixSubtract', + 0, + b'\x00\x01\x0C\x23pixWriteImpliedFormat', + 0, + b'\x00\x01\x15\x23pixWriteMemPng', + 0, + b'\x00\x01\x27\x23pixaDestroy', + 0, + b'\x00\x00\x12\x23pixaGetBox', + 0, + b'\x00\x00\x84\x23pixaGetPix', + 0, + b'\x00\x01\x2A\x23sarrayDestroy', + 0, + b'\x00\x00\xAC\x23selCreateBrick', + 0, + b'\x00\x00\xA6\x23selCreateFromString', + 0, + b'\x00\x01\x2D\x23selDestroy', + 0, + b'\x00\x00\xB3\x23selPrintToString', + 0, + b'\x00\x01\x12\x23setMsgSeverity', + 0, + ), + _struct_unions=( + ( + b'\x00\x00\x01\x33\x00\x00\x00\x02Box', + b'\x00\x00\x05\x11x', + b'\x00\x00\x05\x11y', + b'\x00\x00\x05\x11w', + b'\x00\x00\x05\x11h', + b'\x00\x01\x50\x11refcount', + ), + ( + b'\x00\x00\x01\x34\x00\x00\x00\x02Boxa', + b'\x00\x00\x05\x11n', + b'\x00\x00\x05\x11nalloc', + b'\x00\x01\x50\x11refcount', + b'\x00\x00\x23\x11box', + ), + ( + b'\x00\x00\x01\x36\x00\x00\x00\x02L_Compressed_Data', + b'\x00\x00\x05\x11type', + b'\x00\x01\x4D\x11datacomp', + b'\x00\x00\x8E\x11nbytescomp', + b'\x00\x01\x3E\x11data85', + b'\x00\x00\x8E\x11nbytes85', + b'\x00\x01\x3E\x11cmapdata85', + b'\x00\x01\x3E\x11cmapdatahex', + b'\x00\x00\x05\x11ncolors', + b'\x00\x00\x05\x11w', + b'\x00\x00\x05\x11h', + b'\x00\x00\x05\x11bps', + b'\x00\x00\x05\x11spp', + b'\x00\x00\x05\x11minisblack', + b'\x00\x00\x05\x11predictor', + b'\x00\x00\x8E\x11nbytes', + b'\x00\x00\x05\x11res', + ), + ( + b'\x00\x00\x01\x37\x00\x00\x00\x02Pix', + b'\x00\x01\x50\x11w', + b'\x00\x01\x50\x11h', + b'\x00\x01\x50\x11d', + b'\x00\x01\x50\x11spp', + b'\x00\x01\x50\x11wpl', + b'\x00\x01\x50\x11refcount', + b'\x00\x00\x05\x11xres', + b'\x00\x00\x05\x11yres', + b'\x00\x00\x05\x11informat', + b'\x00\x00\x05\x11special', + b'\x00\x01\x3E\x11text', + b'\x00\x01\x4C\x11colormap', + b'\x00\x01\x4F\x11data', + ), + ( + b'\x00\x00\x01\x39\x00\x00\x00\x02PixColormap', + b'\x00\x01\x31\x11array', + b'\x00\x00\x05\x11depth', + b'\x00\x00\x05\x11nalloc', + b'\x00\x00\x05\x11n', + ), + ( + b'\x00\x00\x01\x38\x00\x00\x00\x02Pixa', + b'\x00\x00\x05\x11n', + b'\x00\x00\x05\x11nalloc', + b'\x00\x01\x50\x11refcount', + b'\x00\x00\x1A\x11pix', + b'\x00\x00\x04\x11boxa', + ), + ( + b'\x00\x00\x01\x3B\x00\x00\x00\x02Sarray', + b'\x00\x00\x05\x11nalloc', + b'\x00\x00\x05\x11n', + b'\x00\x00\x05\x11refcount', + b'\x00\x01\x3D\x11array', + ), + ( + b'\x00\x00\x01\x3C\x00\x00\x00\x02Sel', + b'\x00\x00\x05\x11sy', + b'\x00\x00\x05\x11sx', + b'\x00\x00\x05\x11cy', + b'\x00\x00\x05\x11cx', + b'\x00\x01\x48\x11data', + b'\x00\x01\x3E\x11name', + ), + ), + _enums=( + b'\x00\x00\x01\x41\x00\x00\x00\x16$1\x00L_DEFAULT_ENCODE,L_JPEG_ENCODE,L_G4_ENCODE,L_FLATE_ENCODE,L_JP2K_ENCODE', + b'\x00\x00\x01\x42\x00\x00\x00\x16$2\x00REMOVE_CMAP_TO_BINARY,REMOVE_CMAP_TO_GRAYSCALE,REMOVE_CMAP_TO_FULL_COLOR,REMOVE_CMAP_WITH_ALPHA,REMOVE_CMAP_BASED_ON_SRC', + b'\x00\x00\x01\x43\x00\x00\x00\x16$3\x00L_NOCOPY,L_INSERT,L_COPY,L_CLONE,L_COPY_CLONE', + b'\x00\x00\x01\x44\x00\x00\x00\x16$4\x00L_USE_WIDTHS,L_USE_WINDOWS', + b'\x00\x00\x01\x45\x00\x00\x00\x16$5\x00L_BF_UNKNOWN,L_BF_ANY,L_BF_CODE128,L_BF_EAN8,L_BF_EAN13,L_BF_CODE2OF5,L_BF_CODEI2OF5,L_BF_CODE39,L_BF_CODE93,L_BF_CODABAR,L_BF_UPCA', + b'\x00\x00\x01\x46\x00\x00\x00\x16$6\x00L_SEVERITY_EXTERNAL,L_SEVERITY_ALL,L_SEVERITY_DEBUG,L_SEVERITY_INFO,L_SEVERITY_WARNING,L_SEVERITY_ERROR,L_SEVERITY_NONE', + b'\x00\x00\x01\x47\x00\x00\x00\x16$7\x00SEL_DONT_CARE,SEL_HIT,SEL_MISS', + ), + _typenames=( + b'\x00\x00\x01\x33BOX', + b'\x00\x00\x01\x34BOXA', + b'\x00\x00\x01\x36L_COMP_DATA', + b'\x00\x00\x01\x37PIX', + b'\x00\x00\x01\x38PIXA', + b'\x00\x00\x01\x39PIXCMAP', + b'\x00\x00\x01\x3BSARRAY', + b'\x00\x00\x01\x3CSEL', + b'\x00\x00\x00\x32l_float32', + b'\x00\x00\x01\x40l_float64', + b'\x00\x00\x01\x4Al_int16', + b'\x00\x00\x00\x05l_int32', + b'\x00\x00\x01\x49l_int64', + b'\x00\x00\x01\x4Bl_int8', + b'\x00\x00\x00\x05l_ok', + b'\x00\x00\x01\x52l_uint16', + b'\x00\x00\x01\x50l_uint32', + b'\x00\x00\x01\x51l_uint64', + b'\x00\x00\x01\x4El_uint8', + ), ) diff --git a/src/ocrmypdf/lib/compile_leptonica.py b/src/ocrmypdf/lib/compile_leptonica.py index 4418e7cf..c76dd324 100644 --- a/src/ocrmypdf/lib/compile_leptonica.py +++ b/src/ocrmypdf/lib/compile_leptonica.py @@ -19,7 +19,8 @@ from cffi import FFI ffibuilder = FFI() -ffibuilder.cdef(""" +ffibuilder.cdef( + """ typedef signed char l_int8; typedef unsigned char l_uint8; typedef short l_int16; @@ -202,9 +203,11 @@ enum { SEL_MISS = 2 }; -""") +""" +) -ffibuilder.cdef(""" +ffibuilder.cdef( + """ PIX * pixRead ( const char *filename ); PIX * pixReadMem ( const l_uint8 *data, size_t size ); PIX * pixScale ( PIX *pixs, l_float32 scalex, l_float32 scaley ); @@ -480,7 +483,8 @@ void selDestroy ( SEL **psel ); l_int32 setMsgSeverity(l_int32 newsev); -""") +""" +) ffibuilder.set_source("ocrmypdf.lib._leptonica", None) diff --git a/src/ocrmypdf/optimize.py b/src/ocrmypdf/optimize.py index fe375caf..3c0a32ec 100644 --- a/src/ocrmypdf/optimize.py +++ b/src/ocrmypdf/optimize.py @@ -80,9 +80,11 @@ def extract_image_jbig2(*, pike, root, log, image, xref, options): return None pim, filtdp = result - if pim.bits_per_component == 1 \ - and filtdp != '/JBIG2Decode' \ - and jbig2enc.available(): + if ( + pim.bits_per_component == 1 + and filtdp != '/JBIG2Decode' + and jbig2enc.available() + ): try: imgname = Path(root / '{:08d}'.format(xref)) with imgname.open('wb') as f: @@ -100,8 +102,7 @@ def extract_image_generic(*, pike, root, log, image, xref, options): return None pim, filtdp = result - if filtdp[0] == '/DCTDecode' \ - and options.optimize >= 2: + if filtdp[0] == '/DCTDecode' and options.optimize >= 2: # This is a simple heuristic derived from some training data, that has # about a 70% chance of guessing whether the JPEG is high quality, # and possibly recompressible, or not. The number itself doesn't mean @@ -126,9 +127,11 @@ def extract_image_generic(*, pike, root, log, image, xref, options): except pikepdf.UnsupportedImageTypeError: return None return xref, ext - elif pim.indexed \ - and pim.colorspace in pim.SIMPLE_COLORSPACES \ - and options.optimize >= 3: + elif ( + pim.indexed + and pim.colorspace in pim.SIMPLE_COLORSPACES + and options.optimize >= 3 + ): # Try to improve on indexed images - these are far from low hanging # fruit in most cases pim.as_pil_image().save(png_name(root, xref)) @@ -142,7 +145,6 @@ def extract_image_generic(*, pike, root, log, image, xref, options): return None - def extract_images(pike, root, log, options, extract_fn): """Extract image using extract_fn @@ -172,8 +174,7 @@ def extract_images(pike, root, log, options, extract_fn): image = pike.get_object((xref, 0)) try: result = extract_fn( - pike=pike, root=root, log=log, image=image, - xref=xref, options=options + pike=pike, root=root, log=log, image=image, xref=xref, options=options ) except Exception as e: log.debug("Image xref %s", xref) @@ -190,17 +191,13 @@ def extract_images_generic(pike, root, log, options): jpegs = [] pngs = [] - for _, xref, ext in extract_images( - pike, root, log, options, extract_image_generic): + for _, xref, ext in extract_images(pike, root, log, options, extract_image_generic): log.debug('xref = %s ext = %s', xref, ext) if ext == '.png': pngs.append(xref) elif ext == '.jpg': jpegs.append(xref) - log.debug( - "Optimizable images: " - "JPEGs: %s PNGs: %s", len(jpegs), len(pngs) - ) + log.debug("Optimizable images: " "JPEGs: %s PNGs: %s", len(jpegs), len(pngs)) return jpegs, pngs @@ -209,17 +206,16 @@ def extract_images_jbig2(pike, root, log, options): jbig2_groups = defaultdict(list) for pageno, xref, ext in extract_images( - pike, root, log, options, extract_image_jbig2): + pike, root, log, options, extract_image_jbig2 + ): group = pageno // options.jbig2_page_group_size jbig2_groups[group].append((xref, ext)) # Elide empty groups - jbig2_groups = {group: xrefs for group, xrefs in jbig2_groups.items() - if len(xrefs) > 0} - log.debug( - "Optimizable images: " - "JBIG2 groups: %s", (len(jbig2_groups),) - ) + jbig2_groups = { + group: xrefs for group, xrefs in jbig2_groups.items() if len(xrefs) > 0 + } + log.debug("Optimizable images: " "JBIG2 groups: %s", (len(jbig2_groups),)) return jbig2_groups @@ -233,7 +229,7 @@ def _produce_jbig2_images(jbig2_groups, root, log, options): jbig2enc.convert_group, cwd=fspath(root), infiles=(img_name(root, xref, ext) for xref, ext in xref_exts), - out_prefix=prefix + out_prefix=prefix, ) yield future @@ -247,7 +243,7 @@ def _produce_jbig2_images(jbig2_groups, root, log, options): jbig2enc.convert_single, cwd=fspath(root), infile=img_name(root, xref, ext), - outfile=root / ('{}.{:04d}'.format(prefix, n)) + outfile=root / ('{}.{:04d}'.format(prefix, n)), ) yield future @@ -256,8 +252,7 @@ def _produce_jbig2_images(jbig2_groups, root, log, options): else: jbig2_futures = jbig2_single_futures - with concurrent.futures.ThreadPoolExecutor( - max_workers=options.jobs) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=options.jobs) as executor: futures = jbig2_futures(executor, root, jbig2_groups) for future in concurrent.futures.as_completed(futures): proc = future.result() @@ -286,9 +281,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options): if jbig2_symfile.exists(): jbig2_globals_data = jbig2_symfile.read_bytes() jbig2_globals = pikepdf.Stream(pike, jbig2_globals_data) - jbig2_globals_dict = pikepdf.Dictionary({ - '/JBIG2Globals': jbig2_globals - }) + jbig2_globals_dict = pikepdf.Dictionary({'/JBIG2Globals': jbig2_globals}) elif options.jbig2_page_group_size == 1: jbig2_globals_dict = None else: @@ -302,7 +295,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options): im_obj.write( jbig2_im_data, filter=pikepdf.Name('/JBIG2Decode'), - decode_parms=jbig2_globals_dict + decode_parms=jbig2_globals_dict, ) @@ -316,9 +309,7 @@ def transcode_jpegs(pike, jpegs, root, log, options): # 'close'. Seems to be mostly harmless # https://github.com/python-pillow/Pillow/issues/1144 with Image.open(fspath(in_jpg)) as im: - im.save(fspath(opt_jpg), - optimize=True, - quality=options.jpeg_quality) + im.save(fspath(opt_jpg), optimize=True, quality=options.jpeg_quality) # pylint: disable=no-member if opt_jpg.stat().st_size > in_jpg.stat().st_size: log.debug("xref %s, jpeg, made larger - skip", xref) @@ -326,24 +317,26 @@ def transcode_jpegs(pike, jpegs, root, log, options): compdata = leptonica.CompressedData.open(opt_jpg) im_obj = pike.get_object(xref, 0) - im_obj.write( - compdata.read(), filter=pikepdf.Name('/DCTDecode') - ) + im_obj.write(compdata.read(), filter=pikepdf.Name('/DCTDecode')) def transcode_pngs(pike, pngs, root, log, options): if options.optimize >= 2: png_quality = ( max(10, options.png_quality - 10), - min(100, options.png_quality + 10) + min(100, options.png_quality + 10), ) with concurrent.futures.ThreadPoolExecutor( - max_workers=options.jobs) as executor: + max_workers=options.jobs + ) as executor: for xref in pngs: executor.submit( pngquant.quantize, - png_name(root, xref), png_name(root, xref), - png_quality[0], png_quality[1]) + png_name(root, xref), + png_name(root, xref), + png_quality[0], + png_quality[1], + ) for xref in pngs: im_obj = pike.get_object(xref, 0) @@ -353,9 +346,7 @@ def transcode_pngs(pike, pngs, root, log, options): pix = leptonica.Pix.open(png_name(root, xref)) if pix.depth == 1: pix = pix.invert() # PDF assumes 1 is black for monochrome - compdata = pix.generate_pdf_ci_data( - leptonica.lept.L_FLATE_ENCODE, 0 - ) + compdata = pix.generate_pdf_ci_data(leptonica.lept.L_FLATE_ENCODE, 0) except leptonica.LeptonicaError as e: log.error(e) continue @@ -363,7 +354,7 @@ def transcode_pngs(pike, pngs, root, log, options): # This is what we should be doing: open the compressed data without # transcoding. However this shifts each pixel row by one for some # reason. - #compdata = leptonica.CompressedData.open(png_name(root, xref)) + # compdata = leptonica.CompressedData.open(png_name(root, xref)) if len(compdata) > int(im_obj.stream_dict.Length): continue # If we produced a larger image, don't use @@ -379,8 +370,12 @@ def transcode_pngs(pike, pngs, root, log, options): palette_pdf_string = compdata.get_palette_pdf_string() palette_data = pikepdf.Object.parse(palette_pdf_string) palette_stream = pikepdf.Stream(pike, bytes(palette_data)) - palette = [pikepdf.Name('/Indexed'), pikepdf.Name('/DeviceRGB'), - compdata.ncolors - 1, palette_stream] + palette = [ + pikepdf.Name('/Indexed'), + pikepdf.Name('/DeviceRGB'), + compdata.ncolors - 1, + palette_stream, + ] cs = palette else: if compdata.spp == 1: @@ -391,16 +386,11 @@ def transcode_pngs(pike, pngs, root, log, options): cs = pikepdf.Name('/DeviceCMYK') im_obj.ColorSpace = cs im_obj.write( - compdata.read(), - filter=pikepdf.Name('/FlateDecode'), decode_parms=predictor + compdata.read(), filter=pikepdf.Name('/FlateDecode'), decode_parms=predictor ) -def optimize( - input_file, - output_file, - log, - context): +def optimize(input_file, output_file, log, context): options = context.get_options() if options.optimize == 0: @@ -408,14 +398,11 @@ def optimize( return if options.jpeg_quality == 0: - options.jpeg_quality = \ - DEFAULT_JPEG_QUALITY if options.optimize < 3 else 40 + options.jpeg_quality = DEFAULT_JPEG_QUALITY if options.optimize < 3 else 40 if options.png_quality == 0: - options.png_quality = \ - DEFAULT_PNG_QUALITY if options.optimize < 3 else 30 + options.png_quality = DEFAULT_PNG_QUALITY if options.optimize < 3 else 30 if options.jbig2_page_group_size == 0: - options.jbig2_page_group_size = \ - 10 if options.jbig2_lossy else 1 + options.jbig2_page_group_size = 10 if options.jbig2_lossy else 1 pike = pikepdf.Pdf.open(input_file) @@ -431,15 +418,17 @@ def optimize( target_file = Path(output_file).with_suffix('.opt.pdf') pike.remove_unreferenced_resources() - pike.save(target_file, preserve_pdfa=True, - object_stream_mode=pikepdf.ObjectStreamMode.generate) + pike.save( + target_file, + preserve_pdfa=True, + object_stream_mode=pikepdf.ObjectStreamMode.generate, + ) input_size = Path(input_file).stat().st_size output_size = Path(target_file).stat().st_size ratio = input_size / output_size savings = 1 - output_size / input_size - log.info("Optimize ratio: {:.2f} savings: {:.1f}%".format( - ratio, 100 * savings)) + log.info("Optimize ratio: {:.2f} savings: {:.1f}%".format(ratio, 100 * savings)) if savings < 0: log.info("Optimize did not improve the file - discarded") @@ -455,8 +444,7 @@ def main(infile, outfile, level, jobs=1): class OptimizeOptions: """Emulate ocrmypdf's options""" - def __init__( - self, jobs, optimize, jpeg_quality, png_quality, jb2lossy): + def __init__(self, jobs, optimize, jpeg_quality, png_quality, jb2lossy): self.jobs = jobs self.optimize = optimize self.jpeg_quality = jpeg_quality @@ -473,7 +461,7 @@ def main(infile, outfile, level, jobs=1): optimize=int(level), jpeg_quality=0, # Use default png_quality=0, - jb2lossy=False + jb2lossy=False, ) ctx.set_options(options) diff --git a/src/ocrmypdf/pdfa.py b/src/ocrmypdf/pdfa.py index 413003a7..6350622f 100644 --- a/src/ocrmypdf/pdfa.py +++ b/src/ocrmypdf/pdfa.py @@ -41,7 +41,7 @@ import pikepdf from pikepdf.models.metadata import ( encode_pdf_date as _encode_date, - decode_pdf_date as _decode_date + decode_pdf_date as _decode_date, ) from .helpers import deprecated @@ -49,8 +49,7 @@ from .helpers import deprecated ICC_PROFILE_RELPATH = 'data/sRGB.icc' -SRGB_ICC_PROFILE = pkg_resources.resource_filename( - 'ocrmypdf', ICC_PROFILE_RELPATH) +SRGB_ICC_PROFILE = pkg_resources.resource_filename('ocrmypdf', ICC_PROFILE_RELPATH) # This is a template written in PostScript which is needed to create PDF/A @@ -128,12 +127,7 @@ def _encode_ascii(s: str) -> str: be to implement PdfDocEncoding in pikepdf and encode to that, or handle metadata there. """ - trans = str.maketrans({ - '(': '', - ')': '', - '\\': '', - '\0': '' - }) + trans = str.maketrans({'(': '', ')': '', '\\': '', '\0': ''}) return s.translate(trans).encode('ascii', errors='replace').decode() @@ -163,8 +157,7 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark=None, ascii_docinfo=None) """ t = Template(pdfa_def_template) - result = t.substitute(icc_profile=icc_profile, - icc_identifier=icc_identifier) + result = t.substitute(icc_profile=icc_profile, icc_identifier=icc_identifier) return result @@ -205,8 +198,11 @@ def file_claims_pdfa(filename): pdf = pikepdf.open(filename) pdfmeta = pdf.open_metadata() if not pdfmeta.pdfa_status: - return {'pass': False, 'output': 'pdf', - 'conformance': 'No PDF/A metadata in XMP'} + return { + 'pass': False, + 'output': 'pdf', + 'conformance': 'No PDF/A metadata in XMP', + } valid_part_conforms = {'1A', '1B', '2A', '2B', '2U', '3A', '3B', '3U'} conformance = 'PDF/A-{}'.format(pdfmeta.pdfa_status) pdfa_dict = {} diff --git a/src/ocrmypdf/pdfinfo/__init__.py b/src/ocrmypdf/pdfinfo/__init__.py index 9b0845e8..377e93c9 100644 --- a/src/ocrmypdf/pdfinfo/__init__.py +++ b/src/ocrmypdf/pdfinfo/__init__.py @@ -35,12 +35,11 @@ from .layout import get_page_analysis, get_text_boxes from ..exceptions import EncryptedPdfError -Colorspace = Enum('Colorspace', - 'gray rgb cmyk lab icc index sep devn pattern jpeg2000') +Colorspace = Enum('Colorspace', 'gray rgb cmyk lab icc index sep devn pattern jpeg2000') -Encoding = Enum('Encoding', - 'ccitt jpeg jpeg2000 jbig2 asciihex ascii85 lzw flate ' + \ - 'runlength') +Encoding = Enum( + 'Encoding', 'ccitt jpeg jpeg2000 jbig2 asciihex ascii85 lzw flate ' + 'runlength' +) FRIENDLY_COLORSPACE = { '/DeviceGray': Colorspace.gray, @@ -71,7 +70,7 @@ FRIENDLY_ENCODING = { '/A85': Encoding.ascii85, '/LZW': Encoding.lzw, '/Fl': Encoding.flate, - '/RL': Encoding.runlength + '/RL': Encoding.runlength, } FRIENDLY_COMP = { @@ -79,28 +78,28 @@ FRIENDLY_COMP = { Colorspace.rgb: 3, Colorspace.cmyk: 4, Colorspace.lab: 3, - Colorspace.index: 1 + Colorspace.index: 1, } UNIT_SQUARE = (1.0, 0.0, 0.0, 1.0, 0.0, 0.0) + def _is_unit_square(shorthand): values = map(float, shorthand) pairwise = zip(values, UNIT_SQUARE) return all([isclose(a, b, rel_tol=1e-3) for a, b in pairwise]) -XobjectSettings = namedtuple('XobjectSettings', - ['name', 'shorthand', 'stack_depth']) -InlineSettings = namedtuple('InlineSettings', - ['iimage', 'shorthand', 'stack_depth']) +XobjectSettings = namedtuple('XobjectSettings', ['name', 'shorthand', 'stack_depth']) -ContentsInfo = namedtuple('ContentsInfo', - ['xobject_settings', 'inline_images', 'found_vector']) +InlineSettings = namedtuple('InlineSettings', ['iimage', 'shorthand', 'stack_depth']) -TextboxInfo = namedtuple('TextboxInfo', - ['bbox', 'is_visible', 'is_corrupt']) +ContentsInfo = namedtuple( + 'ContentsInfo', ['xobject_settings', 'inline_images', 'found_vector'] +) + +TextboxInfo = namedtuple('TextboxInfo', ['bbox', 'is_visible', 'is_corrupt']) class VectorInfo: @@ -112,9 +111,9 @@ def _normalize_stack(graphobjs): """Convert runs of qQ's in the stack into single graphobjs""" for operands, operator in graphobjs: operator = str(operator) - if re.match(r'Q*q+$', operator): # Zero or more Q, one or more q - for char in operator: # Split into individual - yield ([], char) # Yield individual + if re.match(r'Q*q+$', operator): # Zero or more Q, one or more q + for char in operator: # Split into individual + yield ([], char) # Yield individual else: yield (operands, operator) @@ -155,15 +154,19 @@ def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE): image_ops = set('BI ID EI q Q Do cm'.split()) operator_whitelist = ' '.join(vector_ops | image_ops) - for n, graphobj in enumerate(_normalize_stack( - pikepdf.parse_content_stream(contentstream, operator_whitelist))): + for n, graphobj in enumerate( + _normalize_stack( + pikepdf.parse_content_stream(contentstream, operator_whitelist) + ) + ): operands, operator = graphobj if operator == 'q': stack.append(ctm) if len(stack) > 32: # See docstring if len(stack) > 128: raise RuntimeError( - "PDF graphics stack overflowed hard limit, operator %i" % n) + "PDF graphics stack overflowed hard limit, operator %i" % n + ) warn("PDF graphics stack overflowed spec limit") elif operator == 'Q': try: @@ -177,14 +180,14 @@ def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE): elif operator == 'Do': image_name = operands[0] settings = XobjectSettings( - name=image_name, shorthand=ctm.shorthand, - stack_depth=len(stack)) + name=image_name, shorthand=ctm.shorthand, stack_depth=len(stack) + ) xobject_settings.append(settings) - elif operator == 'INLINE IMAGE': # BI/ID/EI are grouped into this + elif operator == 'INLINE IMAGE': # BI/ID/EI are grouped into this iimage = operands[0] inline = InlineSettings( - iimage=iimage, shorthand=ctm.shorthand, - stack_depth=len(stack)) + iimage=iimage, shorthand=ctm.shorthand, stack_depth=len(stack) + ) inline_images.append(inline) elif operator in vector_ops: found_vector = True @@ -192,7 +195,8 @@ def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE): return ContentsInfo( xobject_settings=xobject_settings, inline_images=inline_images, - found_vector=found_vector) + found_vector=found_vector, + ) def _get_dpi(ctm_shorthand, image_size): @@ -262,8 +266,7 @@ def _get_dpi(ctm_shorthand, image_size): class ImageInfo: DPI_PREC = Decimal('1.000') - def __init__(self, *, name='', pdfimage=None, inline=None, - shorthand=None): + def __init__(self, *, name='', pdfimage=None, inline=None, shorthand=None): self._name = str(name) self._shorthand = shorthand @@ -348,19 +351,24 @@ class ImageInfo: return _get_dpi(self._shorthand, (self._width, self._height))[1] def __repr__(self): - class_locals = {attr: getattr(self, attr, None) for attr in dir(self) - if not attr.startswith('_')} + class_locals = { + attr: getattr(self, attr, None) + for attr in dir(self) + if not attr.startswith('_') + } return ( "").format(**class_locals) + "{comp} {bpc} {enc} {xres}x{yres}>" + ).format(**class_locals) def _find_inline_images(contentsinfo): "Find inline images in the contentstream" for n, inline in enumerate(contentsinfo.inline_images): - yield ImageInfo(name='inline-%02d' % n, shorthand=inline.shorthand, - inline=inline) + yield ImageInfo( + name='inline-%02d' % n, shorthand=inline.shorthand, inline=inline + ) def _image_xobjects(container): @@ -414,8 +422,7 @@ def _find_regular_images(container, contentsinfo): # these from our DPI calculation for the page. continue - yield ImageInfo(name=draw.name, pdfimage=pdfimage, shorthand= - draw.shorthand) + yield ImageInfo(name=draw.name, pdfimage=pdfimage, shorthand=draw.shorthand) def _find_form_xobject_images(pdf, container, contentsinfo): @@ -446,7 +453,8 @@ def _find_form_xobject_images(pdf, container, contentsinfo): # same object are both very rare. ctm_shorthand = settings.shorthand yield from _process_content_streams( - pdf=pdf, container=form_xobject, shorthand=ctm_shorthand) + pdf=pdf, container=form_xobject, shorthand=ctm_shorthand + ) def _process_content_streams(*, pdf, container, shorthand=None): @@ -470,8 +478,7 @@ def _process_content_streams(*, pdf, container, shorthand=None): if container.get('/Type') == '/Page' and '/Contents' in container: initial_shorthand = shorthand or UNIT_SQUARE - elif container.get('/Type') == '/XObject' and \ - container['/Subtype'] == '/Form': + elif container.get('/Type') == '/XObject' and container['/Subtype'] == '/Form': # Set the CTM to the state it was when the "Do" operator was # encountered that is drawing this instance of the Form XObject ctm = PdfMatrix(shorthand) if shorthand else PdfMatrix.identity() @@ -505,9 +512,9 @@ def _page_has_text(text_blocks, page_width, page_height): margin_ratio = 0.125 interior_bbox = ( margin_ratio * pw, # left - (1 - margin_ratio) * ph, # top - (1 - margin_ratio) * pw, # right - margin_ratio * ph # bottom (first quadrant: bottom < top) + (1 - margin_ratio) * ph, # top + (1 - margin_ratio) * pw, # right + margin_ratio * ph, # bottom (first quadrant: bottom < top) ) def rects_intersect(a, b): @@ -535,8 +542,8 @@ def simplify_textboxes(miner): first_line = box._objs[0] first_char = first_line._objs[0] - visible = (first_char.rendermode != 3) - corrupt = (first_char.get_text() == '\ufffd') + visible = first_char.rendermode != 3 + corrupt = first_char.get_text() == '\ufffd' yield TextboxInfo(box.bbox, visible, corrupt) @@ -552,7 +559,8 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext): if xmltext is not None: bboxes = ghosttext.page_get_textblocks( - fspath(infile), pageno, xmltext=xmltext, height=height_pt) + fspath(infile), pageno, xmltext=xmltext, height=height_pt + ) pageinfo['bboxes'] = bboxes else: pscript5_mode = str(pdf.docinfo.get('/Creator')).startswith('PScript5') @@ -560,9 +568,7 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext): pageinfo['textboxes'] = list(simplify_textboxes(miner)) bboxes = (box.bbox for box in pageinfo['textboxes']) - pageinfo['has_text'] = _page_has_text( - bboxes, width_pt, height_pt - ) + pageinfo['has_text'] = _page_has_text(bboxes, width_pt, height_pt) userunit = page.get('/UserUnit', Decimal(1.0)) if not isinstance(userunit, Decimal): @@ -577,24 +583,24 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext): pageinfo['rotate'] = 0 userunit_shorthand = (userunit, 0, 0, userunit, 0, 0) - contentsinfo = [ci for ci in - _process_content_streams(pdf=pdf, container=page, - shorthand=userunit_shorthand)] + contentsinfo = [ + ci + for ci in _process_content_streams( + pdf=pdf, container=page, shorthand=userunit_shorthand + ) + ] pageinfo['has_vector'] = False if any(isinstance(ci, VectorInfo) for ci in contentsinfo): pageinfo['has_vector'] = True - pageinfo['images'] = [im for im in contentsinfo - if isinstance(im, ImageInfo)] + pageinfo['images'] = [im for im in contentsinfo if isinstance(im, ImageInfo)] if pageinfo['images']: xres = Decimal(max(image.xres for image in pageinfo['images'])) yres = Decimal(max(image.yres for image in pageinfo['images'])) pageinfo['xres'], pageinfo['yres'] = xres, yres - pageinfo['width_pixels'] = \ - int(round(xres * pageinfo['width_inches'])) - pageinfo['height_pixels'] = \ - int(round(yres * pageinfo['height_inches'])) + pageinfo['width_pixels'] = int(round(xres * pageinfo['width_inches'])) + pageinfo['height_pixels'] = int(round(yres * pageinfo['height_inches'])) return pageinfo @@ -689,13 +695,14 @@ class PageInfo: if 'textboxes' not in self._pageinfo: if visible is not None and corrupt is not None: - raise NotImplementedError( - 'Ghostscript textboxes cannot be classified') + raise NotImplementedError('Ghostscript textboxes cannot be classified') return self._pageinfo['bboxes'] - return (obj.bbox for obj in self._pageinfo['textboxes'] - if predicate(obj, visible, corrupt)) - + return ( + obj.bbox + for obj in self._pageinfo['textboxes'] + if predicate(obj, visible, corrupt) + ) @property def xres(self): @@ -718,11 +725,15 @@ class PageInfo: def __repr__(self): return ( - '').format( - self.pageno, self.width_inches, self.height_inches, + '' + ).format( + self.pageno, + self.width_inches, + self.height_inches, self.rotation, - self.xres, self.yres, self.has_text + self.xres, + self.yres, + self.has_text, ) @@ -732,7 +743,8 @@ class PdfInfo: def __init__(self, infile, detailed_page_analysis=False, log=None): self._infile = infile self._pages, pdf = _pdf_get_all_pageinfo( - infile, detailed_page_analysis, log=log) + infile, detailed_page_analysis, log=log + ) self._needs_rendering = pdf.root.get('/NeedsRendering', False) self._has_acroform = '/AcroForm' in pdf.root @@ -772,13 +784,16 @@ class PdfInfo: def __repr__(self): return "".format(len(self)) + def main(): import argparse + parser = argparse.ArgumentParser() parser.add_argument('infile') args = parser.parse_args() info = _pdf_get_all_pageinfo(args.infile) from pprint import pprint + pprint(info) diff --git a/src/ocrmypdf/pdfinfo/ghosttext.py b/src/ocrmypdf/pdfinfo/ghosttext.py index 4b9e13a0..c1a612a5 100644 --- a/src/ocrmypdf/pdfinfo/ghosttext.py +++ b/src/ocrmypdf/pdfinfo/ghosttext.py @@ -23,13 +23,16 @@ from ..exec import ghostscript # Forgive me for I have sinned # I am using regular expressions to parse XML. However the XML in this case, # generated by Ghostscript, is self-consistent enough to be parseable. -regex_remove_char_tags = re.compile(br""" +regex_remove_char_tags = re.compile( + br""" ] # anything single character but > | \">\" # special case: trap ">" )* /> # terminate with '/>' -""", re.VERBOSE) +""", + re.VERBOSE, +) def page_get_textblocks(infile, pageno, xmltext, height): @@ -76,15 +79,14 @@ def extract_text_xml(infile, pdf, pageno=None, log=None): existing_text = regex_remove_char_tags.sub(b' ', existing_text) try: - root = ET.fromstringlist([ - b'\n', existing_text, b'\n' - ]) + root = ET.fromstringlist([b'\n', existing_text, b'\n']) page_xml = root.findall('page') except ET.ParseError as e: log.error( "An error occurred while attempting to retrieve existing text in " "the input file. Will attempt to continue assuming that there is " - "no existing text in the file. The error was:") + "no existing text in the file. The error was:" + ) log.error(e) page_xml = [None] * len(pdf.pages) diff --git a/src/ocrmypdf/pdfinfo/layout.py b/src/ocrmypdf/pdfinfo/layout.py index 733c442a..d6c2d889 100644 --- a/src/ocrmypdf/pdfinfo/layout.py +++ b/src/ocrmypdf/pdfinfo/layout.py @@ -25,11 +25,23 @@ import pdfminer.pdfdevice import pdfminer.pdfinterp from pdfminer.converter import PDFLayoutAnalyzer from pdfminer.glyphlist import glyphname2unicode -from pdfminer.layout import (LAParams, LTChar, LTContainer, LTLayoutContainer, - LTPage, LTTextBox, LTTextLine) +from pdfminer.layout import ( + LAParams, + LTChar, + LTContainer, + LTLayoutContainer, + LTPage, + LTTextBox, + LTTextLine, +) from pdfminer.pdfdocument import PDFTextExtractionNotAllowed -from pdfminer.pdffont import (PDFCIDFont, PDFFont, PDFType3Font, - PDFUnicodeNotDefined, PDFSimpleFont) +from pdfminer.pdffont import ( + PDFCIDFont, + PDFFont, + PDFType3Font, + PDFUnicodeNotDefined, + PDFSimpleFont, +) from pdfminer.pdfpage import PDFPage from pdfminer.utils import bbox2str, fsplit, matrix2str @@ -41,6 +53,7 @@ STRIP_NAME = re.compile(r'[0-9]+') # Unconditional pdfminer patches # + def name2unicode(name): """Fix pdfminer's name2unicode function @@ -62,9 +75,13 @@ def name2unicode(name): if not m: raise KeyError(name) return chr(int(m.group(0))) + + pdfminer.encodingdb.name2unicode = name2unicode original_PDFFont_init = PDFFont.__init__ + + def PDFFont__init__(self, descriptor, widths, default_width=None): original_PDFFont_init(self, descriptor, widths, default_width) # PDF spec says descent should be negative @@ -74,9 +91,13 @@ def PDFFont__init__(self, descriptor, widths, default_width=None): # to misposition text. if self.descent > 0: self.descent = -self.descent + + PDFFont.__init__ = PDFFont__init__ original_PDFSimpleFont_init = PDFSimpleFont.__init__ + + def PDFSimpleFont__init__(self, descriptor, widths, spec): # Font encoding is specified either by a name of # built-in encoding or a dictionary that describes @@ -87,20 +108,25 @@ def PDFSimpleFont__init__(self, descriptor, widths, spec): if not self.unicode_map and 'Encoding' not in spec: self.cid2unicode = {} return + + PDFSimpleFont.__init__ = PDFSimpleFont__init__ # # pdfminer patches when creator is PScript5.dll # + def PDFType3Font__PScript5_get_height(self): - h = self.bbox[3]-self.bbox[1] + h = self.bbox[3] - self.bbox[1] if h == 0: h = self.ascent - self.descent return h * copysign(1.0, self.vscale) + def PDFType3Font__PScript5_get_descent(self): return self.descent * copysign(1.0, self.vscale) + def PDFType3Font__PScript5_get_ascent(self): return self.ascent * copysign(1.0, self.vscale) @@ -109,14 +135,38 @@ class LTStateAwareChar(LTChar): """A subclass of LTChar that tracks text render mode at time of drawing""" __slots__ = ( - 'rendermode', '_text', 'matrix', 'fontname', 'adv', 'upright', 'size', - 'width', 'height', 'bbox', 'x0', 'x1', 'y0', 'y1' + 'rendermode', + '_text', + 'matrix', + 'fontname', + 'adv', + 'upright', + 'size', + 'width', + 'height', + 'bbox', + 'x0', + 'x1', + 'y0', + 'y1', ) - def __init__(self, matrix, font, fontsize, scaling, rise, text, textwidth, - textdisp, textstate, *args): - super().__init__(matrix, font, fontsize, scaling, rise, text, textwidth, - textdisp, *args) + def __init__( + self, + matrix, + font, + fontsize, + scaling, + rise, + text, + textwidth, + textdisp, + textstate, + *args, + ): + super().__init__( + matrix, font, fontsize, scaling, rise, text, textwidth, textdisp, *args + ) self.rendermode = textstate.render def is_compatible(self, obj): @@ -126,8 +176,7 @@ class LTStateAwareChar(LTChar): - the Unicode mapping is known, and both have the same render mode - the Unicode mapping is unknown but both are part of the same font """ - both_unicode_mapped = (isinstance(self._text, str) and - isinstance(obj._text, str)) + both_unicode_mapped = isinstance(self._text, str) and isinstance(obj._text, str) try: if both_unicode_mapped: return self.rendermode == obj.rendermode @@ -143,10 +192,15 @@ class LTStateAwareChar(LTChar): return self._text def __repr__(self): - return ('<%s %s matrix=%s rendermode=%r font=%r adv=%s text=%r>' % - (self.__class__.__name__, bbox2str(self.bbox), - matrix2str(self.matrix), self.rendermode, self.fontname, self.adv, - self.get_text())) + return '<%s %s matrix=%s rendermode=%r font=%r adv=%s text=%r>' % ( + self.__class__.__name__, + bbox2str(self.bbox), + matrix2str(self.matrix), + self.rendermode, + self.fontname, + self.adv, + self.get_text(), + ) class TextPositionTracker(PDFLayoutAnalyzer): @@ -182,13 +236,22 @@ class TextPositionTracker(PDFLayoutAnalyzer): textwidth = font.char_width(cid) textdisp = font.char_disp(cid) item = LTStateAwareChar( - matrix, font, fontsize, scaling, rise, text, - textwidth, textdisp, self.textstate, *args) + matrix, + font, + fontsize, + scaling, + rise, + text, + textwidth, + textdisp, + self.textstate, + *args, + ) self.cur_item.add(item) return item.adv def handle_undefined_char(self, font, cid): - #log.info('undefined: %r, %r', font, cid) + # log.info('undefined: %r, %r', font, cid) return (font.fontname, cid) def receive_layout(self, ltpage): @@ -209,7 +272,7 @@ def get_page_analysis(infile, pageno, pscript5_mode): spec=True, get_ascent=PDFType3Font__PScript5_get_ascent, get_descent=PDFType3Font__PScript5_get_descent, - get_height=PDFType3Font__PScript5_get_height + get_height=PDFType3Font__PScript5_get_height, ) patcher.start() diff --git a/tests/conftest.py b/tests/conftest.py index 6bcbd4e1..60793aa5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -144,27 +144,33 @@ def check_ocrmypdf(input_file, output_file, *args, env=None): assert p.returncode == 0 assert os.path.exists(str(output_file)), "Output file not created" assert os.stat(str(output_file)).st_size > 100, "PDF too small or empty" - assert out == "", \ - "The following was written to stdout and should not have been: \n" + \ - "\n" + out + "\n" + assert out == "", ( + "The following was written to stdout and should not have been: \n" + + "\n" + + out + + "\n" + ) return output_file @pytest.helpers.register -def run_ocrmypdf(input_file, output_file, *args, env=None, - universal_newlines=True): +def run_ocrmypdf(input_file, output_file, *args, env=None, universal_newlines=True): "Run ocrmypdf and let caller deal with results" if env is None: env = os.environ - p_args = OCRMYPDF + [str(arg) for arg in args] + \ - [str(input_file), str(output_file)] + p_args = OCRMYPDF + [str(arg) for arg in args] + [str(input_file), str(output_file)] p = Popen( - p_args, close_fds=True, stdout=PIPE, stderr=PIPE, - universal_newlines=universal_newlines, env=env) + p_args, + close_fds=True, + stdout=PIPE, + stderr=PIPE, + universal_newlines=universal_newlines, + env=env, + ) out, err = p.communicate() - #print(err) + # print(err) return p, out, err @@ -172,6 +178,7 @@ def run_ocrmypdf(input_file, output_file, *args, env=None, @pytest.helpers.register def first_page_dimensions(pdf): from ocrmypdf import pdfinfo + info = pdfinfo.PdfInfo(pdf) page0 = info[0] return (page0.width_inches, page0.height_inches) @@ -179,9 +186,13 @@ def first_page_dimensions(pdf): def pytest_addoption(parser): parser.addoption( - "--runslow", action="store_true", default=False, - help=("run slow tests only useful for development (unlikely to be " - "useful for downstream packagers)") + "--runslow", + action="store_true", + default=False, + help=( + "run slow tests only useful for development (unlikely to be " + "useful for downstream packagers)" + ), ) diff --git a/tests/spoof/gs_feature_elision.py b/tests/spoof/gs_feature_elision.py index da0187ba..91246cd8 100755 --- a/tests/spoof/gs_feature_elision.py +++ b/tests/spoof/gs_feature_elision.py @@ -53,5 +53,6 @@ def main(): sys.exit(0) + if __name__ == '__main__': main() diff --git a/tests/spoof/gs_render_failure.py b/tests/spoof/gs_render_failure.py index 89bcddff..9bafc280 100755 --- a/tests/spoof/gs_render_failure.py +++ b/tests/spoof/gs_render_failure.py @@ -26,7 +26,6 @@ import sys import os - def real_ghostscript(argv): gs_args = ['gs'] + argv[1:] os.execvp("gs", gs_args) diff --git a/tests/spoof/tesseract_badutf8.py b/tests/spoof/tesseract_badutf8.py index 9a97fc46..7981070b 100755 --- a/tests/spoof/tesseract_badutf8.py +++ b/tests/spoof/tesseract_badutf8.py @@ -63,11 +63,14 @@ def main(): sys.exit(1) elif sys.argv[-1] == 'stdout': # input file is at sys.argv[-2] but we don't look at it - print("""Orientation: 0 + print( + """Orientation: 0 Orientation in degrees: 0 Orientation confidence: 100.00 Script: 1 -Script confidence: 100.00""", file=sys.stderr) +Script confidence: 100.00""", + file=sys.stderr, + ) else: print("Spoof doesn't understand arguments", file=sys.stderr) print(sys.argv, file=sys.stderr) diff --git a/tests/spoof/tesseract_big_image_error.py b/tests/spoof/tesseract_big_image_error.py index af6d6c28..44e8b74a 100755 --- a/tests/spoof/tesseract_big_image_error.py +++ b/tests/spoof/tesseract_big_image_error.py @@ -49,16 +49,22 @@ def main(): print('A parameter list would go here\ntextonly_pdf 0\n', file=sys.stderr) sys.exit(0) elif sys.argv[-2] == 'hocr': - print("Image too large: (33830, 14959)\n" - "Error during processing.", file=sys.stderr) + print( + "Image too large: (33830, 14959)\n" "Error during processing.", + file=sys.stderr, + ) sys.exit(1) elif sys.argv[-2] == 'pdf': - print("Image too large: (33830, 14959)\n" - "Error during processing.", file=sys.stderr) + print( + "Image too large: (33830, 14959)\n" "Error during processing.", + file=sys.stderr, + ) sys.exit(1) elif sys.argv[-1] == 'stdout': - print("Image too large: (33830, 14959)\n" - "Error during processing.", file=sys.stderr) + print( + "Image too large: (33830, 14959)\n" "Error during processing.", + file=sys.stderr, + ) sys.exit(1) else: print("Spoof doesn't understand arguments", file=sys.stderr) diff --git a/tests/spoof/tesseract_cache.py b/tests/spoof/tesseract_cache.py index 3a7b3b11..4e2c7ac0 100755 --- a/tests/spoof/tesseract_cache.py +++ b/tests/spoof/tesseract_cache.py @@ -63,12 +63,13 @@ if '_OCRMYPDF_SAVE_PATH' in os.environ: os.environ['PATH'] = os.environ['_OCRMYPDF_SAVE_PATH'] __version__ = subprocess.check_output( - ['tesseract', '--version'], - stderr=subprocess.STDOUT).decode() + ['tesseract', '--version'], stderr=subprocess.STDOUT +).decode() parser = argparse.ArgumentParser( - prog='tesseract-cache', description='cache output of tesseract') + prog='tesseract-cache', description='cache output of tesseract' +) parser.add_argument('-l', '--language', action='append') parser.add_argument('imagename') parser.add_argument('outputbase') @@ -82,6 +83,7 @@ parser.add_argument('--oem', type=int) TESTS_ROOT = Path(__file__).resolve().parent.parent CACHE_ROOT = TESTS_ROOT / 'cache' + def real_tesseract(): tess_args = ['tesseract'] + sys.argv[1:] os.execvp("tesseract", tess_args) @@ -89,8 +91,10 @@ def real_tesseract(): def main(): - if any(opt in sys.argv[1:] for opt in ( - '--print-parameters', '--list-langs', '--version')): + if any( + opt in sys.argv[1:] + for opt in ('--print-parameters', '--list-langs', '--version') + ): real_tesseract() # jump into real tesseract, replacing this process # Convert non-standard but supported -psm to --psm @@ -122,8 +126,7 @@ def main(): cache_folder = Path(CACHE_ROOT) / Path(source).stem / argv_slug cache_folder.mkdir(parents=True, exist_ok=True) - print("Tesseract cache folder {} - ".format(cache_folder), end='', - file=sys.stderr) + print("Tesseract cache folder {} - ".format(cache_folder), end='', file=sys.stderr) if (cache_folder / 'stderr.bin').exists() and not cache_disabled: # Cache hit @@ -138,8 +141,7 @@ def main(): for configfile in args.configfiles: # cp cache -> output tessfile = args.outputbase + '.' + configfile - shutil.copy(str(cache_folder / configfile) + '.bin', - tessfile) + shutil.copy(str(cache_folder / configfile) + '.bin', tessfile) sys.exit(0) # Cache miss @@ -148,8 +150,8 @@ def main(): # Call tesseract print(sys.argv[1:]) p = subprocess.run( - ['tesseract'] + sys.argv[1:], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ['tesseract'] + sys.argv[1:], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) sys.stdout.buffer.write(p.stdout) sys.stderr.buffer.write(p.stderr) @@ -179,10 +181,11 @@ def main(): manifest['python'] = platform.python_version() manifest['argv_slug'] = argv_slug manifest['sourcefile'] = str(Path(source).relative_to(TESTS_ROOT)) + def clean_sys_argv(): for arg in sys.argv[1:]: - yield re.sub(r'.*/com.github.ocrmypdf[^/]+[/](.*)', - r'$TMPDIR/\1', arg) + yield re.sub(r'.*/com.github.ocrmypdf[^/]+[/](.*)', r'$TMPDIR/\1', arg) + manifest['args'] = list(clean_sys_argv()) # pylint: disable=E1101 diff --git a/tests/spoof/tesseract_crash.py b/tests/spoof/tesseract_crash.py index de385783..399da380 100755 --- a/tests/spoof/tesseract_crash.py +++ b/tests/spoof/tesseract_crash.py @@ -53,8 +53,7 @@ def main(): print('List of available languages (1):\neng', file=sys.stderr) sys.exit(0) elif sys.argv[1] == '--print-parameters': - print('A parameter list would go here\ntextonly_pdf 0\n', - file=sys.stderr) + print('A parameter list would go here\ntextonly_pdf 0\n', file=sys.stderr) sys.exit(0) elif sys.argv[-2] == 'hocr': print("KABOOM! Tesseract failed for some reason", file=sys.stderr) @@ -63,8 +62,11 @@ def main(): print("KABOOM! Tesseract failed for some reason", file=sys.stderr) sys.exit(128 + signal.SIGSEGV) elif sys.argv[-1] == 'stdout': - print("libc++abi.dylib: terminating with uncaught exception of type " - "std::bad_alloc: std::bad_alloc", file=sys.stderr) + print( + "libc++abi.dylib: terminating with uncaught exception of type " + "std::bad_alloc: std::bad_alloc", + file=sys.stderr, + ) sys.exit(128 + signal.SIGABRT) else: print("Spoof doesn't understand arguments", file=sys.stderr) diff --git a/tests/spoof/tesseract_noop.py b/tests/spoof/tesseract_noop.py index 7fa554a0..c8ed485e 100755 --- a/tests/spoof/tesseract_noop.py +++ b/tests/spoof/tesseract_noop.py @@ -83,8 +83,9 @@ def main(): elif sys.argv[-2] == 'hocr': inputf = sys.argv[-4] output = sys.argv[-3] - with Image.open(inputf) as im, \ - open(output + '.hocr', 'w', encoding='utf-8') as f: + with Image.open(inputf) as im, open( + output + '.hocr', 'w', encoding='utf-8' + ) as f: w, h = im.size f.write(HOCR_TEMPLATE.format(str(w), str(h))) with open(output + '.txt', 'w') as f: @@ -114,11 +115,14 @@ def main(): f.write('') elif sys.argv[-1] == 'stdout': inputf = sys.argv[-2] - print("""Orientation: 0 + print( + """Orientation: 0 Orientation in degrees: 0 Orientation confidence: 100.00 Script: 1 -Script confidence: 100.00""", file=sys.stderr) +Script confidence: 100.00""", + file=sys.stderr, + ) else: print("Spoof doesn't understand arguments", file=sys.stderr) print(sys.argv, file=sys.stderr) diff --git a/tests/spoof/unpaper_oldversion.py b/tests/spoof/unpaper_oldversion.py index 6572a0d3..ff2e27ea 100755 --- a/tests/spoof/unpaper_oldversion.py +++ b/tests/spoof/unpaper_oldversion.py @@ -23,6 +23,7 @@ import sys + def main(): if sys.argv[1] == '--version': print('0.5') diff --git a/tests/test_hocrtransform.py b/tests/test_hocrtransform.py index e40d3867..54bdc35c 100644 --- a/tests/test_hocrtransform.py +++ b/tests/test_hocrtransform.py @@ -39,7 +39,6 @@ def test_mono_image(blank_hocr, outdir): im.save(outdir / 'mono.tif', format='TIFF') hocr = hocrtransform.HocrTransform(str(blank_hocr), 300) - hocr.to_pdf( - str(outdir / 'mono.pdf'), imageFileName=str(outdir / 'mono.tif')) + hocr.to_pdf(str(outdir / 'mono.pdf'), imageFileName=str(outdir / 'mono.tif')) qpdf.check(str(outdir / 'mono.pdf')) diff --git a/tests/test_lept.py b/tests/test_lept.py index acabfdc4..1f643625 100644 --- a/tests/test_lept.py +++ b/tests/test_lept.py @@ -87,5 +87,4 @@ def test_leptonica_compile(tmpdir): # Compile the library but build it somewhere that won't interfere with # existing compiled library. Also compile in API mode so that we test # the interfaces, even though we use it ABI mode. - ffibuilder.compile(tmpdir=fspath(tmpdir), - target=fspath(tmpdir / 'lepttest.*')) + ffibuilder.compile(tmpdir=fspath(tmpdir), target=fspath(tmpdir / 'lepttest.*')) diff --git a/tests/test_main.py b/tests/test_main.py index b74c03aa..2372e664 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -61,17 +61,23 @@ def spoof_no_tess_no_pdfa(tmpdir_factory): @pytest.fixture(scope='session') def spoof_no_tess_pdfa_warning(tmpdir_factory): - return spoof(tmpdir_factory, tesseract='tesseract_noop.py', gs='gs_feature_elision.py') + return spoof( + tmpdir_factory, tesseract='tesseract_noop.py', gs='gs_feature_elision.py' + ) @pytest.fixture(scope='session') def spoof_no_tess_gs_render_fail(tmpdir_factory): - return spoof(tmpdir_factory, tesseract='tesseract_noop.py', gs='gs_render_failure.py') + return spoof( + tmpdir_factory, tesseract='tesseract_noop.py', gs='gs_render_failure.py' + ) @pytest.fixture(scope='session') def spoof_no_tess_gs_raster_fail(tmpdir_factory): - return spoof(tmpdir_factory, tesseract='tesseract_noop.py', gs='gs_raster_failure.py') + return spoof( + tmpdir_factory, tesseract='tesseract_noop.py', gs='gs_raster_failure.py' + ) @pytest.fixture(scope='session') @@ -86,8 +92,8 @@ def test_quick(spoof_tesseract_cache, resources, outpdf): def test_deskew(spoof_tesseract_noop, resources, outdir): # Run with deskew deskewed_pdf = check_ocrmypdf( - resources / 'skew.pdf', outdir / 'skew.pdf', '-d', - env=spoof_tesseract_noop) + resources / 'skew.pdf', outdir / 'skew.pdf', '-d', env=spoof_tesseract_noop + ) # Now render as an image again and use Leptonica to find the skew angle # to confirm that it was deskewed @@ -102,7 +108,8 @@ def test_deskew(spoof_tesseract_noop, resources, outdir): yres=150, raster_device='pngmono', log=log, - pageno=1) + pageno=1, + ) pix = Pix.open(deskewed_png) skew_angle, skew_confidence = pix.find_skew() @@ -120,8 +127,10 @@ def test_remove_background(spoof_tesseract_noop, resources, outdir): resources / 'congress.jpg', outdir / 'test_remove_bg.pdf', '--remove-background', - '--image-dpi', '150', - env=spoof_tesseract_noop) + '--image-dpi', + '150', + env=spoof_tesseract_noop, + ) log = logging.getLogger() @@ -134,7 +143,8 @@ def test_remove_background(spoof_tesseract_noop, resources, outdir): yres=100, raster_device='png16m', log=log, - pageno=1) + pageno=1, + ) # The output image should contain pure white and black im = Image.open(output_png) @@ -143,22 +153,28 @@ def test_remove_background(spoof_tesseract_noop, resources, outdir): # This will run 5 * 2 * 2 = 20 test cases @pytest.mark.parametrize( - "pdf", - ['palette.pdf', 'cmyk.pdf', 'ccitt.pdf', 'jbig2.pdf', 'lichtenstein.pdf']) + "pdf", ['palette.pdf', 'cmyk.pdf', 'ccitt.pdf', 'jbig2.pdf', 'lichtenstein.pdf'] +) @pytest.mark.parametrize("renderer", ['sandwich', 'hocr']) @pytest.mark.parametrize("output_type", ['pdf', 'pdfa']) -def test_exotic_image(spoof_tesseract_cache, pdf, renderer, output_type, - resources, outdir): +def test_exotic_image( + spoof_tesseract_cache, pdf, renderer, output_type, resources, outdir +): outfile = outdir / 'test_{0}_{1}.pdf'.format(pdf, renderer) check_ocrmypdf( resources / pdf, outfile, '-dc', - '-v', '1', - '--output-type', output_type, + '-v', + '1', + '--output-type', + output_type, '--sidecar', '--skip-text', - '--pdf-renderer', renderer, env=spoof_tesseract_cache) + '--pdf-renderer', + renderer, + env=spoof_tesseract_cache, + ) assert outfile.with_suffix('.pdf.txt').exists() @@ -166,9 +182,15 @@ def test_exotic_image(spoof_tesseract_cache, pdf, renderer, output_type, @pytest.mark.parametrize('renderer', RENDERERS) def test_oversample(spoof_tesseract_cache, renderer, resources, outpdf): oversampled_pdf = check_ocrmypdf( - resources / 'skew.pdf', outpdf, '--oversample', '350', + resources / 'skew.pdf', + outpdf, + '--oversample', + '350', '-f', - '--pdf-renderer', renderer, env=spoof_tesseract_cache) + '--pdf-renderer', + renderer, + env=spoof_tesseract_cache, + ) pdfinfo = PdfInfo(oversampled_pdf) @@ -182,15 +204,17 @@ def test_repeat_ocr(resources, no_outpdf): def test_force_ocr(spoof_tesseract_cache, resources, outpdf): - out = check_ocrmypdf(resources / 'graph_ocred.pdf', outpdf, '-f', - env=spoof_tesseract_cache) + out = check_ocrmypdf( + resources / 'graph_ocred.pdf', outpdf, '-f', env=spoof_tesseract_cache + ) pdfinfo = PdfInfo(out) assert pdfinfo[0].has_text def test_skip_ocr(spoof_tesseract_cache, resources, outpdf): - out = check_ocrmypdf(resources / 'graph_ocred.pdf', outpdf, '-s', - env=spoof_tesseract_cache) + out = check_ocrmypdf( + resources / 'graph_ocred.pdf', outpdf, '-s', env=spoof_tesseract_cache + ) pdfinfo = PdfInfo(out) assert pdfinfo[0].has_text @@ -198,52 +222,84 @@ def test_skip_ocr(spoof_tesseract_cache, resources, outpdf): def test_redo_ocr(spoof_tesseract_cache, resources, outpdf): in_ = resources / 'graph_ocred.pdf' before = PdfInfo(in_, detailed_page_analysis=True) - out = check_ocrmypdf(in_, outpdf, '--redo-ocr', - env=spoof_tesseract_cache) + out = check_ocrmypdf(in_, outpdf, '--redo-ocr', env=spoof_tesseract_cache) after = PdfInfo(out, detailed_page_analysis=True) assert before[0].has_text and after[0].has_text - assert before[0].get_textareas() != after[0].get_textareas(), \ - "Expected text to be different after re-OCR" + assert ( + before[0].get_textareas() != after[0].get_textareas() + ), "Expected text to be different after re-OCR" def test_argsfile(spoof_tesseract_noop, resources, outdir): path_argsfile = outdir / 'test_argsfile.txt' with open(str(path_argsfile), 'w') as argsfile: - print('--title', 'ArgsFile Test', '--author', 'Test Cases', - sep='\n', end='\n', file=argsfile) - check_ocrmypdf(resources / 'graph.pdf', path_argsfile, - '@' + str(outdir / 'test_argsfile.txt'), - env=spoof_tesseract_noop) + print( + '--title', + 'ArgsFile Test', + '--author', + 'Test Cases', + sep='\n', + end='\n', + file=argsfile, + ) + check_ocrmypdf( + resources / 'graph.pdf', + path_argsfile, + '@' + str(outdir / 'test_argsfile.txt'), + env=spoof_tesseract_noop, + ) @pytest.mark.parametrize('renderer', RENDERERS) def test_ocr_timeout(renderer, resources, outpdf): - out = check_ocrmypdf(resources / 'skew.pdf', outpdf, - '--tesseract-timeout', '0', - '--pdf-renderer', renderer) + out = check_ocrmypdf( + resources / 'skew.pdf', + outpdf, + '--tesseract-timeout', + '0', + '--pdf-renderer', + renderer, + ) pdfinfo = PdfInfo(out) assert not pdfinfo[0].has_text def test_skip_big(spoof_tesseract_cache, resources, outpdf): - out = check_ocrmypdf(resources / 'jbig2.pdf', outpdf, - '--skip-big', '1', env=spoof_tesseract_cache) + out = check_ocrmypdf( + resources / 'jbig2.pdf', outpdf, '--skip-big', '1', env=spoof_tesseract_cache + ) pdfinfo = PdfInfo(out) assert not pdfinfo[0].has_text @pytest.mark.parametrize('renderer', RENDERERS) @pytest.mark.parametrize('output_type', ['pdf', 'pdfa']) -def test_maximum_options(spoof_tesseract_cache, renderer, output_type, - resources, outpdf): +def test_maximum_options( + spoof_tesseract_cache, renderer, output_type, resources, outpdf +): check_ocrmypdf( - resources / 'multipage.pdf', outpdf, - '-d', '-c', '-i', '-f', '-k', '--oversample', '300', + resources / 'multipage.pdf', + outpdf, + '-d', + '-c', + '-i', + '-f', + '-k', + '--oversample', + '300', '--remove-background', - '--skip-big', '10', '--title', 'Too Many Weird Files', - '--author', 'py.test', '--pdf-renderer', renderer, - '--output-type', output_type, - env=spoof_tesseract_cache) + '--skip-big', + '10', + '--title', + 'Too Many Weird Files', + '--author', + 'py.test', + '--pdf-renderer', + renderer, + '--output-type', + output_type, + env=spoof_tesseract_cache, + ) def test_tesseract_missing_tessdata(resources, no_outpdf): @@ -251,94 +307,96 @@ def test_tesseract_missing_tessdata(resources, no_outpdf): env['TESSDATA_PREFIX'] = '/tmp' p, _, err = run_ocrmypdf( - resources / 'graph_ocred.pdf', no_outpdf, - '-v', '1', '--skip-text', env=env) + resources / 'graph_ocred.pdf', no_outpdf, '-v', '1', '--skip-text', env=env + ) assert p.returncode == ExitCode.missing_dependency, err def test_invalid_input_pdf(resources, no_outpdf): - p, out, err = run_ocrmypdf( - resources / 'invalid.pdf', no_outpdf) + p, out, err = run_ocrmypdf(resources / 'invalid.pdf', no_outpdf) assert p.returncode == ExitCode.input_file, err def test_blank_input_pdf(resources, outpdf): - p, out, err = run_ocrmypdf( - resources / 'blank.pdf', outpdf) + p, out, err = run_ocrmypdf(resources / 'blank.pdf', outpdf) assert p.returncode == ExitCode.ok -def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources, - no_outpdf): +def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources, no_outpdf): # As a correctness test, make sure that --force-ocr on a PDF with no # content still triggers tesseract. If tesseract crashes, then it was # called. p, _, err = run_ocrmypdf( - resources / 'blank.pdf', no_outpdf, '--force-ocr', - env=spoof_tesseract_crash) + resources / 'blank.pdf', no_outpdf, '--force-ocr', env=spoof_tesseract_crash + ) assert p.returncode == ExitCode.child_process_error, err assert not os.path.exists(no_outpdf) @pytest.mark.skipif( pytest.helpers.is_macos() and pytest.helpers.running_in_travis(), - reason="takes too long to install language packs in Travis macOS homebrew") + reason="takes too long to install language packs in Travis macOS homebrew", +) def test_french(spoof_tesseract_cache, resources, outdir): # Produce a sidecar too - implicit test that system locale is set up # properly sidecar = outdir / 'francais.txt' p, out, err = run_ocrmypdf( - resources / 'francais.pdf', outdir / 'francais.pdf', '-l', 'fra', - '--sidecar', sidecar, - env=spoof_tesseract_cache) + resources / 'francais.pdf', + outdir / 'francais.pdf', + '-l', + 'fra', + '--sidecar', + sidecar, + env=spoof_tesseract_cache, + ) print(os.environ) - assert p.returncode == ExitCode.ok, \ - "This test may fail if Tesseract language packs are missing" + assert ( + p.returncode == ExitCode.ok + ), "This test may fail if Tesseract language packs are missing" def test_klingon(resources, outpdf): - p, out, err = run_ocrmypdf( - resources / 'francais.pdf', outpdf, '-l', 'klz') + p, out, err = run_ocrmypdf(resources / 'francais.pdf', outpdf, '-l', 'klz') assert p.returncode == ExitCode.missing_dependency def test_missing_docinfo(spoof_tesseract_noop, resources, outpdf): p, out, err = run_ocrmypdf( - resources / 'missing_docinfo.pdf', outpdf, '-l', 'eng', '--skip-text', - env=spoof_tesseract_noop) + resources / 'missing_docinfo.pdf', + outpdf, + '-l', + 'eng', + '--skip-text', + env=spoof_tesseract_noop, + ) assert p.returncode == ExitCode.ok, err def test_uppercase_extension(spoof_tesseract_noop, resources, outdir): - shutil.copy( - str(resources / "skew.pdf"), - str(outdir / "UPPERCASE.PDF")) + shutil.copy(str(resources / "skew.pdf"), str(outdir / "UPPERCASE.PDF")) - check_ocrmypdf(outdir / "UPPERCASE.PDF", outdir / "UPPERCASE_OUT.PDF", - env=spoof_tesseract_noop) + check_ocrmypdf( + outdir / "UPPERCASE.PDF", outdir / "UPPERCASE_OUT.PDF", env=spoof_tesseract_noop + ) def test_input_file_not_found(no_outpdf): input_file = "does not exist.pdf" - p, out, err = run_ocrmypdf( - input_file, - no_outpdf) + p, out, err = run_ocrmypdf(input_file, no_outpdf) assert p.returncode == ExitCode.input_file - assert (input_file in out or input_file in err) + assert input_file in out or input_file in err def test_input_file_not_a_pdf(no_outpdf): input_file = __file__ # Try to OCR this file - p, out, err = run_ocrmypdf( - input_file, - no_outpdf) + p, out, err = run_ocrmypdf(input_file, no_outpdf) assert p.returncode == ExitCode.input_file - assert (input_file in out or input_file in err) + assert input_file in out or input_file in err def test_encrypted(resources, no_outpdf): - p, out, err = run_ocrmypdf( - resources / 'skew-encrypted.pdf', no_outpdf) + p, out, err = run_ocrmypdf(resources / 'skew-encrypted.pdf', no_outpdf) assert p.returncode == ExitCode.encrypted_pdf assert out.find('encrypted') @@ -346,28 +404,38 @@ def test_encrypted(resources, no_outpdf): @pytest.mark.parametrize('renderer', RENDERERS) def test_pagesegmode(renderer, spoof_tesseract_cache, resources, outpdf): check_ocrmypdf( - resources / 'skew.pdf', outpdf, - '--tesseract-pagesegmode', '7', - '-v', '1', - '--pdf-renderer', renderer, env=spoof_tesseract_cache) + resources / 'skew.pdf', + outpdf, + '--tesseract-pagesegmode', + '7', + '-v', + '1', + '--pdf-renderer', + renderer, + env=spoof_tesseract_cache, + ) @pytest.mark.parametrize('renderer', RENDERERS) -def test_tesseract_crash(renderer, spoof_tesseract_crash, - resources, no_outpdf): +def test_tesseract_crash(renderer, spoof_tesseract_crash, resources, no_outpdf): p, out, err = run_ocrmypdf( - resources / 'ccitt.pdf', no_outpdf, '-v', '1', - '--pdf-renderer', renderer, env=spoof_tesseract_crash) + resources / 'ccitt.pdf', + no_outpdf, + '-v', + '1', + '--pdf-renderer', + renderer, + env=spoof_tesseract_crash, + ) assert p.returncode == ExitCode.child_process_error assert not os.path.exists(no_outpdf) assert "ERROR" in err -def test_tesseract_crash_autorotate(spoof_tesseract_crash, - resources, no_outpdf): +def test_tesseract_crash_autorotate(spoof_tesseract_crash, resources, no_outpdf): p, out, err = run_ocrmypdf( - resources / 'ccitt.pdf', no_outpdf, - '-r', env=spoof_tesseract_crash) + resources / 'ccitt.pdf', no_outpdf, '-r', env=spoof_tesseract_crash + ) assert p.returncode == ExitCode.child_process_error assert not os.path.exists(no_outpdf) assert "ERROR" in err @@ -376,31 +444,41 @@ def test_tesseract_crash_autorotate(spoof_tesseract_crash, @pytest.mark.parametrize('renderer', RENDERERS) -def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error, - resources, outpdf): +def test_tesseract_image_too_big( + renderer, spoof_tesseract_big_image_error, resources, outpdf +): check_ocrmypdf( - resources / 'hugemono.pdf', outpdf, '-r', - '--pdf-renderer', renderer, - '--max-image-mpixels', '0', - env=spoof_tesseract_big_image_error) + resources / 'hugemono.pdf', + outpdf, + '-r', + '--pdf-renderer', + renderer, + '--max-image-mpixels', + '0', + env=spoof_tesseract_big_image_error, + ) def test_algo4(resources, spoof_tesseract_noop, outpdf): - p, _, _ = run_ocrmypdf(resources / 'encrypted_algo4.pdf', outpdf, - env=spoof_tesseract_noop) + p, _, _ = run_ocrmypdf( + resources / 'encrypted_algo4.pdf', outpdf, env=spoof_tesseract_noop + ) assert p.returncode == ExitCode.encrypted_pdf @pytest.mark.parametrize('renderer', RENDERERS) -def test_non_square_resolution(renderer, spoof_tesseract_cache, - resources, outpdf): +def test_non_square_resolution(renderer, spoof_tesseract_cache, resources, outpdf): # Confirm input image is non-square resolution in_pageinfo = PdfInfo(resources / 'aspect.pdf') assert in_pageinfo[0].xres != in_pageinfo[0].yres check_ocrmypdf( - resources / 'aspect.pdf', outpdf, - '--pdf-renderer', renderer, env=spoof_tesseract_cache) + resources / 'aspect.pdf', + outpdf, + '--pdf-renderer', + renderer, + env=spoof_tesseract_cache, + ) out_pageinfo = PdfInfo(outpdf) @@ -410,17 +488,22 @@ def test_non_square_resolution(renderer, spoof_tesseract_cache, @pytest.mark.parametrize('renderer', RENDERERS) -def test_convert_to_square_resolution(renderer, spoof_tesseract_cache, - resources, outpdf): +def test_convert_to_square_resolution( + renderer, spoof_tesseract_cache, resources, outpdf +): # Confirm input image is non-square resolution in_pageinfo = PdfInfo(resources / 'aspect.pdf') assert in_pageinfo[0].xres != in_pageinfo[0].yres # --force-ocr requires means forced conversion to square resolution check_ocrmypdf( - resources / 'aspect.pdf', outpdf, + resources / 'aspect.pdf', + outpdf, '--force-ocr', - '--pdf-renderer', renderer, env=spoof_tesseract_cache) + '--pdf-renderer', + renderer, + env=spoof_tesseract_cache, + ) out_pageinfo = PdfInfo(outpdf) @@ -430,10 +513,8 @@ def test_convert_to_square_resolution(renderer, spoof_tesseract_cache, assert out_p0.xres == out_p0.yres # Page size should match input page size - assert isclose(in_p0.width_inches, - out_p0.width_inches) - assert isclose(in_p0.height_inches, - out_p0.height_inches) + assert isclose(in_p0.width_inches, out_p0.width_inches) + assert isclose(in_p0.height_inches, out_p0.height_inches) # Because we rasterized the page to produce a new image, it should occupy # the entire page @@ -445,16 +526,20 @@ def test_convert_to_square_resolution(renderer, spoof_tesseract_cache, def test_image_to_pdf(spoof_tesseract_noop, resources, outpdf): check_ocrmypdf( - resources / 'crom.png', outpdf, '--image-dpi', '200', - env=spoof_tesseract_noop) + resources / 'crom.png', outpdf, '--image-dpi', '200', env=spoof_tesseract_noop + ) def test_jbig2_passthrough(spoof_tesseract_cache, resources, outpdf): out = check_ocrmypdf( - resources / 'jbig2.pdf', outpdf, - '--output-type', 'pdf', - '--pdf-renderer', 'hocr', - env=spoof_tesseract_cache) + resources / 'jbig2.pdf', + outpdf, + '--output-type', + 'pdf', + '--pdf-renderer', + 'hocr', + env=spoof_tesseract_cache, + ) out_pageinfo = PdfInfo(out) assert out_pageinfo[0].images[0].enc == Encoding.jbig2 @@ -468,8 +553,13 @@ def test_stdin(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf): with open(input_file, 'rb') as input_stream: p_args = ocrmypdf_exec + ['-', output_file] p = Popen( - p_args, close_fds=True, stdout=PIPE, stderr=PIPE, - stdin=input_stream, env=spoof_tesseract_noop) + p_args, + close_fds=True, + stdout=PIPE, + stderr=PIPE, + stdin=input_stream, + env=spoof_tesseract_noop, + ) out, err = p.communicate() assert p.returncode == ExitCode.ok @@ -483,8 +573,13 @@ def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf): with open(output_file, 'wb') as output_stream: p_args = ocrmypdf_exec + [input_file, '-'] p = Popen( - p_args, close_fds=True, stdout=output_stream, stderr=PIPE, - stdin=DEVNULL, env=spoof_tesseract_noop) + p_args, + close_fds=True, + stdout=output_stream, + stderr=PIPE, + stdin=DEVNULL, + env=spoof_tesseract_noop, + ) out, err = p.communicate() assert p.returncode == ExitCode.ok @@ -492,8 +587,9 @@ def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf): assert qpdf.check(output_file, log=None) -@pytest.mark.skipif(sys.version_info[0:3] >= (3, 6, 4), - reason="issue fixed in Python 3.6.4") +@pytest.mark.skipif( + sys.version_info[0:3] >= (3, 6, 4), reason="issue fixed in Python 3.6.4" +) def test_closed_streams(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf): input_file = str(resources / 'francais.pdf') output_file = str(outpdf) @@ -504,8 +600,14 @@ def test_closed_streams(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf): p_args = ocrmypdf_exec + [input_file, output_file] p = Popen( - p_args, close_fds=True, stdout=None, stderr=PIPE, stdin=None, - env=spoof_tesseract_noop, preexec_fn=evil_closer) + p_args, + close_fds=True, + stdout=None, + stderr=PIPE, + stdin=None, + env=spoof_tesseract_noop, + preexec_fn=evil_closer, + ) out, err = p.communicate() print(err.decode()) assert p.returncode == ExitCode.ok @@ -513,36 +615,32 @@ def test_closed_streams(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf): def test_masks(spoof_tesseract_noop, resources, outpdf): p, out, err = run_ocrmypdf( - resources / 'masks.pdf', outpdf, env=spoof_tesseract_noop) + resources / 'masks.pdf', outpdf, env=spoof_tesseract_noop + ) assert p.returncode == ExitCode.ok -def test_linearized_pdf_and_indirect_object(spoof_tesseract_noop, - resources, outpdf): - check_ocrmypdf( - resources / 'epson.pdf', outpdf, - env=spoof_tesseract_noop) +def test_linearized_pdf_and_indirect_object(spoof_tesseract_noop, resources, outpdf): + check_ocrmypdf(resources / 'epson.pdf', outpdf, env=spoof_tesseract_noop) def test_ghostscript_pdfa_failure(spoof_no_tess_no_pdfa, resources, outpdf): p, out, err = run_ocrmypdf( - resources / 'ccitt.pdf', outpdf, - env=spoof_no_tess_no_pdfa) - assert p.returncode == ExitCode.pdfa_conversion_failed, \ - "Unexpected return when PDF/A fails" + resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_no_pdfa + ) + assert ( + p.returncode == ExitCode.pdfa_conversion_failed + ), "Unexpected return when PDF/A fails" -def test_ghostscript_feature_elision( - spoof_no_tess_pdfa_warning, resources, outpdf): - check_ocrmypdf(resources / 'ccitt.pdf', outpdf, - env=spoof_no_tess_pdfa_warning) +def test_ghostscript_feature_elision(spoof_no_tess_pdfa_warning, resources, outpdf): + check_ocrmypdf(resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_pdfa_warning) def test_very_high_dpi(spoof_tesseract_cache, resources, outpdf): "Checks for a Decimal quantize error with high DPI, etc" - check_ocrmypdf(resources / '2400dpi.pdf', outpdf, - env=spoof_tesseract_cache) + check_ocrmypdf(resources / '2400dpi.pdf', outpdf, env=spoof_tesseract_cache) pdfinfo = PdfInfo(outpdf) image = pdfinfo[0].images[0] @@ -551,9 +649,9 @@ def test_very_high_dpi(spoof_tesseract_cache, resources, outpdf): def test_overlay(spoof_tesseract_noop, resources, outpdf): - check_ocrmypdf(resources / 'overlay.pdf', outpdf, - '--skip-text', - env=spoof_tesseract_noop) + check_ocrmypdf( + resources / 'overlay.pdf', outpdf, '--skip-text', env=spoof_tesseract_noop + ) def test_destination_not_writable(spoof_tesseract_noop, resources, outdir): @@ -563,23 +661,25 @@ def test_destination_not_writable(spoof_tesseract_noop, resources, outdir): protected_file.touch() protected_file.chmod(0o400) # Read-only p, out, err = run_ocrmypdf( - resources / 'jbig2.pdf', protected_file, - env=spoof_tesseract_noop) + resources / 'jbig2.pdf', protected_file, env=spoof_tesseract_noop + ) assert p.returncode == ExitCode.file_access_error, "Expected error" def test_tesseract_config_valid(resources, outdir): cfg_file = outdir / 'test.cfg' with cfg_file.open('w') as f: - f.write('''\ + f.write( + '''\ load_system_dawg 0 language_model_penalty_non_dict_word 0 language_model_penalty_non_freq_dict_word 0 -''') +''' + ) check_ocrmypdf( - resources / 'ccitt.pdf', outdir / 'out.pdf', - '--tesseract-config', cfg_file) + resources / 'ccitt.pdf', outdir / 'out.pdf', '--tesseract-config', cfg_file + ) @pytest.mark.parametrize('renderer', RENDERERS) @@ -587,9 +687,13 @@ def test_tesseract_config_notfound(renderer, resources, outdir): cfg_file = outdir / 'nofile.cfg' p, out, err = run_ocrmypdf( - resources / 'ccitt.pdf', outdir / 'out.pdf', - '--pdf-renderer', renderer, - '--tesseract-config', cfg_file) + resources / 'ccitt.pdf', + outdir / 'out.pdf', + '--pdf-renderer', + renderer, + '--tesseract-config', + cfg_file, + ) assert "Can't open" in err, "No error message about missing config file" assert p.returncode == ExitCode.ok, err @@ -598,14 +702,20 @@ def test_tesseract_config_notfound(renderer, resources, outdir): def test_tesseract_config_invalid(renderer, resources, outdir): cfg_file = outdir / 'test.cfg' with cfg_file.open('w') as f: - f.write('''\ + f.write( + '''\ THIS FILE IS INVALID -''') +''' + ) p, out, err = run_ocrmypdf( - resources / 'ccitt.pdf', outdir / 'out.pdf', - '--pdf-renderer', renderer, - '--tesseract-config', cfg_file) + resources / 'ccitt.pdf', + outdir / 'out.pdf', + '--pdf-renderer', + renderer, + '--tesseract-config', + cfg_file, + ) assert "parameter not found" in err.lower(), "No error message" assert p.returncode == ExitCode.invalid_config @@ -622,9 +732,12 @@ def test_user_words(resources, outdir): if consistent: check_ocrmypdf( - resources / 'crom.png', outdir / 'out.pdf', - '--image-dpi', 150, - '--sidecar', sidecar_before + resources / 'crom.png', + outdir / 'out.pdf', + '--image-dpi', + 150, + '--sidecar', + sidecar_before, ) assert 'cromulent' not in sidecar_before.open().read() @@ -632,10 +745,14 @@ def test_user_words(resources, outdir): f.write('cromulent\n') # a perfectly cromulent word check_ocrmypdf( - resources / 'crom.png', outdir / 'out.pdf', - '--image-dpi', 150, - '--sidecar', sidecar_after, - '--user-words', word_list + resources / 'crom.png', + outdir / 'out.pdf', + '--image-dpi', + 150, + '--sidecar', + sidecar_after, + '--user-words', + word_list, ) if consistent: @@ -643,15 +760,14 @@ def test_user_words(resources, outdir): def test_form_xobject(spoof_tesseract_noop, resources, outpdf): - check_ocrmypdf(resources / 'formxobject.pdf', outpdf, - '--force-ocr', - env=spoof_tesseract_noop) + check_ocrmypdf( + resources / 'formxobject.pdf', outpdf, '--force-ocr', env=spoof_tesseract_noop + ) @pytest.mark.parametrize('renderer', RENDERERS) def test_pagesize_consistency(renderer, resources, outpdf): - first_page_dimensions = pytest.helpers.first_page_dimensions infile = resources / 'linn.pdf' @@ -660,8 +776,14 @@ def test_pagesize_consistency(renderer, resources, outpdf): check_ocrmypdf( infile, - outpdf, '--pdf-renderer', renderer, - '--clean', '--deskew', '--remove-background', '--clean-final') + outpdf, + '--pdf-renderer', + renderer, + '--clean', + '--deskew', + '--remove-background', + '--clean-final', + ) after_dims = first_page_dimensions(outpdf) @@ -670,43 +792,48 @@ def test_pagesize_consistency(renderer, resources, outpdf): def test_skip_big_with_no_images(spoof_tesseract_noop, resources, outpdf): - check_ocrmypdf(resources / 'blank.pdf', outpdf, - '--skip-big', '5', - '--force-ocr', - env=spoof_tesseract_noop) + check_ocrmypdf( + resources / 'blank.pdf', + outpdf, + '--skip-big', + '5', + '--force-ocr', + env=spoof_tesseract_noop, + ) def test_gs_render_failure(spoof_no_tess_gs_render_fail, resources, outpdf): p, out, err = run_ocrmypdf( - resources / 'blank.pdf', outpdf, - env=spoof_no_tess_gs_render_fail) + resources / 'blank.pdf', outpdf, env=spoof_no_tess_gs_render_fail + ) print(err) assert p.returncode == ExitCode.child_process_error def test_gs_raster_failure(spoof_no_tess_gs_raster_fail, resources, outpdf): p, out, err = run_ocrmypdf( - resources / 'ccitt.pdf', outpdf, - env=spoof_no_tess_gs_raster_fail) + resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_gs_raster_fail + ) print(err) assert p.returncode == ExitCode.child_process_error -@pytest.mark.skipif('8.0.0' <= qpdf.version() <= '8.0.1', - reason="qpdf regression on pages with no contents") +@pytest.mark.skipif( + '8.0.0' <= qpdf.version() <= '8.0.1', + reason="qpdf regression on pages with no contents", +) def test_no_contents(spoof_tesseract_noop, resources, outpdf): - check_ocrmypdf(resources / 'no_contents.pdf', outpdf, '--force-ocr', - env=spoof_tesseract_noop) + check_ocrmypdf( + resources / 'no_contents.pdf', outpdf, '--force-ocr', env=spoof_tesseract_noop + ) -@pytest.mark.parametrize('image', [ - 'baiona.png', - 'baiona_gray.png', - 'baiona_alpha.png', - 'congress.jpg' - ]) -def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec, - resources, image, outpdf): +@pytest.mark.parametrize( + 'image', ['baiona.png', 'baiona_gray.png', 'baiona_alpha.png', 'congress.jpg'] +) +def test_compression_preserved( + spoof_tesseract_noop, ocrmypdf_exec, resources, image, outpdf +): input_file = str(resources / image) output_file = str(outpdf) @@ -714,11 +841,23 @@ def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec, # Runs: ocrmypdf - output.pdf < testfile with open(input_file, 'rb') as input_stream: p_args = ocrmypdf_exec + [ - '--optimize', '0', - '--image-dpi', '150', '--output-type', 'pdf', '-', output_file] + '--optimize', + '0', + '--image-dpi', + '150', + '--output-type', + 'pdf', + '-', + output_file, + ] p = Popen( - p_args, close_fds=True, stdout=PIPE, stderr=PIPE, - stdin=input_stream, env=spoof_tesseract_noop) + p_args, + close_fds=True, + stdout=PIPE, + stderr=PIPE, + stdin=input_stream, + env=spoof_tesseract_noop, + ) out, err = p.communicate() if im.mode in ('RGBA', 'LA'): @@ -733,26 +872,26 @@ def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec, pdfimage = pdfinfo[0].images[0] if input_file.endswith('.png'): - assert pdfimage.enc != Encoding.jpeg, \ - "Lossless compression changed to lossy!" + assert pdfimage.enc != Encoding.jpeg, "Lossless compression changed to lossy!" elif input_file.endswith('.jpg'): - assert pdfimage.enc == Encoding.jpeg, \ - "Lossy compression changed to lossless!" + assert pdfimage.enc == Encoding.jpeg, "Lossy compression changed to lossless!" if im.mode.startswith('RGB') or im.mode.startswith('BGR'): - assert pdfimage.color == Colorspace.rgb, \ - "Colorspace changed" + assert pdfimage.color == Colorspace.rgb, "Colorspace changed" elif im.mode.startswith('L'): - assert pdfimage.color == Colorspace.gray, \ - "Colorspace changed" + assert pdfimage.color == Colorspace.gray, "Colorspace changed" -@pytest.mark.parametrize('image,compression', [ - ('baiona.png', 'jpeg'), - ('baiona_gray.png', 'lossless'), - ('congress.jpg', 'lossless') - ]) -def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec, - resources, image, compression, outpdf): +@pytest.mark.parametrize( + 'image,compression', + [ + ('baiona.png', 'jpeg'), + ('baiona_gray.png', 'lossless'), + ('congress.jpg', 'lossless'), + ], +) +def test_compression_changed( + spoof_tesseract_noop, ocrmypdf_exec, resources, image, compression, outpdf +): input_file = str(resources / image) output_file = str(outpdf) @@ -761,13 +900,25 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec, # Runs: ocrmypdf - output.pdf < testfile with open(input_file, 'rb') as input_stream: p_args = ocrmypdf_exec + [ - '--image-dpi', '150', '--output-type', 'pdfa', - '--optimize', '0', - '--pdfa-image-compression', compression, - '-', output_file] + '--image-dpi', + '150', + '--output-type', + 'pdfa', + '--optimize', + '0', + '--pdfa-image-compression', + compression, + '-', + output_file, + ] p = Popen( - p_args, close_fds=True, stdout=PIPE, stderr=PIPE, - stdin=input_stream, env=spoof_tesseract_noop) + p_args, + close_fds=True, + stdout=PIPE, + stderr=PIPE, + stdin=input_stream, + env=spoof_tesseract_noop, + ) out, err = p.communicate() assert p.returncode == ExitCode.ok, err @@ -788,20 +939,21 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec, assert pdfimage.enc not in (Encoding.jpeg, Encoding.jpeg2000) if im.mode.startswith('RGB') or im.mode.startswith('BGR'): - assert pdfimage.color == Colorspace.rgb, \ - "Colorspace changed" + assert pdfimage.color == Colorspace.rgb, "Colorspace changed" elif im.mode.startswith('L'): - assert pdfimage.color == Colorspace.gray, \ - "Colorspace changed" + assert pdfimage.color == Colorspace.gray, "Colorspace changed" def test_sidecar_pagecount(spoof_tesseract_cache, resources, outpdf): sidecar = outpdf + '.txt' check_ocrmypdf( - resources / 'multipage.pdf', outpdf, + resources / 'multipage.pdf', + outpdf, '--skip-text', - '--sidecar', sidecar, - env=spoof_tesseract_cache) + '--sidecar', + sidecar, + env=spoof_tesseract_cache, + ) pdfinfo = PdfInfo(resources / 'multipage.pdf') num_pages = len(pdfinfo) @@ -811,16 +963,15 @@ def test_sidecar_pagecount(spoof_tesseract_cache, resources, outpdf): # There should a formfeed between each pair of pages, so the count of # formfeeds is the page count less one - assert ocr_text.count('\f') == num_pages - 1, \ - "Sidecar page count does not match PDF page count" + assert ( + ocr_text.count('\f') == num_pages - 1 + ), "Sidecar page count does not match PDF page count" def test_sidecar_nonempty(spoof_tesseract_cache, resources, outpdf): sidecar = outpdf + '.txt' check_ocrmypdf( - resources / 'ccitt.pdf', outpdf, - '--sidecar', sidecar, - env=spoof_tesseract_cache + resources / 'ccitt.pdf', outpdf, '--sidecar', sidecar, env=spoof_tesseract_cache ) with open(sidecar, 'r') as f: @@ -834,9 +985,11 @@ def test_pdfa_n(spoof_tesseract_cache, pdfa_level, resources, outpdf): pytest.xfail(reason='Ghostscript >= 9.19 required') check_ocrmypdf( - resources / 'ccitt.pdf', outpdf, - '--output-type', 'pdfa-' + pdfa_level, - env=spoof_tesseract_cache + resources / 'ccitt.pdf', + outpdf, + '--output-type', + 'pdfa-' + pdfa_level, + env=spoof_tesseract_cache, ) pdfa_info = file_claims_pdfa(outpdf) @@ -848,9 +1001,7 @@ def test_bad_locale(): env = os.environ.copy() env['LC_ALL'] = 'C' - p, out, err = run_ocrmypdf( - 'a', 'b', env=env - ) + p, out, err = run_ocrmypdf('a', 'b', env=env) assert out == '', "stdout not clean" assert p.returncode != 0 assert 'configured to use ASCII as encoding' in err, "should whine" @@ -859,9 +1010,11 @@ def test_bad_locale(): @pytest.mark.parametrize('renderer', RENDERERS) def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf): p, out, err = run_ocrmypdf( - resources / 'ccitt.pdf', no_outpdf, - '--pdf-renderer', renderer, - env=spoof_tess_bad_utf8 + resources / 'ccitt.pdf', + no_outpdf, + '--pdf-renderer', + renderer, + env=spoof_tess_bad_utf8, ) assert out == '', "stdout not clean" @@ -871,33 +1024,27 @@ def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf): @pytest.mark.skipif( - PIL.__version__ < '5.0.0', - reason="Pillow < 5.0.0 doesn't raise the exception") + PIL.__version__ < '5.0.0', reason="Pillow < 5.0.0 doesn't raise the exception" +) def test_decompression_bomb(resources, outpdf): - p, out, err = run_ocrmypdf( - resources / 'hugemono.pdf', - outpdf - ) + p, out, err = run_ocrmypdf(resources / 'hugemono.pdf', outpdf) assert 'decompression bomb' in err p, out, err = run_ocrmypdf( - resources / 'hugemono.pdf', - outpdf, - '--max-image-mpixels', '2000' + resources / 'hugemono.pdf', outpdf, '--max-image-mpixels', '2000' ) assert p.returncode == 0 def test_text_curves(spoof_tesseract_noop, resources, outpdf): - check_ocrmypdf( - resources / 'vector.pdf', outpdf, env=spoof_tesseract_noop) + check_ocrmypdf(resources / 'vector.pdf', outpdf, env=spoof_tesseract_noop) info = PdfInfo(outpdf) assert len(info.pages[0].images) == 0, "added images to the vector PDF" check_ocrmypdf( - resources / 'vector.pdf', outpdf, '--force-ocr', - env=spoof_tesseract_noop) + resources / 'vector.pdf', outpdf, '--force-ocr', env=spoof_tesseract_noop + ) info = PdfInfo(outpdf) assert len(info.pages[0].images) != 0, "force did not rasterize" @@ -905,10 +1052,7 @@ def test_text_curves(spoof_tesseract_noop, resources, outpdf): def test_dev_null(spoof_tesseract_noop, resources): p, out, err = run_ocrmypdf( - resources / 'trivial.pdf', - os.devnull, - '--force-ocr', - env=spoof_tesseract_noop + resources / 'trivial.pdf', os.devnull, '--force-ocr', env=spoof_tesseract_noop ) assert p.returncode == 0, "could not send output to /dev/null" assert len(out) == 0, "wrote to stdout" @@ -916,10 +1060,7 @@ def test_dev_null(spoof_tesseract_noop, resources): def test_output_is_dir(spoof_tesseract_noop, resources, outdir): p, out, err = run_ocrmypdf( - resources / 'trivial.pdf', - outdir, - '--force-ocr', - env=spoof_tesseract_noop + resources / 'trivial.pdf', outdir, '--force-ocr', env=spoof_tesseract_noop ) assert p.returncode == ExitCode.file_access_error assert 'is not a writable file' in err @@ -929,20 +1070,14 @@ def test_output_is_symlink(spoof_tesseract_noop, resources, outdir): sym = Path(outdir / 'this_is_a_symlink') sym.symlink_to(outdir / 'out.pdf') p, out, err = run_ocrmypdf( - resources / 'trivial.pdf', - sym, - '--force-ocr', - env=spoof_tesseract_noop + resources / 'trivial.pdf', sym, '--force-ocr', env=spoof_tesseract_noop ) assert p.returncode == ExitCode.ok, err assert (outdir / 'out.pdf').stat().st_size > 0, 'target file not created' def test_livecycle(resources, no_outpdf): - p, _, err = run_ocrmypdf( - resources / 'livecycle.pdf', - no_outpdf - ) + p, _, err = run_ocrmypdf(resources / 'livecycle.pdf', no_outpdf) assert p.returncode == ExitCode.input_file, err diff --git a/tests/test_metadata.py b/tests/test_metadata.py index fcd08fde..d0e2b5d1 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -29,11 +29,7 @@ import pikepdf from pikepdf.models.metadata import decode_pdf_date from ocrmypdf.exceptions import ExitCode -from ocrmypdf.pdfa import ( - file_claims_pdfa, - generate_pdfa_ps, - SRGB_ICC_PROFILE -) +from ocrmypdf.pdfa import file_claims_pdfa, generate_pdfa_ps, SRGB_ICC_PROFILE from ocrmypdf.exec import ghostscript try: @@ -52,17 +48,17 @@ run_ocrmypdf = pytest.helpers.run_ocrmypdf spoof = pytest.helpers.spoof -@pytest.mark.parametrize("output_type", [ - 'pdfa', 'pdf' - ]) -def test_preserve_metadata(spoof_tesseract_noop, output_type, - resources, outpdf): +@pytest.mark.parametrize("output_type", ['pdfa', 'pdf']) +def test_preserve_metadata(spoof_tesseract_noop, output_type, resources, outpdf): pdf_before = pikepdf.open(resources / 'graph.pdf') output = check_ocrmypdf( - resources / 'graph.pdf', outpdf, - '--output-type', output_type, - env=spoof_tesseract_noop) + resources / 'graph.pdf', + outpdf, + '--output-type', + output_type, + env=spoof_tesseract_noop, + ) pdf_after = pikepdf.open(output) @@ -73,21 +69,23 @@ def test_preserve_metadata(spoof_tesseract_noop, output_type, assert pdfa_info['output'] == output_type -@pytest.mark.parametrize("output_type", [ - 'pdfa', 'pdf' - ]) -def test_override_metadata(spoof_tesseract_noop, output_type, resources, - outpdf): +@pytest.mark.parametrize("output_type", ['pdfa', 'pdf']) +def test_override_metadata(spoof_tesseract_noop, output_type, resources, outpdf): input_file = resources / 'c02-22.pdf' german = 'Du siehst den Wald vor lauter Bäumen nicht.' chinese = '孔子' p, out, err = run_ocrmypdf( - input_file, outpdf, - '--title', german, - '--author', chinese, - '--output-type', output_type, - env=spoof_tesseract_noop) + input_file, + outpdf, + '--title', + german, + '--author', + chinese, + '--output-type', + output_type, + env=spoof_tesseract_noop, + ) assert p.returncode == ExitCode.ok, err @@ -114,10 +112,14 @@ def test_high_unicode(spoof_tesseract_noop, resources, no_outpdf): high_unicode = 'U+1030C is: 𐌌' p, out, err = run_ocrmypdf( - input_file, no_outpdf, - '--subject', high_unicode, - '--output-type', 'pdfa', - env=spoof_tesseract_noop) + input_file, + no_outpdf, + '--subject', + high_unicode, + '--output-type', + 'pdfa', + env=spoof_tesseract_noop, + ) assert p.returncode == ExitCode.bad_args, err @@ -125,16 +127,20 @@ def test_high_unicode(spoof_tesseract_noop, resources, no_outpdf): @pytest.mark.skipif(not fitz, reason="test uses fitz") @pytest.mark.parametrize('ocr_option', ['--skip-text', '--force-ocr']) @pytest.mark.parametrize('output_type', ['pdf', 'pdfa']) -def test_bookmarks_preserved(spoof_tesseract_noop, output_type, ocr_option, - resources, outpdf): +def test_bookmarks_preserved( + spoof_tesseract_noop, output_type, ocr_option, resources, outpdf +): input_file = resources / 'toc.pdf' before_toc = fitz.Document(str(input_file)).getToC() check_ocrmypdf( - input_file, outpdf, + input_file, + outpdf, ocr_option, - '--output-type', output_type, - env=spoof_tesseract_noop) + '--output-type', + output_type, + env=spoof_tesseract_noop, + ) after_toc = fitz.Document(str(outpdf)).getToC() print(before_toc) @@ -148,13 +154,14 @@ def seconds_between_dates(date1, date2): @pytest.mark.parametrize('infile', ['trivial.pdf', 'jbig2.pdf']) @pytest.mark.parametrize('output_type', ['pdf', 'pdfa']) -def test_creation_date_preserved(spoof_tesseract_noop, output_type, resources, - infile, outpdf): +def test_creation_date_preserved( + spoof_tesseract_noop, output_type, resources, infile, outpdf +): input_file = resources / infile check_ocrmypdf( - input_file, outpdf, '--output-type', output_type, - env=spoof_tesseract_noop) + input_file, outpdf, '--output-type', output_type, env=spoof_tesseract_noop + ) pdf_before = pikepdf.open(input_file) pdf_after = pikepdf.open(outpdf) @@ -172,13 +179,11 @@ def test_creation_date_preserved(spoof_tesseract_noop, output_type, resources, # We expect that the modified date is quite recent date_after = decode_pdf_date(str(after['/ModDate'])) - assert seconds_between_dates( - date_after, datetime.datetime.now(timezone.utc)) < 1000 + assert seconds_between_dates(date_after, datetime.datetime.now(timezone.utc)) < 1000 @pytest.mark.parametrize('output_type', ['pdf', 'pdfa']) -def test_xml_metadata_preserved(spoof_tesseract_noop, output_type, - resources, outpdf): +def test_xml_metadata_preserved(spoof_tesseract_noop, output_type, resources, outpdf): input_file = resources / 'graph.pdf' try: @@ -191,9 +196,8 @@ def test_xml_metadata_preserved(spoof_tesseract_noop, output_type, before = file_to_dict(str(input_file)) check_ocrmypdf( - input_file, outpdf, - '--output-type', output_type, - env=spoof_tesseract_noop) + input_file, outpdf, '--output-type', output_type, env=spoof_tesseract_noop + ) after = file_to_dict(str(outpdf)) @@ -223,7 +227,7 @@ def test_xml_metadata_preserved(spoof_tesseract_noop, output_type, 'xmp:MetadataDate', 'xmp:CreatorTool', 'xmpMM:DocumentId', - 'xmpMM:DnstanceId' + 'xmpMM:DnstanceId', ] # Cleanup messy data structure @@ -251,8 +255,10 @@ def test_xml_metadata_preserved(spoof_tesseract_noop, output_type, # of several propidx = '{}[1]'.format(prop) if propidx in before: - assert after.get(propidx) == before[propidx] \ - or after.get(prop) == before[propidx] + assert ( + after.get(propidx) == before[propidx] + or after.get(prop) == before[propidx] + ) def test_srgb_in_unicode_path(tmpdir): @@ -270,9 +276,8 @@ def test_srgb_in_unicode_path(tmpdir): def test_kodak_toc(resources, outpdf, spoof_tesseract_noop): output = check_ocrmypdf( - resources / 'kcs.pdf', outpdf, - '--output-type', 'pdf', - env=spoof_tesseract_noop) + resources / 'kcs.pdf', outpdf, '--output-type', 'pdf', env=spoof_tesseract_noop + ) p = pikepdf.open(outpdf) @@ -297,7 +302,8 @@ def test_metadata_fixup_warning(resources, outdir): input_files_groups=input_files, output_file=outdir / 'out.pdf', log=log, - context=context) + context=context, + ) log.warning.assert_not_called() # Now add some metadata that will not be copyable @@ -312,5 +318,6 @@ def test_metadata_fixup_warning(resources, outdir): input_files_groups=input_files, output_file=outdir / 'out.pdf', log=log, - context=context) + context=context, + ) log.warning.assert_called_once() diff --git a/tests/test_optimize.py b/tests/test_optimize.py index e96ffe20..d868541a 100644 --- a/tests/test_optimize.py +++ b/tests/test_optimize.py @@ -46,9 +46,12 @@ def test_mono_not_inverted(resources, outdir): opt.main(infile, outdir / 'out.pdf', level=3) rasterize_pdf( - outdir / 'out.pdf', outdir / 'im.png', - xres=10, yres=10, raster_device='pnggray', - log=logging.getLogger(name='test_mono_flip') + outdir / 'out.pdf', + outdir / 'im.png', + xres=10, + yres=10, + raster_device='pnggray', + log=logging.getLogger(name='test_mono_flip'), ) im = Image.open(fspath(outdir / 'im.png')) @@ -57,9 +60,17 @@ def test_mono_not_inverted(resources, outdir): def test_jpg_png_params(resources, outpdf, spoof_tesseract_noop): check_ocrmypdf( - resources / 'crom.png', outpdf, '--image-dpi', '200', - '--optimize', '3', '--jpg-quality', '50', '--png-quality', '20', - env=spoof_tesseract_noop + resources / 'crom.png', + outpdf, + '--image-dpi', + '200', + '--optimize', + '3', + '--jpg-quality', + '50', + '--png-quality', + '20', + env=spoof_tesseract_noop, ) @@ -67,8 +78,16 @@ def test_jpg_png_params(resources, outpdf, spoof_tesseract_noop): @pytest.mark.parametrize('lossy', [False, True]) def test_jbig2_lossy(lossy, resources, outpdf, spoof_tesseract_noop): args = [ - resources / 'ccitt.pdf', outpdf, '--image-dpi', '200', - '--optimize', 3, '--jpg-quality', '50', '--png-quality', '20' + resources / 'ccitt.pdf', + outpdf, + '--image-dpi', + '200', + '--optimize', + 3, + '--jpg-quality', + '50', + '--png-quality', + '20', ] if lossy: args.append('--jbig2-lossy') @@ -85,8 +104,10 @@ def test_jbig2_lossy(lossy, resources, outpdf, spoof_tesseract_noop): assert len(pim.decode_parms) == 0 -@pytest.mark.skipif(not jbig2enc.available() or not pngquant.available(), - reason='need jbig2enc and pngquant') +@pytest.mark.skipif( + not jbig2enc.available() or not pngquant.available(), + reason='need jbig2enc and pngquant', +) def test_flate_to_jbig2(resources, outdir, spoof_tesseract_noop): # This test requires an image that pngquant is capable of converting to # to 1bpp - so use an existing 1bpp image, convert up, confirm it can @@ -97,9 +118,15 @@ def test_flate_to_jbig2(resources, outdir, spoof_tesseract_noop): im.save(fspath(outdir / 'type8.png')) check_ocrmypdf( - outdir / 'type8.png', outdir / 'out.pdf', - '--image-dpi', '100', '--png-quality', '10', '--optimize', '3', - env=spoof_tesseract_noop + outdir / 'type8.png', + outdir / 'out.pdf', + '--image-dpi', + '100', + '--png-quality', + '10', + '--optimize', + '3', + env=spoof_tesseract_noop, ) pdf = pikepdf.open(outdir / 'out.pdf') diff --git a/tests/test_pdfinfo.py b/tests/test_pdfinfo.py index 9726ed1a..88f2a879 100644 --- a/tests/test_pdfinfo.py +++ b/tests/test_pdfinfo.py @@ -33,12 +33,13 @@ import pickle def test_single_page_text(outdir): filename = outdir / 'text.pdf' - pdf = Canvas(str(filename), pagesize=(8*72, 6*72)) + pdf = Canvas(str(filename), pagesize=(8 * 72, 6 * 72)) text = pdf.beginText() text.setFont('Helvetica', 12) - text.setTextOrigin(1*72, 3*72) - text.textLine("Methink'st thou art a general offence and every" - " man should beat thee.") + text.setTextOrigin(1 * 72, 3 * 72) + text.textLine( + "Methink'st thou art a general offence and every" " man should beat thee." + ) pdf.drawText(text) pdf.showPage() pdf.save() @@ -66,8 +67,8 @@ def test_single_page_image(outdir): im_bytes = im_tmp.read_bytes() pdf_bytes = img2pdf.convert( - im_bytes, producer="img2pdf", with_pdfrw=False, - layout_fun=layout_fun) + im_bytes, producer="img2pdf", with_pdfrw=False, layout_fun=layout_fun + ) filename.write_bytes(pdf_bytes) info = pdfinfo.PdfInfo(filename) @@ -89,7 +90,7 @@ def test_single_page_image(outdir): def test_single_page_inline_image(outdir): filename = outdir / 'image-mono-inline.pdf' - pdf = Canvas(str(filename), pagesize=(8*72, 6*72)) + pdf = Canvas(str(filename), pagesize=(8 * 72, 6 * 72)) with NamedTemporaryFile() as im_tmp: im = Image.new('1', (8, 8), 0) for n in range(8): @@ -157,12 +158,7 @@ def test_regex(): b'', b'', ] - must_not_match = [ - b'', - b'', - b'', - b'' - ] + must_not_match = [b'', b'', b'', b''] for s in must_match: assert rx.match(s) diff --git a/tests/test_qpdf.py b/tests/test_qpdf.py index c6387834..0e925249 100644 --- a/tests/test_qpdf.py +++ b/tests/test_qpdf.py @@ -19,6 +19,7 @@ import pytest import ocrmypdf.exec.qpdf as qpdf + def test_qpdf_error(resources): assert qpdf.check(resources / 'blank.pdf') assert not qpdf.check(__file__) diff --git a/tests/test_rotation.py b/tests/test_rotation.py index 77fed5e7..c210c3a9 100644 --- a/tests/test_rotation.py +++ b/tests/test_rotation.py @@ -36,7 +36,7 @@ from ocrmypdf.exec import ghostscript, tesseract pytestmark = pytest.mark.skipif( leptonica.get_leptonica_version() < 'leptonica-1.72', - reason="Leptonica is too old, correlation doesn't work" + reason="Leptonica is too old, correlation doesn't work", ) check_ocrmypdf = pytest.helpers.check_ocrmypdf @@ -47,23 +47,29 @@ RENDERERS = ['hocr', 'sandwich'] def check_monochrome_correlation( - outdir, - reference_pdf, reference_pageno, test_pdf, test_pageno): + outdir, reference_pdf, reference_pageno, test_pdf, test_pageno +): gslog = logging.getLogger() reference_png = outdir / '{}.ref{:04d}.png'.format( - reference_pdf.name, reference_pageno) - test_png = outdir / '{}.test{:04d}.png'.format( - test_pdf.name, test_pageno) + reference_pdf.name, reference_pageno + ) + test_png = outdir / '{}.test{:04d}.png'.format(test_pdf.name, test_pageno) def rasterize(pdf, pageno, png): if png.exists(): print(png) return ghostscript.rasterize_pdf( - pdf, png, xres=100, yres=100, - raster_device='pngmono', log=gslog, pageno=pageno, - rotation=0) + pdf, + png, + xres=100, + yres=100, + raster_device='pngmono', + log=gslog, + pageno=pageno, + rotation=0, + ) rasterize(reference_pdf, reference_pageno, reference_png) rasterize(test_pdf, test_pageno, test_png) @@ -83,7 +89,7 @@ def test_monochrome_correlation(resources, outdir): reference_pageno=1, # north facing page test_pdf=resources / 'cardinal.pdf', test_pageno=3, # south facing page - ) + ) assert corr < 0.10 corr = check_monochrome_correlation( outdir, @@ -91,7 +97,7 @@ def test_monochrome_correlation(resources, outdir): reference_pageno=2, test_pdf=resources / 'cardinal.pdf', test_pageno=2, - ) + ) assert corr > 0.90 @@ -100,35 +106,55 @@ def test_monochrome_correlation(resources, outdir): def test_autorotate(spoof_tesseract_cache, renderer, resources, outdir): # cardinal.pdf contains four copies of an image rotated in each cardinal # direction - these ones are "burned in" not tagged with /Rotate - out = check_ocrmypdf(resources / 'cardinal.pdf', outdir / 'out.pdf', - '-r', '-v', '1', '--pdf-renderer', renderer, - env=spoof_tesseract_cache) - for n in range(1, 4+1): + out = check_ocrmypdf( + resources / 'cardinal.pdf', + outdir / 'out.pdf', + '-r', + '-v', + '1', + '--pdf-renderer', + renderer, + env=spoof_tesseract_cache, + ) + for n in range(1, 4 + 1): correlation = check_monochrome_correlation( outdir, reference_pdf=resources / 'cardinal.pdf', reference_pageno=1, test_pdf=outdir / 'out.pdf', - test_pageno=n) + test_pageno=n, + ) assert correlation > 0.80 -@pytest.mark.parametrize('threshold, correlation_test', [ - ('1', 'correlation > 0.80'), # Low thresh -> always rotate -> high corr - ('99', 'correlation < 0.10'), # High thres -> never rotate -> low corr -]) +@pytest.mark.parametrize( + 'threshold, correlation_test', + [ + ('1', 'correlation > 0.80'), # Low thresh -> always rotate -> high corr + ('99', 'correlation < 0.10'), # High thres -> never rotate -> low corr + ], +) def test_autorotate_threshold( - spoof_tesseract_cache, threshold, correlation_test, resources, outdir): - out = check_ocrmypdf(resources / 'cardinal.pdf', outdir / 'out.pdf', - '--rotate-pages-threshold', threshold, - '-r', '-v', '1', env=spoof_tesseract_cache) + spoof_tesseract_cache, threshold, correlation_test, resources, outdir +): + out = check_ocrmypdf( + resources / 'cardinal.pdf', + outdir / 'out.pdf', + '--rotate-pages-threshold', + threshold, + '-r', + '-v', + '1', + env=spoof_tesseract_cache, + ) correlation = check_monochrome_correlation( outdir, reference_pdf=resources / 'cardinal.pdf', reference_pageno=1, test_pdf=outdir / 'out.pdf', - test_pageno=3) + test_pageno=3, + ) assert eval(correlation_test) @@ -143,27 +169,31 @@ def test_rotated_skew_timeout(resources, outpdf): input_file = resources / 'rotated_skew.pdf' in_pageinfo = PdfInfo(input_file)[0] - assert in_pageinfo.height_pixels < in_pageinfo.width_pixels, \ - "Expected the input page to be landscape" + assert ( + in_pageinfo.height_pixels < in_pageinfo.width_pixels + ), "Expected the input page to be landscape" assert in_pageinfo.rotation == 90, "Expected a rotated page" out = check_ocrmypdf( - input_file, outpdf, - '--pdf-renderer', 'hocr', - '--deskew', '--tesseract-timeout', '0') + input_file, + outpdf, + '--pdf-renderer', + 'hocr', + '--deskew', + '--tesseract-timeout', + '0', + ) out_pageinfo = PdfInfo(out)[0] w, h = out_pageinfo.width_pixels, out_pageinfo.height_pixels - assert h > w, \ - "Expected the output page to be portrait" + assert h > w, "Expected the output page to be portrait" - assert out_pageinfo.rotation == 0, \ - "Expected no page rotation for output" + assert out_pageinfo.rotation == 0, "Expected no page rotation for output" - assert in_pageinfo.width_pixels == h and \ - in_pageinfo.height_pixels == w, \ - "Expected page rotation to be baked in" + assert ( + in_pageinfo.width_pixels == h and in_pageinfo.height_pixels == w + ), "Expected page rotation to be baked in" def test_rotate_deskew_timeout(resources, outdir): @@ -171,10 +201,13 @@ def test_rotate_deskew_timeout(resources, outdir): resources / 'rotated_skew.pdf', outdir / 'deskewed.pdf', '--rotate-pages', - '--rotate-pages-threshold', '0', + '--rotate-pages-threshold', + '0', '--deskew', - '--tesseract-timeout', '0', - '--pdf-renderer', 'sandwich' + '--tesseract-timeout', + '0', + '--pdf-renderer', + 'sandwich', ) correlation = check_monochrome_correlation( @@ -182,7 +215,8 @@ def test_rotate_deskew_timeout(resources, outdir): reference_pdf=resources / 'ccitt.pdf', reference_pageno=1, test_pdf=outdir / 'deskewed.pdf', - test_pageno=1) + test_pageno=1, + ) # Confirm that the page still got deskewed assert correlation > 0.50 @@ -192,7 +226,6 @@ def test_rotate_deskew_timeout(resources, outdir): @pytest.mark.parametrize('page_angle', (0, 90, 180, 270)) @pytest.mark.parametrize('image_angle', (0, 90, 180, 270)) def test_rotate_page_level(image_angle, page_angle, resources, outdir): - def make_rotate_test(prefix, image_angle, page_angle): im = Image.open(fspath(resources / 'typewriter.png')) if image_angle != 0: @@ -205,7 +238,7 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir): img2pdf.convert( memimg.read(), layout_fun=img2pdf.get_fixed_dpi_layout_fun((200, 200)), - outputstream=mempdf + outputstream=mempdf, ) mempdf.seek(0) pike = pikepdf.open(mempdf) @@ -219,11 +252,13 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir): out = test.with_suffix('.out.pdf') p, _, err = run_ocrmypdf( - test, out, + test, + out, '-O0', '--rotate-pages', - '--rotate-pages-threshold', '0.001', - universal_newlines=False + '--rotate-pages-threshold', + '0.001', + universal_newlines=False, ) err = err.decode('utf-8', errors='replace') assert p.returncode == 0, err @@ -238,4 +273,5 @@ def test_tesseract_orientation(resources, tmpdir): log = Mock() tesseract.get_orientation( # Test results of this are unreliable - tmpdir / '000001.png', engine_mode='3', timeout=10, log=log) + tmpdir / '000001.png', engine_mode='3', timeout=10, log=log + ) diff --git a/tests/test_tess4.py b/tests/test_tess4.py index ce4c2ed3..d2d25a5f 100644 --- a/tests/test_tess4.py +++ b/tests/test_tess4.py @@ -84,10 +84,11 @@ def tess4_available(): return False + # Skip all tests in this file if not tesseract 4 pytestmark = pytest.mark.skipif( - not tess4_available(), - reason="tesseract 4.0 with textonly_pdf feature required") + not tess4_available(), reason="tesseract 4.0 with textonly_pdf feature required" +) check_ocrmypdf = pytest.helpers.check_ocrmypdf run_ocrmypdf = pytest.helpers.run_ocrmypdf @@ -97,9 +98,13 @@ spoof = pytest.helpers.spoof def test_textonly_pdf(ensure_tess4, resources, outdir): check_ocrmypdf( resources / 'linn.pdf', - outdir / 'linn_textonly.pdf', '--pdf-renderer', 'sandwich', - '--sidecar', outdir / 'foo.txt', - env=ensure_tess4) + outdir / 'linn_textonly.pdf', + '--pdf-renderer', + 'sandwich', + '--sidecar', + outdir / 'foo.txt', + env=ensure_tess4, + ) def test_pagesize_consistency_tess4(ensure_tess4, resources, outpdf): @@ -111,9 +116,15 @@ def test_pagesize_consistency_tess4(ensure_tess4, resources, outpdf): check_ocrmypdf( infile, - outpdf, '--pdf-renderer', 'sandwich', - '--clean', '--deskew', '--remove-background', '--clean-final', - env=ensure_tess4) + outpdf, + '--pdf-renderer', + 'sandwich', + '--clean', + '--deskew', + '--remove-background', + '--clean-final', + env=ensure_tess4, + ) after_dims = pytest.helpers.first_page_dimensions(outpdf) @@ -122,16 +133,19 @@ def test_pagesize_consistency_tess4(ensure_tess4, resources, outpdf): @pytest.mark.parametrize('basename', ['graph_ocred.pdf', 'cardinal.pdf']) -def test_skip_pages_does_not_replicate( - ensure_tess4, resources, basename, outdir): +def test_skip_pages_does_not_replicate(ensure_tess4, resources, basename, outdir): infile = resources / basename outpdf = outdir / basename check_ocrmypdf( infile, - outpdf, '--pdf-renderer', 'sandwich', '--force-ocr', - '--tesseract-timeout', '0', - env=ensure_tess4 + outpdf, + '--pdf-renderer', + 'sandwich', + '--force-ocr', + '--tesseract-timeout', + '0', + env=ensure_tess4, ) info_in = pdfinfo.PdfInfo(infile) @@ -149,8 +163,12 @@ def test_content_preservation(ensure_tess4, resources, outpdf): check_ocrmypdf( infile, - outpdf, '--pdf-renderer', 'sandwich', '--tesseract-timeout', '0', - env=ensure_tess4 + outpdf, + '--pdf-renderer', + 'sandwich', + '--tesseract-timeout', + '0', + env=ensure_tess4, ) info = pdfinfo.PdfInfo(outpdf) diff --git a/tests/test_unpaper.py b/tests/test_unpaper.py index c236a6ca..6409f624 100644 --- a/tests/test_unpaper.py +++ b/tests/test_unpaper.py @@ -27,27 +27,27 @@ check_ocrmypdf = pytest.helpers.check_ocrmypdf run_ocrmypdf = pytest.helpers.run_ocrmypdf spoof = pytest.helpers.spoof + @pytest.fixture(scope='session') def spoof_unpaper_oldversion(tmpdir_factory): return spoof(tmpdir_factory, unpaper='unpaper_oldversion.py') -@pytest.mark.skipif(True, - reason="needs new fixture implementation") +@pytest.mark.skipif(True, reason="needs new fixture implementation") def test_no_unpaper(resources, no_outpdf): # p, out, err = run_ocrmypdf( - resources / 'c02-22.pdf', no_outpdf, '--clean', env=os.environ) + resources / 'c02-22.pdf', no_outpdf, '--clean', env=os.environ + ) assert p.returncode == ExitCode.missing_dependency def test_old_unpaper(spoof_unpaper_oldversion, resources, no_outpdf): p, out, err = run_ocrmypdf( - resources / 'c02-22.pdf', no_outpdf, '--clean', - env=spoof_unpaper_oldversion) + resources / 'c02-22.pdf', no_outpdf, '--clean', env=spoof_unpaper_oldversion + ) assert p.returncode == ExitCode.missing_dependency def test_clean(spoof_tesseract_noop, resources, outpdf): - check_ocrmypdf(resources / 'skew.pdf', outpdf, '-c', - env=spoof_tesseract_noop) \ No newline at end of file + check_ocrmypdf(resources / 'skew.pdf', outpdf, '-c', env=spoof_tesseract_noop) diff --git a/tests/test_userunit.py b/tests/test_userunit.py index 5a2d3e24..6120a7fa 100644 --- a/tests/test_userunit.py +++ b/tests/test_userunit.py @@ -46,14 +46,13 @@ def test_userunit_ghostscript_fails(poster, no_outpdf): def test_userunit_qpdf_passes(spoof_tesseract_cache, poster, outpdf): before = PdfInfo(poster) - check_ocrmypdf(poster, outpdf, '--output-type=pdf', - env=spoof_tesseract_cache) + check_ocrmypdf(poster, outpdf, '--output-type=pdf', env=spoof_tesseract_cache) after = PdfInfo(outpdf) assert isclose(before[0].width_inches, after[0].width_inches) def test_rotate_interaction(spoof_tesseract_cache, poster, outpdf): - check_ocrmypdf(poster, outpdf, '--output-type=pdf', - '--rotate-pages', - env=spoof_tesseract_cache) + check_ocrmypdf( + poster, outpdf, '--output-type=pdf', '--rotate-pages', env=spoof_tesseract_cache + )