menekis/OCRmyPDF
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reformat with black

Browse Source
This commit is contained in:
James R. Barlow
2018-12-30 01:27:49 -08:00
parent 80bd7de580
commit 06308a22ce
44 changed files with 2459 additions and 1558 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ocrmypdf/__init__.py
+12 -4
View File
@@ -25,10 +25,18 @@ __version__ = pkg_resources.get_distribution('ocrmypdf').version
VERSION = __version__
from .exceptions import (
ExitCode, BadArgsError, PdfMergeFailedError, MissingDependencyError,
UnsupportedImageFormatError, DpiError, OutputFileAccessError,
PriorOcrFoundError, InputFileError, SubprocessOutputError,
EncryptedPdfError, TesseractConfigError
ExitCode,
BadArgsError,
PdfMergeFailedError,
MissingDependencyError,
UnsupportedImageFormatError,
DpiError,
OutputFileAccessError,
PriorOcrFoundError,
InputFileError,
SubprocessOutputError,
EncryptedPdfError,
TesseractConfigError,
)
from . import helpers
src/ocrmypdf/__main__.py
+376 -228
View File
@@ -35,13 +35,18 @@ import ruffus.proxy_logger as proxy_logger
from ._jobcontext import JobContext, JobContextManager, cleanup_working_files
from ._pipeline import build_pipeline
from .pdfa import file_claims_pdfa
from .helpers import re_symlink, is_file_writable, \
available_cpu_count
from .helpers import re_symlink, is_file_writable, available_cpu_count
from .exec import tesseract, qpdf, ghostscript
from . import PROGRAM_NAME, VERSION
from .exceptions import ExitCode, ExitCodeException, MissingDependencyError, \
InputFileError, BadArgsError, OutputFileAccessError
from .exceptions import (
ExitCode,
ExitCodeException,
MissingDependencyError,
InputFileError,
BadArgsError,
OutputFileAccessError,
)
from . import exceptions as ocrmypdf_exceptions
from ._unicodefun import verify_python3_env
@@ -49,9 +54,8 @@ from ._unicodefun import verify_python3_env
# -------------
# External dependencies
HOCR_OK_LANGS = frozenset([
'eng', 'deu', 'spa', 'ita', 'por'
])
HOCR_OK_LANGS = frozenset(['eng', 'deu', 'spa', 'ita', 'por'])
def complain(message):
print(*textwrap.wrap(message), file=sys.stderr)
@@ -69,24 +73,26 @@ verify_python3_env()
if not tesseract.v4:
complain(
"Please install tesseract 4.0.0 or newer "
"(currently installed version is {1})".format(
tesseract.version()))
"(currently installed version is {1})".format(tesseract.version())
)
sys.exit(ExitCode.missing_dependency)
# -------------
# Parser
def numeric(basetype, min_=None, max_=None):
"""Validator for numeric params"""
min_ = basetype(min_) if min_ is not None else None
max_ = basetype(max_) if max_ is not None else None
def _numeric(string):
value = basetype(string)
if (min_ is not None and value < min_
or max_ is not None and value > max_):
if min_ is not None and value < min_ or max_ is not None and value > max_:
msg = "%r not in valid range %r" % (string, (min_, max_))
raise argparse.ArgumentTypeError(msg)
return value
_numeric.__name__ = basetype.__name__
return _numeric
@@ -142,259 +148,366 @@ ocrmypdf so it is already installed.
Online documentation is located at:
https://ocrmypdf.readthedocs.io/en/latest/introduction.html
""")
""",
)
parser.add_argument(
'input_file', metavar="input_pdf_or_image",
'input_file',
metavar="input_pdf_or_image",
help="PDF file containing the images to be OCRed (or '-' to read from "
"standard input)")
"standard input)",
)
parser.add_argument(
'output_file', metavar="output_pdf",
'output_file',
metavar="output_pdf",
help="Output searchable PDF file (or '-' to write to standard output). "
"Existing files will be ovewritten. If same as input file, the "
"input file will be updated only if processing is successful.")
"Existing files will be ovewritten. If same as input file, the "
"input file will be updated only if processing is successful.",
)
parser.add_argument(
'-l', '--language', action='append',
'-l',
'--language',
action='append',
help="Language(s) of the file to be OCRed (see tesseract --list-langs for "
"all language packs installed in your system). Use -l eng+deu for "
"multiple languages.")
"all language packs installed in your system). Use -l eng+deu for "
"multiple languages.",
)
parser.add_argument(
'--image-dpi', metavar='DPI', type=int,
help="For input image instead of PDF, use this DPI instead of file's.")
'--image-dpi',
metavar='DPI',
type=int,
help="For input image instead of PDF, use this DPI instead of file's.",
)
parser.add_argument(
'--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'],
'--output-type',
choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'],
default='pdfa',
help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for "
"long term archiving (default, recommended) but may not suitable "
"for users who want their file altered as little as possible. 'pdfa' "
"also has problems with full Unicode text. 'pdf' attempts to "
"preserve file contents as much as possible. 'pdf-a1' creates a "
"PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'. 'pdf-a3' creates a "
"PDF/A3-b file."
)
"long term archiving (default, recommended) but may not suitable "
"for users who want their file altered as little as possible. 'pdfa' "
"also has problems with full Unicode text. 'pdf' attempts to "
"preserve file contents as much as possible. 'pdf-a1' creates a "
"PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'. 'pdf-a3' creates a "
"PDF/A3-b file.",
)
# Use null string '\0' as sentinel to indicate the user supplied no argument,
# since that is the only invalid character for filepaths on all platforms
# bool('\0') is True in Python
parser.add_argument(
'--sidecar', nargs='?', const='\0', default=None, metavar='FILE',
'--sidecar',
nargs='?',
const='\0',
default=None,
metavar='FILE',
help="Generate sidecar text files that contain the same text recognized "
"by Tesseract. This may be useful for building a OCR text database. "
"If FILE is omitted, the sidecar file be named {output_file}.txt "
"If FILE is set to '-', the sidecar is written to stdout (a "
"convenient way to preview OCR quality). The output file and sidecar "
"may not both use stdout at the same time.")
"by Tesseract. This may be useful for building a OCR text database. "
"If FILE is omitted, the sidecar file be named {output_file}.txt "
"If FILE is set to '-', the sidecar is written to stdout (a "
"convenient way to preview OCR quality). The output file and sidecar "
"may not both use stdout at the same time.",
)
parser.add_argument(
'--version', action='version', version=VERSION,
help="Print program version and exit")
'--version',
action='version',
version=VERSION,
help="Print program version and exit",
)
jobcontrol = parser.add_argument_group(
"Job control options")
jobcontrol = parser.add_argument_group("Job control options")
jobcontrol.add_argument(
'-j', '--jobs', metavar='N', type=numeric(int, 0, 256),
help="Use up to N CPU cores simultaneously (default: use all).")
'-j',
'--jobs',
metavar='N',
type=numeric(int, 0, 256),
help="Use up to N CPU cores simultaneously (default: use all).",
)
jobcontrol.add_argument(
'-q', '--quiet', action='store_true', help="Suppress INFO messages")
'-q', '--quiet', action='store_true', help="Suppress INFO messages"
)
jobcontrol.add_argument(
'-v', '--verbose', const="+", default=[], nargs='?', action="append",
help="Print more verbose messages for each additional verbose level")
'-v',
'--verbose',
const="+",
default=[],
nargs='?',
action="append",
help="Print more verbose messages for each additional verbose level",
)
metadata = parser.add_argument_group(
"Metadata options",
"Set output PDF/A metadata (default: copy input document's metadata)")
"Set output PDF/A metadata (default: copy input document's metadata)",
)
metadata.add_argument(
'--title', type=str,
help="Set document title (place multiple words in quotes)")
metadata.add_argument(
'--author', type=str,
help="Set document author")
metadata.add_argument(
'--subject', type=str,
help="Set document subject description")
metadata.add_argument(
'--keywords', type=str,
help="Set document keywords")
'--title', type=str, help="Set document title (place multiple words in quotes)"
)
metadata.add_argument('--author', type=str, help="Set document author")
metadata.add_argument('--subject', type=str, help="Set document subject description")
metadata.add_argument('--keywords', type=str, help="Set document keywords")
preprocessing = parser.add_argument_group(
"Image preprocessing options",
"Options to improve the quality of the final PDF and OCR")
"Options to improve the quality of the final PDF and OCR",
)
preprocessing.add_argument(
'-r', '--rotate-pages', action='store_true',
help="Automatically rotate pages based on detected text orientation")
'-r',
'--rotate-pages',
action='store_true',
help="Automatically rotate pages based on detected text orientation",
)
preprocessing.add_argument(
'--remove-background', action='store_true',
'--remove-background',
action='store_true',
help="Attempt to remove background from gray or color pages, setting it "
"to white ")
"to white ",
)
preprocessing.add_argument(
'-d', '--deskew', action='store_true',
help="Deskew each page before performing OCR")
'-d', '--deskew', action='store_true', help="Deskew each page before performing OCR"
)
preprocessing.add_argument(
'-c', '--clean', action='store_true',
'-c',
'--clean',
action='store_true',
help="Clean pages from scanning artifacts before performing OCR, and send "
"the cleaned page to OCR, but do not include the cleaned page in "
"the output")
"the cleaned page to OCR, but do not include the cleaned page in "
"the output",
)
preprocessing.add_argument(
'-i', '--clean-final', action='store_true',
'-i',
'--clean-final',
action='store_true',
help="Clean page as above, and incorporate the cleaned image in the final "
"PDF. Might remove desired content.")
"PDF. Might remove desired content.",
)
preprocessing.add_argument(
'--oversample', metavar='DPI', type=numeric(int, 0, 5000), default=0,
'--oversample',
metavar='DPI',
type=numeric(int, 0, 5000),
default=0,
help="Oversample images to at least the specified DPI, to improve OCR "
"results slightly")
"results slightly",
)
preprocessing.add_argument(
'--remove-vectors', action='store_true',
'--remove-vectors',
action='store_true',
help="EXPERIMENTAL. Mask out any vector objects in the PDF so that they "
"will not be included in OCR. This can eliminate false characters.")
"will not be included in OCR. This can eliminate false characters.",
)
preprocessing.add_argument(
'--mask-barcodes', action='store_true',
'--mask-barcodes',
action='store_true',
help="EXPERIMENTAL. Mask out any barcodes that appear in the PDF so they are not "
"considered during OCR. Barcodes can introduce false characters into "
"OCR.")
"considered during OCR. Barcodes can introduce false characters into "
"OCR.",
)
preprocessing.add_argument(
'--threshold', action='store_true',
'--threshold',
action='store_true',
help="EXPERIMENTAL. Threshold image to 1bpp before sending it to Tesseract for OCR. Can "
"improve OCR quality compared to Tesseract's thresholder.")
"improve OCR quality compared to Tesseract's thresholder.",
)
ocrsettings = parser.add_argument_group(
"OCR options",
"Control how OCR is applied")
ocrsettings = parser.add_argument_group("OCR options", "Control how OCR is applied")
ocrsettings.add_argument(
'-f', '--force-ocr', action='store_true',
'-f',
'--force-ocr',
action='store_true',
help="Rasterize any text or vector objects on each page, apply OCR, and "
"save the rastered output (this rewrites the PDF)")
"save the rastered output (this rewrites the PDF)",
)
ocrsettings.add_argument(
'-s', '--skip-text', action='store_true',
'-s',
'--skip-text',
action='store_true',
help="Skip OCR on any pages that already contain text, but include the "
"page in final output; useful for PDFs that contain a mix of "
"images, text pages, and/or previously OCRed pages")
"page in final output; useful for PDFs that contain a mix of "
"images, text pages, and/or previously OCRed pages",
)
ocrsettings.add_argument(
'--redo-ocr', action='store_true',
'--redo-ocr',
action='store_true',
help="Attempt to detect and remove the hidden OCR layer from files that "
"were previously OCRed with OCRmyPDF or another program. Apply OCR "
"to text found in raster images. Existing visible text objects will "
"not be changed. If there is no existing OCR, OCR will be added.")
"were previously OCRed with OCRmyPDF or another program. Apply OCR "
"to text found in raster images. Existing visible text objects will "
"not be changed. If there is no existing OCR, OCR will be added.",
)
ocrsettings.add_argument(
'--skip-big', type=numeric(float, 0, 5000), metavar='MPixels',
'--skip-big',
type=numeric(float, 0, 5000),
metavar='MPixels',
help="Skip OCR on pages larger than the specified amount of megapixels, "
"but include skipped pages in final output")
"but include skipped pages in final output",
)
optimizing = parser.add_argument_group(
"Optimization options",
"Control how the PDF is optimized after OCR"
"Optimization options", "Control how the PDF is optimized after OCR"
)
optimizing.add_argument(
'-O', '--optimize', type=int, choices=range(0, 4), default=1,
help=("Control how PDF is optimized after processing:"
'-O',
'--optimize',
type=int,
choices=range(0, 4),
default=1,
help=(
"Control how PDF is optimized after processing:"
"0 - do not optimize; "
"1 - do safe, lossless optimizations (default); "
"2 - do some lossy optimizations; "
"3 - do aggressive lossy optimizations (including lossy JBIG2)"
)
),
)
optimizing.add_argument(
'--jpeg-quality', type=numeric(int, 0, 100), default=0, metavar='Q',
help=("Adjust JPEG quality level for JPEG optimization. "
"100 is best quality and largest output size; "
"1 is lowest quality and smallest output; "
"0 uses the default."
)
'--jpeg-quality',
type=numeric(int, 0, 100),
default=0,
metavar='Q',
help=(
"Adjust JPEG quality level for JPEG optimization. "
"100 is best quality and largest output size; "
"1 is lowest quality and smallest output; "
"0 uses the default."
),
)
optimizing.add_argument(
'--jpg-quality', type=numeric(int, 0, 100), default=0, metavar='Q',
'--jpg-quality',
type=numeric(int, 0, 100),
default=0,
metavar='Q',
dest='jpeg_quality',
help=argparse.SUPPRESS # Alias for --jpeg-quality
help=argparse.SUPPRESS, # Alias for --jpeg-quality
)
optimizing.add_argument(
'--png-quality', type=numeric(int, 0, 100), default=0, metavar='Q',
help=("Adjust PNG quality level to use when quantizing PNGs. "
"Values have same meaning as with --jpeg-quality"
)
'--png-quality',
type=numeric(int, 0, 100),
default=0,
metavar='Q',
help=(
"Adjust PNG quality level to use when quantizing PNGs. "
"Values have same meaning as with --jpeg-quality"
),
)
optimizing.add_argument(
'--jbig2-lossy', action='store_true',
help=("Enable JBIG2 lossy mode (better compression, not suitable for some "
"use cases - see documentation)."
)
'--jbig2-lossy',
action='store_true',
help=(
"Enable JBIG2 lossy mode (better compression, not suitable for some "
"use cases - see documentation)."
),
)
optimizing.add_argument(
'--jbig2-page-group-size', type=numeric(int, 1, 10000), default=0,
'--jbig2-page-group-size',
type=numeric(int, 1, 10000),
default=0,
metavar='N',
# Adjust number of pages to consider at once for JBIG2 compression
help=argparse.SUPPRESS
help=argparse.SUPPRESS,
)
advanced = parser.add_argument_group(
"Advanced",
"Advanced options to control Tesseract's OCR behavior")
"Advanced", "Advanced options to control Tesseract's OCR behavior"
)
advanced.add_argument(
'--max-image-mpixels', action='store', type=numeric(float, 0),
'--max-image-mpixels',
action='store',
type=numeric(float, 0),
metavar='MPixels',
help="Set maximum number of pixels to unpack before treating an image as a "
"decompression bomb",
default=128.0)
"decompression bomb",
default=128.0,
)
advanced.add_argument(
'--tesseract-config', action='append', metavar='CFG', default=[],
help="Additional Tesseract configuration files -- see documentation")
'--tesseract-config',
action='append',
metavar='CFG',
default=[],
help="Additional Tesseract configuration files -- see documentation",
)
advanced.add_argument(
'--tesseract-pagesegmode', action='store', type=int, metavar='PSM',
'--tesseract-pagesegmode',
action='store',
type=int,
metavar='PSM',
choices=range(0, 14),
help="Set Tesseract page segmentation mode (see tesseract --help)")
help="Set Tesseract page segmentation mode (see tesseract --help)",
)
advanced.add_argument(
'--tesseract-oem', action='store', type=int, metavar='MODE',
'--tesseract-oem',
action='store',
type=int,
metavar='MODE',
choices=range(0, 4),
help=("Set Tesseract 4.0 OCR engine mode: "
"0 - original Tesseract only; "
"1 - neural nets LSTM only; "
"2 - Tesseract + LSTM; "
"3 - default.")
)
help=(
"Set Tesseract 4.0 OCR engine mode: "
"0 - original Tesseract only; "
"1 - neural nets LSTM only; "
"2 - Tesseract + LSTM; "
"3 - default."
),
)
advanced.add_argument(
'--pdf-renderer',
choices=['auto', 'hocr', 'sandwich'], default='auto',
choices=['auto', 'hocr', 'sandwich'],
default='auto',
help="Choose OCR PDF renderer - the default option is to let OCRmyPDF "
"choose. See documentation for discussion."
)
"choose. See documentation for discussion.",
)
advanced.add_argument(
'--tesseract-timeout', default=180.0, type=numeric(float, 0),
'--tesseract-timeout',
default=180.0,
type=numeric(float, 0),
metavar='SECONDS',
help='Give up on OCR after the timeout, but copy the preprocessed page '
'into the final output')
'into the final output',
)
advanced.add_argument(
'--rotate-pages-threshold', default=14.0, type=numeric(float, 0, 1000),
'--rotate-pages-threshold',
default=14.0,
type=numeric(float, 0, 1000),
metavar='CONFIDENCE',
help="Only rotate pages when confidence is above this value (arbitrary "
"units reported by tesseract)")
"units reported by tesseract)",
)
advanced.add_argument(
'--pdfa-image-compression', choices=['auto', 'jpeg', 'lossless'],
'--pdfa-image-compression',
choices=['auto', 'jpeg', 'lossless'],
default='auto',
help="Specify how to compress images in the output PDF/A. 'auto' lets "
"OCRmyPDF decide. 'jpeg' changes all grayscale and color images to "
"JPEG compression. 'lossless' uses PNG-style lossless compression "
"for all images. Monochrome images are always compressed using a "
"lossless codec. Compression settings "
"are applied to all pages, including those for which OCR was "
"skipped. Not supported for --output-type=pdf ; that setting "
"preserves the original compression of all images.")
"OCRmyPDF decide. 'jpeg' changes all grayscale and color images to "
"JPEG compression. 'lossless' uses PNG-style lossless compression "
"for all images. Monochrome images are always compressed using a "
"lossless codec. Compression settings "
"are applied to all pages, including those for which OCR was "
"skipped. Not supported for --output-type=pdf ; that setting "
"preserves the original compression of all images.",
)
advanced.add_argument(
'--user-words', metavar='FILE',
'--user-words',
metavar='FILE',
help="Specify the location of the Tesseract user words file. This is a "
"list of words Tesseract should consider while performing OCR in "
"addition to its standard language dictionaries. This can improve "
"OCR quality especially for specialized and technical documents.")
"list of words Tesseract should consider while performing OCR in "
"addition to its standard language dictionaries. This can improve "
"OCR quality especially for specialized and technical documents.",
)
advanced.add_argument(
'--user-patterns', metavar='FILE',
help="Specify the location of the Tesseract user patterns file.")
'--user-patterns',
metavar='FILE',
help="Specify the location of the Tesseract user patterns file.",
)
debugging = parser.add_argument_group(
"Debugging",
"Arguments to help with troubleshooting and debugging")
"Debugging", "Arguments to help with troubleshooting and debugging"
)
debugging.add_argument(
'-k', '--keep-temporary-files', action='store_true',
help="Keep temporary files (helpful for debugging)")
'-k',
'--keep-temporary-files',
action='store_true',
help="Keep temporary files (helpful for debugging)",
)
debugging.add_argument(
'--flowchart', type=str,
help="Generate the pipeline execution flowchart")
'--flowchart', type=str, help="Generate the pipeline execution flowchart"
)
def check_options_languages(options, _log):
@@ -409,8 +522,9 @@ def check_options_languages(options, _log):
if not languages.issubset(tesseract.languages()):
msg = (
"The installed version of tesseract does not have language "
"data for the following requested languages: \n")
for lang in (languages - tesseract.languages()):
"data for the following requested languages: \n"
)
for lang in languages - tesseract.languages():
msg += lang + '\n'
raise MissingDependencyError(msg)
@@ -427,18 +541,18 @@ def check_options_output(options, log):
msg = (
"The 'hocr' PDF renderer is known to cause problems with one "
"or more of the languages in your document. Use "
"--pdf-renderer auto (the default) to avoid this issue.")
"--pdf-renderer auto (the default) to avoid this issue."
)
log.warning(msg)
if ghostscript.version() < '9.20' \
and options.output_type != 'pdf' \
and not is_latin:
if ghostscript.version() < '9.20' and options.output_type != 'pdf' and not is_latin:
# https://bugs.ghostscript.com/show_bug.cgi?id=696874
# Ghostscript < 9.20 fails to encode multibyte characters properly
msg = (
"The installed version of Ghostscript does not work correctly "
"with the OCR languages you specified. Use --output-type pdf or "
"upgrade to Ghostscript 9.20 or later to avoid this issue.")
"upgrade to Ghostscript 9.20 or later to avoid this issue."
)
msg += "Found Ghostscript {}".format(ghostscript.version())
log.warning(msg)
@@ -455,8 +569,14 @@ def check_options_output(options, log):
)
lossless_reconstruction = False
if not any((options.deskew, options.clean_final, options.force_ocr,
options.remove_background)):
if not any(
(
options.deskew,
options.clean_final,
options.force_ocr,
options.remove_background,
)
):
lossless_reconstruction = True
options.lossless_reconstruction = lossless_reconstruction
@@ -464,7 +584,7 @@ def check_options_output(options, log):
raise argparse.ArgumentError(
None,
"--redo-ocr is not currently compatible with --deskew, "
"--clean-final, and --remove-background"
"--clean-final, and --remove-background",
)
@@ -473,8 +593,8 @@ def check_options_sidecar(options, log):
if options.output_file == '-':
raise argparse.ArgumentError(
None,
"--sidecar filename must be specified when output file is "
"stdout.")
"--sidecar filename must be specified when output file is " "stdout.",
)
options.sidecar = options.output_file + '.txt'
@@ -483,10 +603,12 @@ def _optional_program_required(name, version_fn, min_version, for_argument):
if version_fn() < min_version:
raise MissingDependencyError(
"The installed '{}' is not supported. "
"Install version {} or newer.".format(name, min_version))
"Install version {} or newer.".format(name, min_version)
)
except (FileNotFoundError, MissingDependencyError):
raise MissingDependencyError(
"Install the '{}' program to use {}.".format(name, for_argument))
"Install the '{}' program to use {}.".format(name, for_argument)
)
def _optional_program_recommended(name, version_fn, min_version, for_argument):
@@ -494,7 +616,8 @@ def _optional_program_recommended(name, version_fn, min_version, for_argument):
if version_fn() < min_version:
raise MissingDependencyError(
"The installed '{}' is not supported. "
"Install version {} or newer.".format(name, min_version))
"Install version {} or newer.".format(name, min_version)
)
except (FileNotFoundError, MissingDependencyError):
complain(
"For best results, install the optional program '{}' to use the "
@@ -505,6 +628,7 @@ def _optional_program_recommended(name, version_fn, min_version, for_argument):
def check_options_preprocessing(options, log):
if any((options.clean, options.clean_final)):
from .exec import unpaper
_optional_program_required(
'unpaper', unpaper.version, '6.1', '--clean, --clean-final'
)
@@ -512,27 +636,27 @@ def check_options_preprocessing(options, log):
def check_options_ocr_behavior(options, log):
exclusive_options = sum(
[(1 if opt else 0)
for opt in (options.force_ocr, options.skip_text, options.redo_ocr)
[
(1 if opt else 0)
for opt in (options.force_ocr, options.skip_text, options.redo_ocr)
]
)
if exclusive_options >= 2:
raise argparse.ArgumentError(
None,
"Error: choose only one of --force-ocr, --skip-text, --redo-ocr.")
None, "Error: choose only one of --force-ocr, --skip-text, --redo-ocr."
)
def check_options_optimizing(options, log):
if options.optimize >= 2:
from .exec import pngquant, jbig2enc
_optional_program_required(
'pngquant', pngquant.version, '2.0.1', '--optimize {2,3}'
)
if options.jbig2_lossy:
_optional_program_required(
'jbig2', jbig2enc.version, '0.28', '--jbig2-lossy'
)
_optional_program_required('jbig2', jbig2enc.version, '0.28', '--jbig2-lossy')
elif options.optimize >= 2:
# Although we use JBIG2 for optimize=1, don't nag about it unless the
# user is asking for more optimization
@@ -540,9 +664,9 @@ def check_options_optimizing(options, log):
'jbig2', jbig2enc.version, '0.28', '--optimize {2,3}'
)
if options.optimize == 0 and any([
options.jbig2_lossy, options.png_quality, options.jpeg_quality
]):
if options.optimize == 0 and any(
[options.jbig2_lossy, options.png_quality, options.jpeg_quality]
):
log.warning(
"The arguments --jbig2-lossy, --png-quality, and --jpeg-quality "
"will be ignored because --optimize=0."
@@ -551,24 +675,23 @@ def check_options_optimizing(options, log):
def check_options_advanced(options, log):
if options.tesseract_oem and not tesseract.v4():
log.warning(
"--tesseract-oem requires Tesseract 4.x -- argument ignored")
if options.pdfa_image_compression != 'auto' and \
options.output_type.startswith('pdfa'):
log.warning("--tesseract-oem requires Tesseract 4.x -- argument ignored")
if options.pdfa_image_compression != 'auto' and options.output_type.startswith(
'pdfa'
):
log.warning(
"--pdfa-image-compression argument has no effect when "
"--output-type is not 'pdfa', 'pdfa-1', or 'pdfa-2'"
)
if tesseract.v4() and (options.user_words or options.user_patterns):
log.warning(
'Tesseract 4.x ignores --user-words, so this has no effect')
log.warning('Tesseract 4.x ignores --user-words, so this has no effect')
def check_options_metadata(options, log):
import unicodedata
docinfo = [options.title, options.author, options.keywords,
options.subject]
docinfo = [options.title, options.author, options.keywords, options.subject]
for s in (m for m in docinfo if m):
for c in s:
if unicodedata.category(c) == 'Co' or ord(c) >= 0x10000:
@@ -576,11 +699,12 @@ def check_options_metadata(options, log):
"One of the metadata strings contains "
"an unsupported Unicode character: '{}' (U+{})".format(
c, hex(ord(c))[2:].upper()
))
)
)
def check_options_pillow(options, log):
PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1000000)
PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1_000_000)
if PIL.Image.MAX_IMAGE_PIXELS == 0:
PIL.Image.MAX_IMAGE_PIXELS = None
@@ -686,8 +810,10 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
log.error(exc_msg)
elif exc_name == 'PIL.Image.DecompressionBombError':
msg = cleanup_ruffus_error_message(exc_value)
msg += ("\nUse the --max-image-mpixels argument to set increase the "
"maximum number of megapixels to accept.")
msg += (
"\nUse the --max-image-mpixels argument to set increase the "
"maximum number of megapixels to accept."
)
log.error(msg)
exit_code = ExitCode.input_file
@@ -753,8 +879,7 @@ def check_closed_streams(options):
if sys.stdin is None:
if options.input_file == '-':
print("Trying to read from stdin but stdin seems closed",
file=sys.stderr)
print("Trying to read from stdin but stdin seems closed", file=sys.stderr)
return False
sys.stdin = open(os.devnull, 'r')
@@ -763,10 +888,15 @@ def check_closed_streams(options):
# Can't replace stdout if the user is piping
# If this case can even happen, it must be some kind of weird
# stream.
print(textwrap.dedent("""\
print(
textwrap.dedent(
"""\
Output was set to stdout '-' but the stream attached to
stdout does not support the flush() system call. This
will fail."""), file=sys.stderr)
will fail."""
),
file=sys.stderr,
)
return False
sys.stdout = open(os.devnull, 'w')
@@ -774,15 +904,12 @@ def check_closed_streams(options):
def log_page_orientations(pdfinfo, _log):
direction = {0: 'n', 90: 'e',
180: 's', 270: 'w'}
direction = {0: 'n', 90: 'e', 180: 's', 270: 'w'}
orientations = []
for n, page in enumerate(pdfinfo):
angle = page.rotation or 0
if angle != 0:
orientations.append('{0}{1}'.format(
n + 1,
direction.get(angle, '')))
orientations.append('{0}{1}'.format(n + 1, direction.get(angle, '')))
if orientations:
_log.info('Page orientations detected: ' + ' '.join(orientations))
@@ -799,12 +926,19 @@ def check_environ(options, _log):
'OCRMYPDF_TESSERACT',
'OCRMYPDF_QPDF',
'OCRMYPDF_GS',
'OCRMYPDF_UNPAPER')
'OCRMYPDF_UNPAPER',
)
for k in old_envvars:
if k in os.environ:
_log.warning(textwrap.dedent("""\
_log.warning(
textwrap.dedent(
"""\
OCRmyPDF no longer uses the environment variable {}.
Change PATH to select alternate programs.""".format(k)))
Change PATH to select alternate programs.""".format(
k
)
)
)
def check_input_file(options, _log, start_input_file):
@@ -813,6 +947,7 @@ def check_input_file(options, _log, start_input_file):
_log.info('reading file from standard input')
with open(start_input_file, 'wb') as stream_buffer:
from shutil import copyfileobj
copyfileobj(sys.stdin.buffer, stream_buffer)
else:
try:
@@ -825,15 +960,22 @@ def check_input_file(options, _log, start_input_file):
def check_requested_output_file(options, _log):
if options.output_file == '-':
if sys.stdout.isatty():
_log.error(textwrap.dedent("""\
_log.error(
textwrap.dedent(
"""\
Output was set to stdout '-' but it looks like stdout
is connected to a terminal. Please redirect stdout to a
file."""))
file."""
)
)
raise BadArgsError()
elif not is_file_writable(options.output_file):
_log.error(
"Output file location (" + options.output_file + ") " +
"is not a writable file.")
"Output file location ("
+ options.output_file
+ ") "
+ "is not a writable file."
)
raise OutputFileAccessError()
@@ -853,27 +995,33 @@ def report_output_file_size(options, _log, input_file, output_file):
'clean_final',
'remove_background',
'oversample',
'force_ocr'
'force_ocr',
}
for arg in image_preproc:
attr = getattr(options, arg, None)
if not attr:
continue
reasons.append(
"The argument --{} was issued, causing transcoding.".format(
arg.replace('_', '-')))
"The argument --{} was issued, causing transcoding.".format(
arg.replace('_', '-')
)
)
if reasons:
explanation = (
"Possible reasons for this include:\n" + '\n'.join(reasons) + "\n")
explanation = "Possible reasons for this include:\n" + '\n'.join(reasons) + "\n"
else:
explanation = (
"No reason for this increase is known. Please report this issue.")
explanation = "No reason for this increase is known. Please report this issue."
_log.warning(textwrap.dedent("""\
_log.warning(
textwrap.dedent(
"""\
The output file size is {:.2f}× larger than the input file.
{}
""".format(ratio, explanation)))
""".format(
ratio, explanation
)
)
)
def run_pipeline(args=None):
@@ -888,7 +1036,8 @@ def run_pipeline(args=None):
logger_args = {'verbose': options.verbose, 'quiet': options.quiet}
_log, _log_mutex = proxy_logger.make_shared_logger_and_proxy(
logging_factory, __name__, logger_args)
logging_factory, __name__, logger_args
)
preamble(_log)
check_options(options, _log)
@@ -899,7 +1048,8 @@ def run_pipeline(args=None):
complain(
"You are using qpdf version {0} which has known issues including "
"security vulnerabilities with certain malformed PDFs. Consider "
"upgrading to version 7.0.0 or newer.".format(qpdf.version()))
"upgrading to version 7.0.0 or newer.".format(qpdf.version())
)
if ghostscript.version() == '9.24':
complain(
@@ -927,10 +1077,8 @@ def run_pipeline(args=None):
try:
work_folder = mkdtemp(prefix="com.github.ocrmypdf.")
options.history_file = os.path.join(
work_folder, 'ruffus_history.sqlite')
start_input_file = os.path.join(
work_folder, 'origin')
options.history_file = os.path.join(work_folder, 'ruffus_history.sqlite')
start_input_file = os.path.join(work_folder, 'origin')
check_input_file(options, _log, start_input_file)
check_requested_output_file(options, _log)
@@ -983,12 +1131,12 @@ def run_pipeline(args=None):
_log.warning('Output file: The generated PDF is INVALID')
return ExitCode.invalid_output_pdf
report_output_file_size(options, _log, start_input_file,
options.output_file)
report_output_file_size(options, _log, start_input_file, options.output_file)
pdfinfo = context.get_pdfinfo()
if options.verbose:
from pprint import pformat
_log.debug(pformat(pdfinfo))
log_page_orientations(pdfinfo, _log)
src/ocrmypdf/_jobcontext.py
+4 -2
View File
@@ -77,8 +77,10 @@ class JobContextManager(SyncManager):
def cleanup_working_files(work_folder, options):
if options.keep_temporary_files:
print("Temporary working files saved at:\n{0}".format(work_folder),
file=sys.stderr)
print(
"Temporary working files saved at:\n{0}".format(work_folder),
file=sys.stderr,
)
else:
with suppress(FileNotFoundError):
shutil.rmtree(work_folder)
src/ocrmypdf/_pipeline.py
+253 -273
View File
File diff suppressed because it is too large Load Diff
src/ocrmypdf/_unicodefun.py
+10 -5
View File
@@ -47,6 +47,7 @@ def verify_python3_env(): # pragma: no cover
try:
import locale
fs_enc = codecs.lookup(locale.getpreferredencoding()).name
except Exception:
fs_enc = 'ascii'
@@ -56,8 +57,10 @@ def verify_python3_env(): # pragma: no cover
extra = ''
if os.name == 'posix':
import subprocess
rv = subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()[0]
rv = subprocess.Popen(
['locale', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE
).communicate()[0]
good_locales = set()
has_c_utf8 = False
@@ -108,6 +111,8 @@ def verify_python3_env(): # pragma: no cover
'is not supported'
) % bad_locale
raise RuntimeError('ocrmypdf will abort further execution because Python 3 '
'was configured to use ASCII as encoding for the '
'environment.' + extra)
raise RuntimeError(
'ocrmypdf will abort further execution because Python 3 '
'was configured to use ASCII as encoding for the '
'environment.' + extra
)
src/ocrmypdf/_weave.py
+30 -35
View File
@@ -89,8 +89,8 @@ def strip_invisible_text(pdf, page, log):
def _weave_layers_graft(
*, pdf_base, page_num, text, font, font_key, procset, rotation,
strip_old_text, log):
*, pdf_base, page_num, text, font, font_key, procset, rotation, strip_old_text, log
):
"""Insert the text layer from text page 0 on to pdf_base at page_num"""
log.debug("Grafting")
@@ -108,7 +108,7 @@ def _weave_layers_graft(
stream = bytearray(pdf_text_contents)
pattern = b'/Im1 Do'
idx = stream.find(pattern)
stream[idx:(idx + len(pattern))] = b' ' * len(pattern)
stream[idx : (idx + len(pattern))] = b' ' * len(pattern)
pdf_text_contents = bytes(stream)
base_page = pdf_base.pages.p(page_num)
@@ -117,12 +117,10 @@ def _weave_layers_graft(
# content may have a rotation applied. Wrap the text stream with a rotation
# so it will be oriented the same way as the rest of the page content.
# (Previous versions OCRmyPDF rotated the content layer to match the text.)
mediabox = [float(pdf_text.pages[0].MediaBox[v])
for v in range(4)]
mediabox = [float(pdf_text.pages[0].MediaBox[v]) for v in range(4)]
wt, ht = mediabox[2] - mediabox[0], mediabox[3] - mediabox[1]
mediabox = [float(base_page.MediaBox[v])
for v in range(4)]
mediabox = [float(base_page.MediaBox[v]) for v in range(4)]
wp, hp = mediabox[2] - mediabox[0], mediabox[3] - mediabox[1]
translate = pikepdf.PdfMatrix().translated(-wt / 2, -ht / 2)
@@ -147,11 +145,7 @@ def _weave_layers_graft(
# for a size different between initial and text PDF, then untranslate
ctm = translate @ rotate @ scale @ untranslate
pdf_text_contents = (
b'q %s cm\n' % ctm.encode() +
pdf_text_contents +
b'\nQ\n'
)
pdf_text_contents = b'q %s cm\n' % ctm.encode() + pdf_text_contents + b'\nQ\n'
new_text_layer = pikepdf.Stream(pdf_base, pdf_text_contents)
@@ -254,8 +248,7 @@ def _fix_toc(pdf_base, pageref_remap, log):
if not isinstance(dest_node, pikepdf.Array):
return
pageref = dest_node[0]
if pageref['/Type'] == '/Page' and \
pageref.objgen in pageref_remap:
if pageref['/Type'] == '/Page' and pageref.objgen in pageref_remap:
new_objgen = pageref_remap[pageref.objgen]
dest_node[0] = pdf_base.get_object(new_objgen)
@@ -278,11 +271,7 @@ def _fix_toc(pdf_base, pageref_remap, log):
_traverse_toc(pdf_base, visit_remap_dest, log)
def weave_layers(
infiles,
output_file,
log,
context):
def weave_layers(infiles, output_file, log, context):
"""Apply text layer and/or image layer changes to baseline file
This is where the magic happens. infiles will be the main PDF to modify,
@@ -313,6 +302,7 @@ def weave_layers(
return page_number(key)
except ValueError:
return -1
flat_inputs = sorted(flatten_groups(infiles), key=input_sorter)
groups = groupby(flat_inputs, key=input_sorter)
@@ -333,7 +323,8 @@ def weave_layers(
_traverse_toc(pdf_base, None, log)
procset = pdf_base.make_indirect(
pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]'))
pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]')
)
# Iterate rest
for page_num, layers in groups:
@@ -341,12 +332,8 @@ def weave_layers(
log.debug(page_num)
log.debug(layers)
text = next(
(ii for ii in layers if ii.endswith('.text.pdf')), None
)
image = next(
(ii for ii in layers if ii.endswith('.image-layer.pdf')), None
)
text = next((ii for ii in layers if ii.endswith('.text.pdf')), None)
image = next((ii for ii in layers if ii.endswith('.image-layer.pdf')), None)
if text and not font:
font, font_key = _find_font(text, pdf_base)
@@ -378,23 +365,30 @@ def weave_layers(
content_rotation = autorotate_correction
text_rotation = autorotate_correction
text_misaligned = (text_rotation - content_rotation) % 360
log.debug('%r', [
text_rotation, autorotate_correction, text_misaligned,
content_rotation]
log.debug(
'%r',
[text_rotation, autorotate_correction, text_misaligned, content_rotation],
)
if text and font:
# Graft the text layer onto this page, whether new or old
strip_old = context.get_options().redo_ocr
_weave_layers_graft(
pdf_base=pdf_base, page_num=page_num, text=text, font=font,
font_key=font_key, rotation=text_misaligned, procset=procset,
strip_old_text=strip_old, log=log
pdf_base=pdf_base,
page_num=page_num,
text=text,
font=font,
font_key=font_key,
rotation=text_misaligned,
procset=procset,
strip_old_text=strip_old,
log=log,
)
# Correct the rotation if applicable
pdf_base.pages[page_num - 1].Rotate = \
(content_rotation - autorotate_correction) % 360
pdf_base.pages[page_num - 1].Rotate = (
content_rotation - autorotate_correction
) % 360
if len(keep_open) > 100:
# qpdf limitations require us to keep files open when we intend
@@ -404,7 +398,8 @@ def weave_layers(
# even if page 1 doesn't use it, so we have a way to get it back.
page0 = pdf_base.pages[0]
_update_page_resources(
page=page0, font=font, font_key=font_key, procset=procset)
page=page0, font=font, font_key=font_key, procset=procset
)
interim = output_file + '_working{}.pdf'.format(page_num)
pdf_base.save(interim)
del pdf_base
src/ocrmypdf/exceptions.py
+10 -4
View File
@@ -19,6 +19,7 @@
from enum import IntEnum
from textwrap import dedent
class ExitCode(IntEnum):
ok = 0
bad_args = 1
@@ -52,7 +53,8 @@ class BadArgsError(ExitCodeException):
class PdfMergeFailedError(ExitCodeException):
exit_code = ExitCode.input_file
message = dedent('''\
message = dedent(
'''\
Failed to merge PDF image layer with OCR layer
Usually this happens because the input PDF file is malformed and
@@ -60,7 +62,9 @@ class PdfMergeFailedError(ExitCodeException):
Try using
ocrmypdf --pdf-renderer sandwich [..other args..]
''')
'''
)
class MissingDependencyError(ExitCodeException):
exit_code = ExitCode.missing_dependency
@@ -92,7 +96,8 @@ class SubprocessOutputError(ExitCodeException):
class EncryptedPdfError(ExitCodeException):
exit_code = ExitCode.encrypted_pdf
message = dedent('''\
message = dedent(
'''\
Input PDF is encrypted. The encryption must be removed to
perform OCR.
@@ -101,7 +106,8 @@ class EncryptedPdfError(ExitCodeException):
You can remove the encryption using
qpdf --decrypt [--password=[password]] infilename
''')
'''
)
class TesseractConfigError(ExitCodeException):
src/ocrmypdf/exec/__init__.py
+19 -15
View File
@@ -24,36 +24,40 @@ from subprocess import run, STDOUT, PIPE, CalledProcessError
from ..exceptions import MissingDependencyError
def get_version(program, *,
version_arg='--version', regex=r'(\d+(\.\d+)*)'):
def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
"Get the version of the specified program"
args_prog = [
program,
version_arg
]
args_prog = [program, version_arg]
try:
proc = run(
args_prog, close_fds=True, universal_newlines=True,
stdout=PIPE, stderr=STDOUT, check=True)
args_prog,
close_fds=True,
universal_newlines=True,
stdout=PIPE,
stderr=STDOUT,
check=True,
)
output = proc.stdout
except FileNotFoundError as e:
raise MissingDependencyError(
"Could not find program '{}' on the PATH".format(
program)) from e
"Could not find program '{}' on the PATH".format(program)
) from e
except CalledProcessError as e:
if e.returncode < 0:
raise MissingDependencyError(
"Ran program '{}' but it exited with an error:\n{}".format(
program, e.output)) from e
program, e.output
)
) from e
raise MissingDependencyError(
"Could not find program '{}' on the PATH".format(
program)) from e
"Could not find program '{}' on the PATH".format(program)
) from e
try:
version = re.match(regex, output.strip()).group(1)
except AttributeError as e:
raise MissingDependencyError(
("The program '{}' did not report its version. "
"Message was:\n{}").format(program, output)
("The program '{}' did not report its version. " "Message was:\n{}").format(
program, output
)
)
return version
src/ocrmypdf/exec/ghostscript.py
+88 -63
View File
@@ -69,39 +69,46 @@ def extract_text(input_file, pageno=1):
"""
if pageno is not None:
pages = [
'-dFirstPage=%i' % pageno,
'-dLastPage=%i' % pageno
]
pages = ['-dFirstPage=%i' % pageno, '-dLastPage=%i' % pageno]
else:
pages = []
args_gs = [
'gs',
'-dQUIET',
'-dSAFER',
'-dBATCH',
'-dNOPAUSE',
'-sDEVICE=txtwrite',
'-dTextFormat=0',
] + pages + [
'-o', '-',
fspath(input_file)
]
args_gs = (
[
'gs',
'-dQUIET',
'-dSAFER',
'-dBATCH',
'-dNOPAUSE',
'-sDEVICE=txtwrite',
'-dTextFormat=0',
]
+ pages
+ ['-o', '-', fspath(input_file)]
)
p = run(args_gs, stdout=PIPE, stderr=PIPE)
if p.returncode != 0:
raise SubprocessOutputError(
'Ghostscript text extraction failed\n%s\n%s\n%s' % (
input_file, p.stdout.decode(), p.stderr.decode()
)
'Ghostscript text extraction failed\n%s\n%s\n%s'
% (input_file, p.stdout.decode(), p.stderr.decode())
)
return p.stdout
def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
pageno=1, page_dpi=None, rotation=None, filter_vector=False):
def rasterize_pdf(
input_file,
output_file,
xres,
yres,
raster_device,
log,
pageno=1,
page_dpi=None,
rotation=None,
filter_vector=False,
):
"""Rasterize one page of a PDF at resolution (xres, yres) in canvas units.
The image is sized to match the integer pixels dimensions implied by
@@ -126,26 +133,30 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
page_dpi = res
with NamedTemporaryFile(delete=True) as tmp:
args_gs = [
'gs',
'-dQUIET',
'-dSAFER',
'-dBATCH',
'-dNOPAUSE',
'-sDEVICE=%s' % raster_device,
'-dFirstPage=%i' % pageno,
'-dLastPage=%i' % pageno,
'-r{0}x{1}'.format(str(int_res[0]), str(int_res[1])),
] + (['-dFILTERVECTOR'] if filter_vector else []) + [
'-o', tmp.name,
'-dAutoRotatePages=/None', # Probably has no effect on raster
'-f',
fspath(input_file)
]
args_gs = (
[
'gs',
'-dQUIET',
'-dSAFER',
'-dBATCH',
'-dNOPAUSE',
'-sDEVICE=%s' % raster_device,
'-dFirstPage=%i' % pageno,
'-dLastPage=%i' % pageno,
'-r{0}x{1}'.format(str(int_res[0]), str(int_res[1])),
]
+ (['-dFILTERVECTOR'] if filter_vector else [])
+ [
'-o',
tmp.name,
'-dAutoRotatePages=/None', # Probably has no effect on raster
'-f',
fspath(input_file),
]
)
log.debug(args_gs)
p = run(args_gs, stdout=PIPE, stderr=STDOUT,
universal_newlines=True)
p = run(args_gs, stdout=PIPE, stderr=STDOUT, universal_newlines=True)
if _gs_error_reported(p.stdout):
log.error(p.stdout)
else:
@@ -162,12 +173,16 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
tmp.seek(0)
with Image.open(tmp) as im:
expected_size = round(im.size[0] / int_res[0] * res[0]), \
round(im.size[1] / int_res[1] * res[1])
expected_size = (
round(im.size[0] / int_res[0] * res[0]),
round(im.size[1] / int_res[1] * res[1]),
)
if expected_size != im.size or page_dpi != (xres, yres):
log.debug(
"Ghostscript: resize output image {} -> {}".format(
im.size, expected_size))
im.size, expected_size
)
)
im = im.resize(expected_size)
if rotation is not None:
@@ -185,8 +200,15 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
im.save(fspath(output_file), dpi=page_dpi)
def generate_pdfa(pdf_pages, output_file, compression, log,
threads=1, pdf_version='1.5', pdfa_part='2'):
def generate_pdfa(
pdf_pages,
output_file,
compression,
log,
threads=1,
pdf_version='1.5',
pdfa_part='2',
):
"""Generate a PDF/A.
The pdf_pages, a list files, will be merged into output_file. One or more
@@ -240,26 +262,29 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
# nb no need to specify ProcessColorModel when ColorConversionStrategy
# is set; see:
# https://bugs.ghostscript.com/show_bug.cgi?id=699392
args_gs = [
"gs",
"-dQUIET",
"-dBATCH",
"-dNOPAUSE",
"-dCompatibilityLevel=" + str(pdf_version),
"-dNumRenderingThreads=" + str(threads),
"-sDEVICE=pdfwrite",
"-dAutoRotatePages=/None",
"-sColorConversionStrategy=" + strategy
] + compression_args + [
"-dJPEGQ=95",
"-dPDFA=" + pdfa_part,
"-dPDFACompatibilityPolicy=1",
"-sOutputFile=" + gs_pdf.name,
]
args_gs = (
[
"gs",
"-dQUIET",
"-dBATCH",
"-dNOPAUSE",
"-dCompatibilityLevel=" + str(pdf_version),
"-dNumRenderingThreads=" + str(threads),
"-sDEVICE=pdfwrite",
"-dAutoRotatePages=/None",
"-sColorConversionStrategy=" + strategy,
]
+ compression_args
+ [
"-dJPEGQ=95",
"-dPDFA=" + pdfa_part,
"-dPDFACompatibilityPolicy=1",
"-sOutputFile=" + gs_pdf.name,
]
)
args_gs.extend(fspath(s) for s in pdf_pages) # Stringify Path objs
log.debug(args_gs)
p = run(args_gs, stdout=PIPE, stderr=STDOUT,
universal_newlines=True)
p = run(args_gs, stdout=PIPE, stderr=STDOUT, universal_newlines=True)
if _gs_error_reported(p.stdout):
log.error(p.stdout)
@@ -270,7 +295,7 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
log.debug(
"Ghostscript had to remove PDF 'overprinting' from the "
"input file to complete PDF/A conversion. "
)
)
else:
log.debug(p.stdout)
src/ocrmypdf/exec/jbig2enc.py
+2 -6
View File
@@ -42,7 +42,7 @@ def convert_group(*, cwd, infiles, out_prefix):
out_prefix,
'-s', # symbol mode (lossy)
# '-r', # refinement mode (lossless symbol mode, currently disabled in
# jbig2)
# jbig2)
'-p',
]
args.extend(infiles)
@@ -52,11 +52,7 @@ def convert_group(*, cwd, infiles, out_prefix):
def convert_single(*, cwd, infile, outfile):
args = [
'jbig2',
'-p',
infile
]
args = ['jbig2', '-p', infile]
with open(outfile, 'wb') as fstdout:
proc = run(args, cwd=cwd, stdout=fstdout, stderr=PIPE)
proc.check_returncode()
src/ocrmypdf/exec/pngquant.py
+5 -3
View File
@@ -40,10 +40,12 @@ def quantize(input_file, output_file, quality_min, quality_max):
'pngquant',
'--force',
'--skip-if-larger',
'--output', output_file,
'--quality', '{}-{}'.format(quality_min, quality_max),
'--output',
output_file,
'--quality',
'{}-{}'.format(quality_min, quality_max),
'--',
input_file
input_file,
]
proc = run(args)
proc.check_returncode()
src/ocrmypdf/exec/qpdf.py
+4 -10
View File
@@ -18,7 +18,7 @@
from subprocess import CalledProcessError, STDOUT, PIPE, run
from functools import lru_cache
from . import get_version
from . import get_version
from os import fspath
@@ -28,22 +28,16 @@ def version():
def check(input_file, log=None):
args_qpdf = [
'qpdf',
'--check',
fspath(input_file)
]
args_qpdf = ['qpdf', '--check', fspath(input_file)]
if log is None:
import logging as log
try:
run(args_qpdf, stderr=STDOUT, stdout=PIPE, universal_newlines=True,
check=True)
run(args_qpdf, stderr=STDOUT, stdout=PIPE, universal_newlines=True, check=True)
except CalledProcessError as e:
if e.returncode == 2:
log.error("%s: not a valid PDF, and could not repair it.",
input_file)
log.error("%s: not a valid PDF, and could not repair it.", input_file)
log.error("Details:")
log.error(e.output)
elif e.returncode == 3:
src/ocrmypdf/exec/tesseract.py
+67 -61
View File
@@ -21,7 +21,14 @@ import shutil
from functools import lru_cache
from collections import namedtuple
from textwrap import dedent
from subprocess import CalledProcessError, TimeoutExpired, check_output, STDOUT, run, PIPE
from subprocess import (
CalledProcessError,
TimeoutExpired,
check_output,
STDOUT,
run,
PIPE,
)
from contextlib import suppress
from os import fspath
@@ -29,9 +36,7 @@ from ..exceptions import MissingDependencyError, TesseractConfigError
from ..helpers import page_number
from . import get_version
OrientationConfidence = namedtuple(
'OrientationConfidence',
('angle', 'confidence'))
OrientationConfidence = namedtuple('OrientationConfidence', ('angle', 'confidence'))
HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
@@ -68,18 +73,12 @@ def has_textonly_pdf():
Available in v4.00.00alpha since January 2017. Best to
parse the parameter list
"""
args_tess = [
'tesseract',
'--print-parameters',
'pdf'
]
args_tess = ['tesseract', '--print-parameters', 'pdf']
params = ''
try:
params = check_output(
args_tess, universal_newlines=True, stderr=STDOUT)
params = check_output(args_tess, universal_newlines=True, stderr=STDOUT)
except CalledProcessError as e:
print("Could not --print-parameters from tesseract",
file=sys.stderr)
print("Could not --print-parameters from tesseract", file=sys.stderr)
raise MissingDependencyError from e
if 'textonly_pdf' in params:
return True
@@ -89,21 +88,19 @@ def has_textonly_pdf():
@lru_cache(maxsize=1)
def languages():
def lang_error(output):
msg = dedent("""Tesseract failed to report available languages.
msg = dedent(
"""Tesseract failed to report available languages.
Output from Tesseract:
-----------
""")
"""
)
msg += output
print(msg, file=sys.stderr)
args_tess = [
'tesseract',
'--list-langs'
]
args_tess = ['tesseract', '--list-langs']
try:
proc = run(
args_tess, universal_newlines=True, stdout=PIPE, stderr=STDOUT,
check=True
args_tess, universal_newlines=True, stdout=PIPE, stderr=STDOUT, check=True
)
output = proc.stdout
except CalledProcessError as e:
@@ -118,9 +115,7 @@ def languages():
def tess_base_args(langs, engine_mode):
args = [
'tesseract',
]
args = ['tesseract']
if langs:
args.extend(['-l', '+'.join(langs)])
if engine_mode is not None and v4():
@@ -130,20 +125,22 @@ def tess_base_args(langs, engine_mode):
def get_orientation(input_file, engine_mode, timeout: float, log):
args_tesseract = tess_base_args(['osd'], engine_mode) + [
'--psm', '0',
'--psm',
'0',
fspath(input_file),
'stdout'
'stdout',
]
try:
stdout = check_output(
args_tesseract, stderr=STDOUT, timeout=timeout)
stdout = check_output(args_tesseract, stderr=STDOUT, timeout=timeout)
except TimeoutExpired:
return OrientationConfidence(angle=0, confidence=0.0)
except CalledProcessError as e:
tesseract_log_output(log, e.output, input_file)
if (b'Too few characters. Skipping this page' in e.output or
b'Image too large' in e.output):
if (
b'Too few characters. Skipping this page' in e.output
or b'Image too large' in e.output
):
return OrientationConfidence(0, 0)
raise e from e
else:
@@ -156,8 +153,8 @@ def get_orientation(input_file, engine_mode, timeout: float, log):
angle = int(osd.get('Orientation in degrees', 0))
oc = OrientationConfidence(
angle=angle,
confidence=float(osd.get('Orientation confidence', 0)))
angle=angle, confidence=float(osd.get('Orientation confidence', 0))
)
return oc
@@ -167,9 +164,12 @@ def tesseract_log_output(log, stdout, input_file):
try:
text = stdout.decode()
except UnicodeDecodeError:
log.error(prefix + "command line output was not utf-8. " +
"This usually means Tesseract's language packs do not match "
"the installed version of Tesseract.")
log.error(
prefix
+ "command line output was not utf-8. "
+ "This usually means Tesseract's language packs do not match "
"the installed version of Tesseract."
)
text = stdout.decode('utf-8', 'backslashreplace')
lines = text.splitlines()
@@ -219,10 +219,18 @@ def _generate_null_hocr(output_hocr, output_sidecar, image):
f.write('[skipped page]')
def generate_hocr(input_file, output_files, language: list, engine_mode,
tessconfig: list,
timeout: float, pagesegmode: int, user_words, user_patterns,
log):
def generate_hocr(
input_file,
output_files,
language: list,
engine_mode,
tessconfig: list,
timeout: float,
pagesegmode: int,
user_words,
user_patterns,
log,
):
output_hocr = next(o for o in output_files if o.endswith('.hocr'))
output_sidecar = next(o for o in output_files if o.endswith('.txt'))
@@ -241,17 +249,10 @@ def generate_hocr(input_file, output_files, language: list, engine_mode,
# Reminder: test suite tesseract spoofers will break after any changes
# to the number of order parameters here
args_tesseract.extend([
input_file,
prefix,
'hocr',
'txt'
] + tessconfig)
args_tesseract.extend([input_file, prefix, 'hocr', 'txt'] + tessconfig)
try:
log.debug(args_tesseract)
stdout = check_output(
args_tesseract, stderr=STDOUT,
timeout=timeout)
stdout = check_output(args_tesseract, stderr=STDOUT, timeout=timeout)
except TimeoutExpired:
# Generate a HOCR file with no recognized text if tesseract times out
# Temporary workaround to hocrTransform not being able to function if
@@ -289,10 +290,22 @@ def use_skip_page(text_only, skip_pdf, output_pdf, output_text):
out.write(b'')
def generate_pdf(*, input_image, skip_pdf=None, output_pdf, output_text,
language: list, engine_mode, text_only: bool,
tessconfig: list, timeout: float, pagesegmode: int,
user_words, user_patterns, log):
def generate_pdf(
*,
input_image,
skip_pdf=None,
output_pdf,
output_text,
language: list,
engine_mode,
text_only: bool,
tessconfig: list,
timeout: float,
pagesegmode: int,
user_words,
user_patterns,
log,
):
'''Use Tesseract to render a PDF.
input_image -- image to analyze
@@ -326,18 +339,11 @@ def generate_pdf(*, input_image, skip_pdf=None, output_pdf, output_text,
# Reminder: test suite tesseract spoofers might break after any changes
# to the number of order parameters here
args_tesseract.extend([
input_image,
prefix,
'pdf',
'txt'
] + tessconfig)
args_tesseract.extend([input_image, prefix, 'pdf', 'txt'] + tessconfig)
try:
log.debug(args_tesseract)
stdout = check_output(
args_tesseract, stderr=STDOUT,
timeout=timeout)
stdout = check_output(args_tesseract, stderr=STDOUT, timeout=timeout)
if os.path.exists(prefix + '.txt'):
shutil.move(prefix + '.txt', output_text)
except TimeoutExpired:
src/ocrmypdf/exec/unpaper.py
+23 -19
View File
@@ -40,11 +40,7 @@ def version():
def run(input_file, output_file, dpi, log, mode_args):
args_unpaper = [
'unpaper',
'-v',
'--dpi', str(dpi)
] + mode_args
args_unpaper = ['unpaper', '-v', '--dpi', str(dpi)] + mode_args
SUFFIXES = {'1': '.pbm', 'L': '.pgm', 'RGB': '.ppm'}
@@ -68,8 +64,9 @@ def run(input_file, output_file, dpi, log, mode_args):
im.close()
raise MissingDependencyError() from e
with NamedTemporaryFile(suffix=suffix) as input_pnm, \
NamedTemporaryFile(suffix=suffix, mode="r+b") as output_pnm:
with NamedTemporaryFile(suffix=suffix) as input_pnm, NamedTemporaryFile(
suffix=suffix, mode="r+b"
) as output_pnm:
im.save(input_pnm, format='PPM')
im.close()
@@ -78,9 +75,8 @@ def run(input_file, output_file, dpi, log, mode_args):
args_unpaper.extend([input_pnm.name, output_pnm.name])
try:
stdout = check_output(
args_unpaper, close_fds=True,
universal_newlines=True, stderr=STDOUT,
)
args_unpaper, close_fds=True, universal_newlines=True, stderr=STDOUT
)
except CalledProcessError as e:
log.debug(e.output)
raise e from e
@@ -91,12 +87,20 @@ def run(input_file, output_file, dpi, log, mode_args):
def clean(input_file, output_file, dpi, log):
run(input_file, output_file, dpi, log, [
'--layout', 'none',
'--mask-scan-size', '100', # don't blank out narrow columns
'--no-border-align', # don't align visible content to borders
'--no-mask-center', # don't center visible content within page
'--no-grayfilter', # don't remove light gray areas
'--no-blackfilter', # don't remove solid black areas
'--no-deskew', # don't deskew
])
run(
input_file,
output_file,
dpi,
log,
[
'--layout',
'none',
'--mask-scan-size',
'100', # don't blank out narrow columns
'--no-border-align', # don't align visible content to borders
'--no-mask-center', # don't center visible content within page
'--no-grayfilter', # don't remove light gray areas
'--no-blackfilter', # don't remove solid black areas
'--no-deskew', # don't deskew
],
)
src/ocrmypdf/helpers.py
+21 -17
View File
@@ -38,32 +38,29 @@ def re_symlink(input_file, soft_link_name, log=None):
# Guard against soft linking to oneself
if input_file == soft_link_name:
prdebug("Warning: No symbolic link made. You are using " +
"the original data directory as the working directory.")
prdebug(
"Warning: No symbolic link made. You are using "
+ "the original data directory as the working directory."
)
return
# Soft link already exists: delete for relink?
if os.path.lexists(soft_link_name):
# do not delete or overwrite real (non-soft link) file
if not os.path.islink(soft_link_name):
raise FileExistsError(
"%s exists and is not a link" % soft_link_name)
raise FileExistsError("%s exists and is not a link" % soft_link_name)
try:
os.unlink(soft_link_name)
except OSError:
prdebug("Can't unlink %s" % (soft_link_name))
if not os.path.exists(input_file):
raise FileNotFoundError(
"trying to create a broken symlink to %s" % input_file)
raise FileNotFoundError("trying to create a broken symlink to %s" % input_file)
prdebug("os.symlink(%s, %s)" % (input_file, soft_link_name))
# Create symbolic link using absolute path
os.symlink(
os.path.abspath(input_file),
soft_link_name
)
os.symlink(os.path.abspath(input_file), soft_link_name)
def is_iterable_notstr(thing):
@@ -83,13 +80,14 @@ def available_cpu_count():
try:
import psutil
return psutil.cpu_count()
except (ImportError, AttributeError):
pass
warnings.warn(
"Could not get CPU count. Assuming one (1) CPU."
"Use -j N to set manually.")
"Could not get CPU count. Assuming one (1) CPU." "Use -j N to set manually."
)
return 1
@@ -108,8 +106,10 @@ def is_file_writable(test_file):
# p.is_file() throws an exception in some cases
if p.exists() and p.is_file():
return os.access(
os.fspath(p), os.W_OK,
effective_ids=(os.access in os.supports_effective_ids))
os.fspath(p),
os.W_OK,
effective_ids=(os.access in os.supports_effective_ids),
)
else:
try:
fp = p.open('wb')
@@ -132,12 +132,16 @@ def flatten_groups(groups):
def deprecated(func):
"""Warn that function is deprecated"""
@wraps(func)
def new_func(*args, **kwargs):
warnings.simplefilter('always', DeprecationWarning) # turn off filter
warnings.warn("Call to deprecated function {}.".format(func.__name__),
category=DeprecationWarning,
stacklevel=2)
warnings.warn(
"Call to deprecated function {}.".format(func.__name__),
category=DeprecationWarning,
stacklevel=2,
)
warnings.simplefilter('default', DeprecationWarning) # reset filter
return func(*args, **kwargs)
return new_func
src/ocrmypdf/hocrtransform.py
+107 -67
View File
@@ -44,7 +44,7 @@ class HocrTransformError(Exception):
pass
class HocrTransform():
class HocrTransform:
"""
A class for converting documents from the hOCR format.
@@ -53,17 +53,16 @@ class HocrTransform():
"""
box_pattern = re.compile(r'bbox((\s+\d+){4})')
baseline_pattern = re.compile(r'''
baseline_pattern = re.compile(
r'''
baseline \s+
([\-\+]?\d*\.?\d*) \s+ # +/- decimal float
([\-\+]?\d+) # +/- int''', re.VERBOSE)
ligatures = str.maketrans({
'ff': 'ff',
'ffi': 'f‌f‌i',
'ffl': 'f‌f‌l',
'fi': 'fi',
'fl': 'fl',
})
([\-\+]?\d+) # +/- int''',
re.VERBOSE,
)
ligatures = str.maketrans(
{'ff': 'ff', 'ffi': 'f‌f‌i', 'ffl': 'f‌f‌l', 'fi': 'fi', 'fl': 'fl'}
)
def __init__(self, hocrFileName, dpi):
self.dpi = dpi
@@ -78,8 +77,7 @@ class HocrTransform():
# get dimension in pt (not pixel!!!!) of the OCRed image
self.width, self.height = None, None
for div in self.hocr.findall(
".//%sdiv[@class='ocr_page']" % (self.xmlns)):
for div in self.hocr.findall(".//%sdiv[@class='ocr_page']" % (self.xmlns)):
coords = self.element_coordinates(div)
pt_coords = self.pt_from_pixel(coords)
self.width = pt_coords.x2 - pt_coords.x1
@@ -144,8 +142,7 @@ class HocrTransform():
"""
Returns the quantity in PDF units (pt) given quantity in pixels
"""
return Rect._make(
(c / self.dpi * inch) for c in pxl)
return Rect._make((c / self.dpi * inch) for c in pxl)
@classmethod
def replace_unsupported_chars(cls, s):
@@ -156,8 +153,15 @@ class HocrTransform():
"""
return s.translate(cls.ligatures)
def to_pdf(self, outFileName, imageFileName=None, showBoundingboxes=False,
fontname="Helvetica", invisibleText=False, interwordSpaces=False):
def to_pdf(
self,
outFileName,
imageFileName=None,
showBoundingboxes=False,
fontname="Helvetica",
invisibleText=False,
interwordSpaces=False,
):
"""
Creates a PDF file with an image superimposed on top of the text.
Text is positioned according to the bounding box of the lines in
@@ -168,17 +172,15 @@ class HocrTransform():
"""
# create the PDF file
# page size in points (1/72 in.)
pdf = Canvas(
outFileName, pagesize=(self.width, self.height), pageCompression=1)
pdf = Canvas(outFileName, pagesize=(self.width, self.height), pageCompression=1)
# draw bounding box for each paragraph
# light blue for bounding box of paragraph
pdf.setStrokeColorRGB(0, 1, 1)
# light blue for bounding box of paragraph
pdf.setFillColorRGB(0, 1, 1)
pdf.setLineWidth(0) # no line for bounding box
for elem in self.hocr.findall(
".//%sp[@class='%s']" % (self.xmlns, "ocr_par")):
pdf.setLineWidth(0) # no line for bounding box
for elem in self.hocr.findall(".//%sp[@class='%s']" % (self.xmlns, "ocr_par")):
elemtxt = self._get_element_text(elem).rstrip()
if len(elemtxt) == 0:
@@ -190,38 +192,58 @@ class HocrTransform():
# draw the bbox border
if showBoundingboxes:
pdf.rect(
pt.x1, self.height - pt.y2, pt.x2 - pt.x1, pt.y2 - pt.y1,
fill=1)
pt.x1, self.height - pt.y2, pt.x2 - pt.x1, pt.y2 - pt.y1, fill=1
)
found_lines = False
for line in self.hocr.findall(
".//%sspan[@class='%s']" % (self.xmlns, "ocr_line")):
".//%sspan[@class='%s']" % (self.xmlns, "ocr_line")
):
found_lines = True
self._do_line(pdf, line, "ocrx_word", fontname, invisibleText,
interwordSpaces, showBoundingboxes)
self._do_line(
pdf,
line,
"ocrx_word",
fontname,
invisibleText,
interwordSpaces,
showBoundingboxes,
)
if not found_lines:
# Tesseract did not report any lines (just words)
root = self.hocr.find(".//%sdiv[@class='%s']" % (self.xmlns, "ocr_page"))
self._do_line(pdf, root, "ocrx_word", fontname, invisibleText,
interwordSpaces, showBoundingboxes)
self._do_line(
pdf,
root,
"ocrx_word",
fontname,
invisibleText,
interwordSpaces,
showBoundingboxes,
)
# put the image on the page, scaled to fill the page
if imageFileName is not None:
pdf.drawImage(imageFileName, 0, 0,
width=self.width, height=self.height)
pdf.drawImage(imageFileName, 0, 0, width=self.width, height=self.height)
# finish up the page and save it
pdf.showPage()
pdf.save()
@classmethod
def polyval(cls, poly, x):
return x * poly[0] + poly[1]
def _do_line(self, pdf, line, elemclass, fontname, invisibleText,
interwordSpaces, showBoundingboxes):
def _do_line(
self,
pdf,
line,
elemclass,
fontname,
invisibleText,
interwordSpaces,
showBoundingboxes,
):
pxl_line_coords = self.element_coordinates(line)
line_box = self.pt_from_pixel(pxl_line_coords)
line_height = line_box.y2 - line_box.y1
@@ -254,23 +276,20 @@ class HocrTransform():
pdf.setLineWidth(0.5)
# negate slope because it is defined as a rise/run in pixel
# coordinates and page coordinates have the y axis flipped
pdf.line(line_box.x1,
baseline_y2,
line_box.x2,
self.polyval((-slope, baseline_y2),
line_box.x2 - line_box.x1))
pdf.line(
line_box.x1,
baseline_y2,
line_box.x2,
self.polyval((-slope, baseline_y2), line_box.x2 - line_box.x1),
)
# light green for bounding box of word/line
pdf.setDash(6, 3)
pdf.setStrokeColorRGB(1, 0, 0)
text.setTextTransform(
cos_a, -sin_a, sin_a, cos_a,
line_box.x1, baseline_y2
)
text.setTextTransform(cos_a, -sin_a, sin_a, cos_a, line_box.x1, baseline_y2)
pdf.setFillColorRGB(0, 0, 0) # text in black
elements = line.findall(
".//%sspan[@class='%s']" % (self.xmlns, elemclass))
elements = line.findall(".//%sspan[@class='%s']" % (self.xmlns, elemclass))
for elem in elements:
elemtxt = self._get_element_text(elem).strip()
elemtxt = self.replace_unsupported_chars(elemtxt)
@@ -287,22 +306,22 @@ class HocrTransform():
# though it would look better, because it will interfere with
# naive text extraction. \n does not work either.
elemtxt += ' '
box = Rect._make((
box.x1,
line_box.y1,
box.x2 + pdf.stringWidth(' ', fontname, line_height),
line_box.y2))
box = Rect._make(
(
box.x1,
line_box.y1,
box.x2 + pdf.stringWidth(' ', fontname, line_height),
line_box.y2,
)
)
box_width = box.x2 - box.x1
font_width = pdf.stringWidth(elemtxt, fontname, fontsize)
# draw the bbox border
if showBoundingboxes:
pdf.rect(
box.x1,
self.height - line_box.y2,
box_width,
line_height,
fill=0)
box.x1, self.height - line_box.y2, box_width, line_height, fill=0
)
# Adjust relative position of cursor
# This is equivalent to:
@@ -331,19 +350,40 @@ class HocrTransform():
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Convert hocr file to PDF')
parser.add_argument('-b', '--boundingboxes', action="store_true",
default=False, help='Show bounding boxes borders')
parser.add_argument('-r', '--resolution', type=int,
default=300,
help='Resolution of the image that was OCRed')
parser.add_argument('-i', '--image', default=None,
help='Path to the image to be placed above the text')
parser.add_argument('--interword-spaces', action='store_true',
default=False, help='Add spaces between words')
parser.add_argument('hocrfile', help='Path to the hocr file to be parsed')
parser.add_argument(
'outputfile', help='Path to the PDF file to be generated')
'-b',
'--boundingboxes',
action="store_true",
default=False,
help='Show bounding boxes borders',
)
parser.add_argument(
'-r',
'--resolution',
type=int,
default=300,
help='Resolution of the image that was OCRed',
)
parser.add_argument(
'-i',
'--image',
default=None,
help='Path to the image to be placed above the text',
)
parser.add_argument(
'--interword-spaces',
action='store_true',
default=False,
help='Add spaces between words',
)
parser.add_argument('hocrfile', help='Path to the hocr file to be parsed')
parser.add_argument('outputfile', help='Path to the PDF file to be generated')
args = parser.parse_args()
hocr = HocrTransform(args.hocrfile, args.resolution)
hocr.to_pdf(args.outputfile, args.image, args.boundingboxes, interwordSpaces=args.interword_spaces)
hocr.to_pdf(
args.outputfile,
args.image,
args.boundingboxes,
interwordSpaces=args.interword_spaces,
)
src/ocrmypdf/leptonica.py
+147 -96
View File
@@ -69,14 +69,14 @@ class _LeptonicaErrorTrap:
def __enter__(self):
from io import UnsupportedOperation
self.tmpfile = TemporaryFile()
# Save the old stderr, and redirect stderr to temporary file
sys.stderr.flush()
try:
self.copy_of_stderr = os.dup(sys.stderr.fileno())
os.dup2(self.tmpfile.fileno(), sys.stderr.fileno(),
inheritable=False)
os.dup2(self.tmpfile.fileno(), sys.stderr.fileno(), inheritable=False)
except UnsupportedOperation:
self.copy_of_stderr = None
return
@@ -185,9 +185,13 @@ class Pix(LeptonicaObject):
def __repr__(self):
if self._cdata:
s = "<leptonica.Pix image size={0}x{1} depth={2}{4} at 0x{3:x}>"
return s.format(self._cdata.w, self._cdata.h, self._cdata.d,
int(ffi.cast('intptr_t', self._cdata)),
'(colormapped)' if self._cdata.colormap else '')
return s.format(
self._cdata.w,
self._cdata.h,
self._cdata.d,
int(ffi.cast('intptr_t', self._cdata)),
'(colormapped)' if self._cdata.colormap else '',
)
else:
return "<leptonica.Pix image NULL>"
@@ -289,8 +293,7 @@ class Pix(LeptonicaObject):
with _LeptonicaErrorTrap():
return cls(lept.pixRead(os.fsencode(filename)))
def write_implied_format(
self, path, jpeg_quality=0, jpeg_progressive=0):
def write_implied_format(self, path, jpeg_quality=0, jpeg_progressive=0):
"""Write pix to the filename, with the extension indicating format.
jpeg_quality -- quality (iff JPEG; 1 - 100, 0 for default)
@@ -299,8 +302,8 @@ class Pix(LeptonicaObject):
filename = fspath(path)
with _LeptonicaErrorTrap():
lept.pixWriteImpliedFormat(
os.fsencode(filename),
self._cdata, jpeg_quality, jpeg_progressive)
os.fsencode(filename), self._cdata, jpeg_quality, jpeg_progressive
)
@classmethod
def frompil(self, pillow_image):
@@ -401,11 +404,13 @@ class Pix(LeptonicaObject):
"""
with _LeptonicaErrorTrap():
return Pix(lept.pixRemoveColormapGeneral(
self._cdata, removal_type, lept.L_COPY))
return Pix(
lept.pixRemoveColormapGeneral(self._cdata, removal_type, lept.L_COPY)
)
def otsu_adaptive_threshold(
self, tile_size=(300, 300), kernel_size=(4, 4), scorefract=0.1):
self, tile_size=(300, 300), kernel_size=(4, 4), scorefract=0.1
):
with _LeptonicaErrorTrap():
sx, sy = tile_size
smoothx, smoothy = kernel_size
@@ -413,20 +418,23 @@ class Pix(LeptonicaObject):
pix = Pix(lept.pixConvertTo8(self._cdata, 0))
result = lept.pixOtsuAdaptiveThreshold(
pix._cdata,
sx, sy,
smoothx, smoothy,
scorefract,
ffi.NULL,
p_pix)
pix._cdata, sx, sy, smoothx, smoothy, scorefract, ffi.NULL, p_pix
)
if result == 0:
return Pix(p_pix[0])
else:
return None
def otsu_threshold_on_background_norm(
self, mask=None, tile_size=(10, 15), thresh=100, mincount=50,
bgval=255, kernel_size=(2, 2), scorefract=0.1):
self,
mask=None,
tile_size=(10, 15),
thresh=100,
mincount=50,
bgval=255,
kernel_size=(2, 2),
scorefract=0.1,
):
with _LeptonicaErrorTrap():
sx, sy = tile_size
smoothx, smoothy = kernel_size
@@ -438,17 +446,27 @@ class Pix(LeptonicaObject):
thresh_pix = lept.pixOtsuThreshOnBackgroundNorm(
pix._cdata,
mask,
sx, sy,
thresh, mincount, bgval,
smoothx, smoothy,
sx,
sy,
thresh,
mincount,
bgval,
smoothx,
smoothy,
scorefract,
ffi.NULL
ffi.NULL,
)
return Pix(thresh_pix)
def masked_threshold_on_background_norm(
self, mask=None, tile_size=(10, 15), thresh=100, mincount=50,
kernel_size=(2, 2), scorefract=0.1):
self,
mask=None,
tile_size=(10, 15),
thresh=100,
mincount=50,
kernel_size=(2, 2),
scorefract=0.1,
):
with _LeptonicaErrorTrap():
sx, sy = tile_size
smoothx, smoothy = kernel_size
@@ -460,74 +478,91 @@ class Pix(LeptonicaObject):
thresh_pix = lept.pixMaskedThreshOnBackgroundNorm(
pix._cdata,
mask,
sx, sy,
thresh, mincount,
smoothx, smoothy,
sx,
sy,
thresh,
mincount,
smoothx,
smoothy,
scorefract,
ffi.NULL
ffi.NULL,
)
return Pix(thresh_pix)
def crop_to_foreground(
self, threshold=128, mindist=70, erasedist=30, pagenum=0,
showmorph=0, display=0, pdfdir=ffi.NULL):
self,
threshold=128,
mindist=70,
erasedist=30,
pagenum=0,
showmorph=0,
display=0,
pdfdir=ffi.NULL,
):
with _LeptonicaErrorTrap():
cropbox = Box(lept.pixFindPageForeground(
self._cdata,
threshold,
mindist,
erasedist,
pagenum,
showmorph,
display,
pdfdir))
cropbox = Box(
lept.pixFindPageForeground(
self._cdata,
threshold,
mindist,
erasedist,
pagenum,
showmorph,
display,
pdfdir,
)
)
cropped_pix = lept.pixClipRectangle(
self._cdata,
cropbox._cdata,
ffi.NULL)
cropped_pix = lept.pixClipRectangle(self._cdata, cropbox._cdata, ffi.NULL)
return Pix(cropped_pix)
def clean_background_to_white(
self, mask=None, grayscale=None, gamma=1.0, black=0, white=255):
self, mask=None, grayscale=None, gamma=1.0, black=0, white=255
):
with _LeptonicaErrorTrap():
return Pix(lept.pixCleanBackgroundToWhite(
self._cdata,
mask or ffi.NULL,
grayscale or ffi.NULL,
gamma,
black,
white))
return Pix(
lept.pixCleanBackgroundToWhite(
self._cdata,
mask or ffi.NULL,
grayscale or ffi.NULL,
gamma,
black,
white,
)
)
def gamma_trc(self, gamma=1.0, minval=0, maxval=255):
with _LeptonicaErrorTrap():
return Pix(lept.pixGammaTRC(
ffi.NULL,
self._cdata,
gamma,
minval,
maxval
))
return Pix(lept.pixGammaTRC(ffi.NULL, self._cdata, gamma, minval, maxval))
def background_norm(
self, mask=None, grayscale=None, tile_size=(10, 15), fg_threshold=60,
min_count=40, bg_val=200, smooth_kernel=(2, 1)):
self,
mask=None,
grayscale=None,
tile_size=(10, 15),
fg_threshold=60,
min_count=40,
bg_val=200,
smooth_kernel=(2, 1),
):
# Background norm doesn't work on color mapped Pix, so remove colormap
target_pix = self.remove_colormap(lept.REMOVE_CMAP_BASED_ON_SRC)
with _LeptonicaErrorTrap():
return Pix(lept.pixBackgroundNorm(
target_pix._cdata,
mask or ffi.NULL,
grayscale or ffi.NULL,
tile_size[0],
tile_size[1],
fg_threshold,
min_count,
bg_val,
smooth_kernel[0],
smooth_kernel[1]
))
return Pix(
lept.pixBackgroundNorm(
target_pix._cdata,
mask or ffi.NULL,
grayscale or ffi.NULL,
tile_size[0],
tile_size[1],
fg_threshold,
min_count,
bg_val,
smooth_kernel[0],
smooth_kernel[1],
)
)
@staticmethod
@lru_cache(maxsize=1)
@@ -544,8 +579,7 @@ class Pix(LeptonicaObject):
raise LeptonicaError("Leptonica version is too old")
correlation = ffi.new('float *', 0.0)
result = lept.pixCorrelationBinary(pix1._cdata, pix2._cdata,
correlation)
result = lept.pixCorrelationBinary(pix1._cdata, pix2._cdata, correlation)
if result != 0:
raise LeptonicaError("Correlation failed")
return correlation[0]
@@ -553,8 +587,7 @@ class Pix(LeptonicaObject):
def generate_pdf_ci_data(self, type_, quality):
"Convert to PDF data, with transcoding"
p_compdata = ffi.new('L_COMP_DATA **')
result = lept.pixGenerateCIData(self._cdata, type_, quality, 0,
p_compdata)
result = lept.pixGenerateCIData(self._cdata, type_, quality, 0, p_compdata)
if result != 0:
raise LeptonicaError("Generate PDF data failed")
return CompressedData(p_compdata[0])
@@ -569,13 +602,15 @@ class Pix(LeptonicaObject):
pixa_candidates = PixArray(lept.pixExtractBarcodes(pix._cdata, 0))
if not pixa_candidates:
return
sarray = StringArray(lept.pixReadBarcodes(
pixa_candidates._cdata,
lept.L_BF_ANY,
lept.L_USE_WIDTHS,
ffi.NULL,
0
))
sarray = StringArray(
lept.pixReadBarcodes(
pixa_candidates._cdata,
lept.L_BF_ANY,
lept.L_USE_WIDTHS,
ffi.NULL,
0,
)
)
except (LeptonicaError, ValueError, IndexError) as e:
return
finally:
@@ -635,7 +670,8 @@ class CompressedData(LeptonicaObject):
p_compdata = ffi.new('L_COMP_DATA **')
result = lept.l_generateCIDataForPdf(
os.fsencode(filename), ffi.NULL, jpeg_quality, p_compdata)
os.fsencode(filename), ffi.NULL, jpeg_quality, p_compdata
)
if result != 0:
raise LeptonicaError("CompressedData.open")
return CompressedData(p_compdata[0])
@@ -689,7 +725,8 @@ class Box(LeptonicaObject):
def __repr__(self):
if self._cdata:
return '<leptonica.Box x={0} y={1} w={2} h={3}>'.format(
self.x, self.y, self.w, self.h)
self.x, self.y, self.w, self.h
)
return '<leptonica.Box NULL>'
@property
@@ -808,15 +845,22 @@ def deskew(infile, outfile, dpi):
raise LeptonicaIOError("Failed to open destination file: %s" % outfile)
def remove_background(infile, outfile, tile_size=(40, 60), gamma=1.0,
black_threshold=70, white_threshold=190):
def remove_background(
infile,
outfile,
tile_size=(40, 60),
gamma=1.0,
black_threshold=70,
white_threshold=190,
):
try:
pix = Pix.open(infile)
except LeptonicaIOError:
raise LeptonicaIOError("Failed to open file: %s" % infile)
pix = pix.background_norm(tile_size=tile_size).gamma_trc(
gamma, black_threshold, white_threshold)
gamma, black_threshold, white_threshold
)
try:
pix.write_implied_format(outfile)
@@ -825,15 +869,22 @@ def remove_background(infile, outfile, tile_size=(40, 60), gamma=1.0,
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description="Python wrapper to access Leptonica")
parser = argparse.ArgumentParser(description="Python wrapper to access Leptonica")
subparsers = parser.add_subparsers(title='commands',
description='supported operations')
subparsers = parser.add_subparsers(
title='commands', description='supported operations'
)
parser_deskew = subparsers.add_parser('deskew')
parser_deskew.add_argument('-r', '--dpi', dest='dpi', action='store',
type=int, default=300, help='input resolution')
parser_deskew.add_argument(
'-r',
'--dpi',
dest='dpi',
action='store',
type=int,
default=300,
help='input resolution',
)
parser_deskew.add_argument('infile', help='image to deskew')
parser_deskew.add_argument('outfile', help='deskewed output image')
parser_deskew.set_defaults(func=deskew)
src/ocrmypdf/lib/_leptonica.py
+323 -7
View File
File diff suppressed because one or more lines are too long
src/ocrmypdf/lib/compile_leptonica.py
+8 -4
View File
@@ -19,7 +19,8 @@
from cffi import FFI
ffibuilder = FFI()
ffibuilder.cdef("""
ffibuilder.cdef(
"""
typedef signed char l_int8;
typedef unsigned char l_uint8;
typedef short l_int16;
@@ -202,9 +203,11 @@ enum {
SEL_MISS = 2
};
""")
"""
)
ffibuilder.cdef("""
ffibuilder.cdef(
"""
PIX * pixRead ( const char *filename );
PIX * pixReadMem ( const l_uint8 *data, size_t size );
PIX * pixScale ( PIX *pixs, l_float32 scalex, l_float32 scaley );
@@ -480,7 +483,8 @@ void selDestroy ( SEL **psel );
l_int32
setMsgSeverity(l_int32 newsev);
""")
"""
)
ffibuilder.set_source("ocrmypdf.lib._leptonica", None)
src/ocrmypdf/optimize.py
+56 -68
View File
@@ -80,9 +80,11 @@ def extract_image_jbig2(*, pike, root, log, image, xref, options):
return None
pim, filtdp = result
if pim.bits_per_component == 1 \
and filtdp != '/JBIG2Decode' \
and jbig2enc.available():
if (
pim.bits_per_component == 1
and filtdp != '/JBIG2Decode'
and jbig2enc.available()
):
try:
imgname = Path(root / '{:08d}'.format(xref))
with imgname.open('wb') as f:
@@ -100,8 +102,7 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
return None
pim, filtdp = result
if filtdp[0] == '/DCTDecode' \
and options.optimize >= 2:
if filtdp[0] == '/DCTDecode' and options.optimize >= 2:
# This is a simple heuristic derived from some training data, that has
# about a 70% chance of guessing whether the JPEG is high quality,
# and possibly recompressible, or not. The number itself doesn't mean
@@ -126,9 +127,11 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
except pikepdf.UnsupportedImageTypeError:
return None
return xref, ext
elif pim.indexed \
and pim.colorspace in pim.SIMPLE_COLORSPACES \
and options.optimize >= 3:
elif (
pim.indexed
and pim.colorspace in pim.SIMPLE_COLORSPACES
and options.optimize >= 3
):
# Try to improve on indexed images - these are far from low hanging
# fruit in most cases
pim.as_pil_image().save(png_name(root, xref))
@@ -142,7 +145,6 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
return None
def extract_images(pike, root, log, options, extract_fn):
"""Extract image using extract_fn
@@ -172,8 +174,7 @@ def extract_images(pike, root, log, options, extract_fn):
image = pike.get_object((xref, 0))
try:
result = extract_fn(
pike=pike, root=root, log=log, image=image,
xref=xref, options=options
pike=pike, root=root, log=log, image=image, xref=xref, options=options
)
except Exception as e:
log.debug("Image xref %s", xref)
@@ -190,17 +191,13 @@ def extract_images_generic(pike, root, log, options):
jpegs = []
pngs = []
for _, xref, ext in extract_images(
pike, root, log, options, extract_image_generic):
for _, xref, ext in extract_images(pike, root, log, options, extract_image_generic):
log.debug('xref = %s ext = %s', xref, ext)
if ext == '.png':
pngs.append(xref)
elif ext == '.jpg':
jpegs.append(xref)
log.debug(
"Optimizable images: "
"JPEGs: %s PNGs: %s", len(jpegs), len(pngs)
)
log.debug("Optimizable images: " "JPEGs: %s PNGs: %s", len(jpegs), len(pngs))
return jpegs, pngs
@@ -209,17 +206,16 @@ def extract_images_jbig2(pike, root, log, options):
jbig2_groups = defaultdict(list)
for pageno, xref, ext in extract_images(
pike, root, log, options, extract_image_jbig2):
pike, root, log, options, extract_image_jbig2
):
group = pageno // options.jbig2_page_group_size
jbig2_groups[group].append((xref, ext))
# Elide empty groups
jbig2_groups = {group: xrefs for group, xrefs in jbig2_groups.items()
if len(xrefs) > 0}
log.debug(
"Optimizable images: "
"JBIG2 groups: %s", (len(jbig2_groups),)
)
jbig2_groups = {
group: xrefs for group, xrefs in jbig2_groups.items() if len(xrefs) > 0
}
log.debug("Optimizable images: " "JBIG2 groups: %s", (len(jbig2_groups),))
return jbig2_groups
@@ -233,7 +229,7 @@ def _produce_jbig2_images(jbig2_groups, root, log, options):
jbig2enc.convert_group,
cwd=fspath(root),
infiles=(img_name(root, xref, ext) for xref, ext in xref_exts),
out_prefix=prefix
out_prefix=prefix,
)
yield future
@@ -247,7 +243,7 @@ def _produce_jbig2_images(jbig2_groups, root, log, options):
jbig2enc.convert_single,
cwd=fspath(root),
infile=img_name(root, xref, ext),
outfile=root / ('{}.{:04d}'.format(prefix, n))
outfile=root / ('{}.{:04d}'.format(prefix, n)),
)
yield future
@@ -256,8 +252,7 @@ def _produce_jbig2_images(jbig2_groups, root, log, options):
else:
jbig2_futures = jbig2_single_futures
with concurrent.futures.ThreadPoolExecutor(
max_workers=options.jobs) as executor:
with concurrent.futures.ThreadPoolExecutor(max_workers=options.jobs) as executor:
futures = jbig2_futures(executor, root, jbig2_groups)
for future in concurrent.futures.as_completed(futures):
proc = future.result()
@@ -286,9 +281,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
if jbig2_symfile.exists():
jbig2_globals_data = jbig2_symfile.read_bytes()
jbig2_globals = pikepdf.Stream(pike, jbig2_globals_data)
jbig2_globals_dict = pikepdf.Dictionary({
'/JBIG2Globals': jbig2_globals
})
jbig2_globals_dict = pikepdf.Dictionary({'/JBIG2Globals': jbig2_globals})
elif options.jbig2_page_group_size == 1:
jbig2_globals_dict = None
else:
@@ -302,7 +295,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
im_obj.write(
jbig2_im_data,
filter=pikepdf.Name('/JBIG2Decode'),
decode_parms=jbig2_globals_dict
decode_parms=jbig2_globals_dict,
)
@@ -316,9 +309,7 @@ def transcode_jpegs(pike, jpegs, root, log, options):
# 'close'. Seems to be mostly harmless
# https://github.com/python-pillow/Pillow/issues/1144
with Image.open(fspath(in_jpg)) as im:
im.save(fspath(opt_jpg),
optimize=True,
quality=options.jpeg_quality)
im.save(fspath(opt_jpg), optimize=True, quality=options.jpeg_quality)
# pylint: disable=no-member
if opt_jpg.stat().st_size > in_jpg.stat().st_size:
log.debug("xref %s, jpeg, made larger - skip", xref)
@@ -326,24 +317,26 @@ def transcode_jpegs(pike, jpegs, root, log, options):
compdata = leptonica.CompressedData.open(opt_jpg)
im_obj = pike.get_object(xref, 0)
im_obj.write(
compdata.read(), filter=pikepdf.Name('/DCTDecode')
)
im_obj.write(compdata.read(), filter=pikepdf.Name('/DCTDecode'))
def transcode_pngs(pike, pngs, root, log, options):
if options.optimize >= 2:
png_quality = (
max(10, options.png_quality - 10),
min(100, options.png_quality + 10)
min(100, options.png_quality + 10),
)
with concurrent.futures.ThreadPoolExecutor(
max_workers=options.jobs) as executor:
max_workers=options.jobs
) as executor:
for xref in pngs:
executor.submit(
pngquant.quantize,
png_name(root, xref), png_name(root, xref),
png_quality[0], png_quality[1])
png_name(root, xref),
png_name(root, xref),
png_quality[0],
png_quality[1],
)
for xref in pngs:
im_obj = pike.get_object(xref, 0)
@@ -353,9 +346,7 @@ def transcode_pngs(pike, pngs, root, log, options):
pix = leptonica.Pix.open(png_name(root, xref))
if pix.depth == 1:
pix = pix.invert() # PDF assumes 1 is black for monochrome
compdata = pix.generate_pdf_ci_data(
leptonica.lept.L_FLATE_ENCODE, 0
)
compdata = pix.generate_pdf_ci_data(leptonica.lept.L_FLATE_ENCODE, 0)
except leptonica.LeptonicaError as e:
log.error(e)
continue
@@ -363,7 +354,7 @@ def transcode_pngs(pike, pngs, root, log, options):
# This is what we should be doing: open the compressed data without
# transcoding. However this shifts each pixel row by one for some
# reason.
#compdata = leptonica.CompressedData.open(png_name(root, xref))
# compdata = leptonica.CompressedData.open(png_name(root, xref))
if len(compdata) > int(im_obj.stream_dict.Length):
continue # If we produced a larger image, don't use
@@ -379,8 +370,12 @@ def transcode_pngs(pike, pngs, root, log, options):
palette_pdf_string = compdata.get_palette_pdf_string()
palette_data = pikepdf.Object.parse(palette_pdf_string)
palette_stream = pikepdf.Stream(pike, bytes(palette_data))
palette = [pikepdf.Name('/Indexed'), pikepdf.Name('/DeviceRGB'),
compdata.ncolors - 1, palette_stream]
palette = [
pikepdf.Name('/Indexed'),
pikepdf.Name('/DeviceRGB'),
compdata.ncolors - 1,
palette_stream,
]
cs = palette
else:
if compdata.spp == 1:
@@ -391,16 +386,11 @@ def transcode_pngs(pike, pngs, root, log, options):
cs = pikepdf.Name('/DeviceCMYK')
im_obj.ColorSpace = cs
im_obj.write(
compdata.read(),
filter=pikepdf.Name('/FlateDecode'), decode_parms=predictor
compdata.read(), filter=pikepdf.Name('/FlateDecode'), decode_parms=predictor
)
def optimize(
input_file,
output_file,
log,
context):
def optimize(input_file, output_file, log, context):
options = context.get_options()
if options.optimize == 0:
@@ -408,14 +398,11 @@ def optimize(
return
if options.jpeg_quality == 0:
options.jpeg_quality = \
DEFAULT_JPEG_QUALITY if options.optimize < 3 else 40
options.jpeg_quality = DEFAULT_JPEG_QUALITY if options.optimize < 3 else 40
if options.png_quality == 0:
options.png_quality = \
DEFAULT_PNG_QUALITY if options.optimize < 3 else 30
options.png_quality = DEFAULT_PNG_QUALITY if options.optimize < 3 else 30
if options.jbig2_page_group_size == 0:
options.jbig2_page_group_size = \
10 if options.jbig2_lossy else 1
options.jbig2_page_group_size = 10 if options.jbig2_lossy else 1
pike = pikepdf.Pdf.open(input_file)
@@ -431,15 +418,17 @@ def optimize(
target_file = Path(output_file).with_suffix('.opt.pdf')
pike.remove_unreferenced_resources()
pike.save(target_file, preserve_pdfa=True,
object_stream_mode=pikepdf.ObjectStreamMode.generate)
pike.save(
target_file,
preserve_pdfa=True,
object_stream_mode=pikepdf.ObjectStreamMode.generate,
)
input_size = Path(input_file).stat().st_size
output_size = Path(target_file).stat().st_size
ratio = input_size / output_size
savings = 1 - output_size / input_size
log.info("Optimize ratio: {:.2f} savings: {:.1f}%".format(
ratio, 100 * savings))
log.info("Optimize ratio: {:.2f} savings: {:.1f}%".format(ratio, 100 * savings))
if savings < 0:
log.info("Optimize did not improve the file - discarded")
@@ -455,8 +444,7 @@ def main(infile, outfile, level, jobs=1):
class OptimizeOptions:
"""Emulate ocrmypdf's options"""
def __init__(
self, jobs, optimize, jpeg_quality, png_quality, jb2lossy):
def __init__(self, jobs, optimize, jpeg_quality, png_quality, jb2lossy):
self.jobs = jobs
self.optimize = optimize
self.jpeg_quality = jpeg_quality
@@ -473,7 +461,7 @@ def main(infile, outfile, level, jobs=1):
optimize=int(level),
jpeg_quality=0, # Use default
png_quality=0,
jb2lossy=False
jb2lossy=False,
)
ctx.set_options(options)
src/ocrmypdf/pdfa.py
+9 -13
View File
@@ -41,7 +41,7 @@ import pikepdf
from pikepdf.models.metadata import (
encode_pdf_date as _encode_date,
decode_pdf_date as _decode_date
decode_pdf_date as _decode_date,
)
from .helpers import deprecated
@@ -49,8 +49,7 @@ from .helpers import deprecated
ICC_PROFILE_RELPATH = 'data/sRGB.icc'
SRGB_ICC_PROFILE = pkg_resources.resource_filename(
'ocrmypdf', ICC_PROFILE_RELPATH)
SRGB_ICC_PROFILE = pkg_resources.resource_filename('ocrmypdf', ICC_PROFILE_RELPATH)
# This is a template written in PostScript which is needed to create PDF/A
@@ -128,12 +127,7 @@ def _encode_ascii(s: str) -> str:
be to implement PdfDocEncoding in pikepdf and encode to that, or handle
metadata there.
"""
trans = str.maketrans({
'(': '',
')': '',
'\\': '',
'\0': ''
})
trans = str.maketrans({'(': '', ')': '', '\\': '', '\0': ''})
return s.translate(trans).encode('ascii', errors='replace').decode()
@@ -163,8 +157,7 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark=None, ascii_docinfo=None)
"""
t = Template(pdfa_def_template)
result = t.substitute(icc_profile=icc_profile,
icc_identifier=icc_identifier)
result = t.substitute(icc_profile=icc_profile, icc_identifier=icc_identifier)
return result
@@ -205,8 +198,11 @@ def file_claims_pdfa(filename):
pdf = pikepdf.open(filename)
pdfmeta = pdf.open_metadata()
if not pdfmeta.pdfa_status:
return {'pass': False, 'output': 'pdf',
'conformance': 'No PDF/A metadata in XMP'}
return {
'pass': False,
'output': 'pdf',
'conformance': 'No PDF/A metadata in XMP',
}
valid_part_conforms = {'1A', '1B', '2A', '2B', '2U', '3A', '3B', '3U'}
conformance = 'PDF/A-{}'.format(pdfmeta.pdfa_status)
pdfa_dict = {}
src/ocrmypdf/pdfinfo/__init__.py
+82 -67
View File
@@ -35,12 +35,11 @@ from .layout import get_page_analysis, get_text_boxes
from ..exceptions import EncryptedPdfError
Colorspace = Enum('Colorspace',
'gray rgb cmyk lab icc index sep devn pattern jpeg2000')
Colorspace = Enum('Colorspace', 'gray rgb cmyk lab icc index sep devn pattern jpeg2000')
Encoding = Enum('Encoding',
'ccitt jpeg jpeg2000 jbig2 asciihex ascii85 lzw flate ' + \
'runlength')
Encoding = Enum(
'Encoding', 'ccitt jpeg jpeg2000 jbig2 asciihex ascii85 lzw flate ' + 'runlength'
)
FRIENDLY_COLORSPACE = {
'/DeviceGray': Colorspace.gray,
@@ -71,7 +70,7 @@ FRIENDLY_ENCODING = {
'/A85': Encoding.ascii85,
'/LZW': Encoding.lzw,
'/Fl': Encoding.flate,
'/RL': Encoding.runlength
'/RL': Encoding.runlength,
}
FRIENDLY_COMP = {
@@ -79,28 +78,28 @@ FRIENDLY_COMP = {
Colorspace.rgb: 3,
Colorspace.cmyk: 4,
Colorspace.lab: 3,
Colorspace.index: 1
Colorspace.index: 1,
}
UNIT_SQUARE = (1.0, 0.0, 0.0, 1.0, 0.0, 0.0)
def _is_unit_square(shorthand):
values = map(float, shorthand)
pairwise = zip(values, UNIT_SQUARE)
return all([isclose(a, b, rel_tol=1e-3) for a, b in pairwise])
XobjectSettings = namedtuple('XobjectSettings',
['name', 'shorthand', 'stack_depth'])
InlineSettings = namedtuple('InlineSettings',
['iimage', 'shorthand', 'stack_depth'])
XobjectSettings = namedtuple('XobjectSettings', ['name', 'shorthand', 'stack_depth'])
ContentsInfo = namedtuple('ContentsInfo',
['xobject_settings', 'inline_images', 'found_vector'])
InlineSettings = namedtuple('InlineSettings', ['iimage', 'shorthand', 'stack_depth'])
TextboxInfo = namedtuple('TextboxInfo',
['bbox', 'is_visible', 'is_corrupt'])
ContentsInfo = namedtuple(
'ContentsInfo', ['xobject_settings', 'inline_images', 'found_vector']
)
TextboxInfo = namedtuple('TextboxInfo', ['bbox', 'is_visible', 'is_corrupt'])
class VectorInfo:
@@ -112,9 +111,9 @@ def _normalize_stack(graphobjs):
"""Convert runs of qQ's in the stack into single graphobjs"""
for operands, operator in graphobjs:
operator = str(operator)
if re.match(r'Q*q+$', operator): # Zero or more Q, one or more q
for char in operator: # Split into individual
yield ([], char) # Yield individual
if re.match(r'Q*q+$', operator): # Zero or more Q, one or more q
for char in operator: # Split into individual
yield ([], char) # Yield individual
else:
yield (operands, operator)
@@ -155,15 +154,19 @@ def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
image_ops = set('BI ID EI q Q Do cm'.split())
operator_whitelist = ' '.join(vector_ops | image_ops)
for n, graphobj in enumerate(_normalize_stack(
pikepdf.parse_content_stream(contentstream, operator_whitelist))):
for n, graphobj in enumerate(
_normalize_stack(
pikepdf.parse_content_stream(contentstream, operator_whitelist)
)
):
operands, operator = graphobj
if operator == 'q':
stack.append(ctm)
if len(stack) > 32: # See docstring
if len(stack) > 128:
raise RuntimeError(
"PDF graphics stack overflowed hard limit, operator %i" % n)
"PDF graphics stack overflowed hard limit, operator %i" % n
)
warn("PDF graphics stack overflowed spec limit")
elif operator == 'Q':
try:
@@ -177,14 +180,14 @@ def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
elif operator == 'Do':
image_name = operands[0]
settings = XobjectSettings(
name=image_name, shorthand=ctm.shorthand,
stack_depth=len(stack))
name=image_name, shorthand=ctm.shorthand, stack_depth=len(stack)
)
xobject_settings.append(settings)
elif operator == 'INLINE IMAGE': # BI/ID/EI are grouped into this
elif operator == 'INLINE IMAGE': # BI/ID/EI are grouped into this
iimage = operands[0]
inline = InlineSettings(
iimage=iimage, shorthand=ctm.shorthand,
stack_depth=len(stack))
iimage=iimage, shorthand=ctm.shorthand, stack_depth=len(stack)
)
inline_images.append(inline)
elif operator in vector_ops:
found_vector = True
@@ -192,7 +195,8 @@ def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
return ContentsInfo(
xobject_settings=xobject_settings,
inline_images=inline_images,
found_vector=found_vector)
found_vector=found_vector,
)
def _get_dpi(ctm_shorthand, image_size):
@@ -262,8 +266,7 @@ def _get_dpi(ctm_shorthand, image_size):
class ImageInfo:
DPI_PREC = Decimal('1.000')
def __init__(self, *, name='', pdfimage=None, inline=None,
shorthand=None):
def __init__(self, *, name='', pdfimage=None, inline=None, shorthand=None):
self._name = str(name)
self._shorthand = shorthand
@@ -348,19 +351,24 @@ class ImageInfo:
return _get_dpi(self._shorthand, (self._width, self._height))[1]
def __repr__(self):
class_locals = {attr: getattr(self, attr, None) for attr in dir(self)
if not attr.startswith('_')}
class_locals = {
attr: getattr(self, attr, None)
for attr in dir(self)
if not attr.startswith('_')
}
return (
"<ImageInfo '{name}' {type_} {width}x{height} {color} "
"{comp} {bpc} {enc} {xres}x{yres}>").format(**class_locals)
"{comp} {bpc} {enc} {xres}x{yres}>"
).format(**class_locals)
def _find_inline_images(contentsinfo):
"Find inline images in the contentstream"
for n, inline in enumerate(contentsinfo.inline_images):
yield ImageInfo(name='inline-%02d' % n, shorthand=inline.shorthand,
inline=inline)
yield ImageInfo(
name='inline-%02d' % n, shorthand=inline.shorthand, inline=inline
)
def _image_xobjects(container):
@@ -414,8 +422,7 @@ def _find_regular_images(container, contentsinfo):
# these from our DPI calculation for the page.
continue
yield ImageInfo(name=draw.name, pdfimage=pdfimage, shorthand=
draw.shorthand)
yield ImageInfo(name=draw.name, pdfimage=pdfimage, shorthand=draw.shorthand)
def _find_form_xobject_images(pdf, container, contentsinfo):
@@ -446,7 +453,8 @@ def _find_form_xobject_images(pdf, container, contentsinfo):
# same object are both very rare.
ctm_shorthand = settings.shorthand
yield from _process_content_streams(
pdf=pdf, container=form_xobject, shorthand=ctm_shorthand)
pdf=pdf, container=form_xobject, shorthand=ctm_shorthand
)
def _process_content_streams(*, pdf, container, shorthand=None):
@@ -470,8 +478,7 @@ def _process_content_streams(*, pdf, container, shorthand=None):
if container.get('/Type') == '/Page' and '/Contents' in container:
initial_shorthand = shorthand or UNIT_SQUARE
elif container.get('/Type') == '/XObject' and \
container['/Subtype'] == '/Form':
elif container.get('/Type') == '/XObject' and container['/Subtype'] == '/Form':
# Set the CTM to the state it was when the "Do" operator was
# encountered that is drawing this instance of the Form XObject
ctm = PdfMatrix(shorthand) if shorthand else PdfMatrix.identity()
@@ -505,9 +512,9 @@ def _page_has_text(text_blocks, page_width, page_height):
margin_ratio = 0.125
interior_bbox = (
margin_ratio * pw, # left
(1 - margin_ratio) * ph, # top
(1 - margin_ratio) * pw, # right
margin_ratio * ph # bottom (first quadrant: bottom < top)
(1 - margin_ratio) * ph, # top
(1 - margin_ratio) * pw, # right
margin_ratio * ph, # bottom (first quadrant: bottom < top)
)
def rects_intersect(a, b):
@@ -535,8 +542,8 @@ def simplify_textboxes(miner):
first_line = box._objs[0]
first_char = first_line._objs[0]
visible = (first_char.rendermode != 3)
corrupt = (first_char.get_text() == '\ufffd')
visible = first_char.rendermode != 3
corrupt = first_char.get_text() == '\ufffd'
yield TextboxInfo(box.bbox, visible, corrupt)
@@ -552,7 +559,8 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
if xmltext is not None:
bboxes = ghosttext.page_get_textblocks(
fspath(infile), pageno, xmltext=xmltext, height=height_pt)
fspath(infile), pageno, xmltext=xmltext, height=height_pt
)
pageinfo['bboxes'] = bboxes
else:
pscript5_mode = str(pdf.docinfo.get('/Creator')).startswith('PScript5')
@@ -560,9 +568,7 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
pageinfo['textboxes'] = list(simplify_textboxes(miner))
bboxes = (box.bbox for box in pageinfo['textboxes'])
pageinfo['has_text'] = _page_has_text(
bboxes, width_pt, height_pt
)
pageinfo['has_text'] = _page_has_text(bboxes, width_pt, height_pt)
userunit = page.get('/UserUnit', Decimal(1.0))
if not isinstance(userunit, Decimal):
@@ -577,24 +583,24 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
pageinfo['rotate'] = 0
userunit_shorthand = (userunit, 0, 0, userunit, 0, 0)
contentsinfo = [ci for ci in
_process_content_streams(pdf=pdf, container=page,
shorthand=userunit_shorthand)]
contentsinfo = [
ci
for ci in _process_content_streams(
pdf=pdf, container=page, shorthand=userunit_shorthand
)
]
pageinfo['has_vector'] = False
if any(isinstance(ci, VectorInfo) for ci in contentsinfo):
pageinfo['has_vector'] = True
pageinfo['images'] = [im for im in contentsinfo
if isinstance(im, ImageInfo)]
pageinfo['images'] = [im for im in contentsinfo if isinstance(im, ImageInfo)]
if pageinfo['images']:
xres = Decimal(max(image.xres for image in pageinfo['images']))
yres = Decimal(max(image.yres for image in pageinfo['images']))
pageinfo['xres'], pageinfo['yres'] = xres, yres
pageinfo['width_pixels'] = \
int(round(xres * pageinfo['width_inches']))
pageinfo['height_pixels'] = \
int(round(yres * pageinfo['height_inches']))
pageinfo['width_pixels'] = int(round(xres * pageinfo['width_inches']))
pageinfo['height_pixels'] = int(round(yres * pageinfo['height_inches']))
return pageinfo
@@ -689,13 +695,14 @@ class PageInfo:
if 'textboxes' not in self._pageinfo:
if visible is not None and corrupt is not None:
raise NotImplementedError(
'Ghostscript textboxes cannot be classified')
raise NotImplementedError('Ghostscript textboxes cannot be classified')
return self._pageinfo['bboxes']
return (obj.bbox for obj in self._pageinfo['textboxes']
if predicate(obj, visible, corrupt))
return (
obj.bbox
for obj in self._pageinfo['textboxes']
if predicate(obj, visible, corrupt)
)
@property
def xres(self):
@@ -718,11 +725,15 @@ class PageInfo:
def __repr__(self):
return (
'<PageInfo '
'pageno={} {}"x{}" rotation={} res={}x{} has_text={}>').format(
self.pageno, self.width_inches, self.height_inches,
'<PageInfo ' 'pageno={} {}"x{}" rotation={} res={}x{} has_text={}>'
).format(
self.pageno,
self.width_inches,
self.height_inches,
self.rotation,
self.xres, self.yres, self.has_text
self.xres,
self.yres,
self.has_text,
)
@@ -732,7 +743,8 @@ class PdfInfo:
def __init__(self, infile, detailed_page_analysis=False, log=None):
self._infile = infile
self._pages, pdf = _pdf_get_all_pageinfo(
infile, detailed_page_analysis, log=log)
infile, detailed_page_analysis, log=log
)
self._needs_rendering = pdf.root.get('/NeedsRendering', False)
self._has_acroform = '/AcroForm' in pdf.root
@@ -772,13 +784,16 @@ class PdfInfo:
def __repr__(self):
return "<PdfInfo('...'), page count={}>".format(len(self))
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('infile')
args = parser.parse_args()
info = _pdf_get_all_pageinfo(args.infile)
from pprint import pprint
pprint(info)
src/ocrmypdf/pdfinfo/ghosttext.py
+8 -6
View File
@@ -23,13 +23,16 @@ from ..exec import ghostscript
# Forgive me for I have sinned
# I am using regular expressions to parse XML. However the XML in this case,
# generated by Ghostscript, is self-consistent enough to be parseable.
regex_remove_char_tags = re.compile(br"""
regex_remove_char_tags = re.compile(
br"""
<char\b
(?: [^>] # anything single character but >
| \">\" # special case: trap ">"
)*
/> # terminate with '/>'
""", re.VERBOSE)
""",
re.VERBOSE,
)
def page_get_textblocks(infile, pageno, xmltext, height):
@@ -76,15 +79,14 @@ def extract_text_xml(infile, pdf, pageno=None, log=None):
existing_text = regex_remove_char_tags.sub(b' ', existing_text)
try:
root = ET.fromstringlist([
b'<document>\n', existing_text, b'</document>\n'
])
root = ET.fromstringlist([b'<document>\n', existing_text, b'</document>\n'])
page_xml = root.findall('page')
except ET.ParseError as e:
log.error(
"An error occurred while attempting to retrieve existing text in "
"the input file. Will attempt to continue assuming that there is "
"no existing text in the file. The error was:")
"no existing text in the file. The error was:"
)
log.error(e)
page_xml = [None] * len(pdf.pages)
src/ocrmypdf/pdfinfo/layout.py
+84 -21
View File
@@ -25,11 +25,23 @@ import pdfminer.pdfdevice
import pdfminer.pdfinterp
from pdfminer.converter import PDFLayoutAnalyzer
from pdfminer.glyphlist import glyphname2unicode
from pdfminer.layout import (LAParams, LTChar, LTContainer, LTLayoutContainer,
LTPage, LTTextBox, LTTextLine)
from pdfminer.layout import (
LAParams,
LTChar,
LTContainer,
LTLayoutContainer,
LTPage,
LTTextBox,
LTTextLine,
)
from pdfminer.pdfdocument import PDFTextExtractionNotAllowed
from pdfminer.pdffont import (PDFCIDFont, PDFFont, PDFType3Font,
PDFUnicodeNotDefined, PDFSimpleFont)
from pdfminer.pdffont import (
PDFCIDFont,
PDFFont,
PDFType3Font,
PDFUnicodeNotDefined,
PDFSimpleFont,
)
from pdfminer.pdfpage import PDFPage
from pdfminer.utils import bbox2str, fsplit, matrix2str
@@ -41,6 +53,7 @@ STRIP_NAME = re.compile(r'[0-9]+')
# Unconditional pdfminer patches
#
def name2unicode(name):
"""Fix pdfminer's name2unicode function
@@ -62,9 +75,13 @@ def name2unicode(name):
if not m:
raise KeyError(name)
return chr(int(m.group(0)))
pdfminer.encodingdb.name2unicode = name2unicode
original_PDFFont_init = PDFFont.__init__
def PDFFont__init__(self, descriptor, widths, default_width=None):
original_PDFFont_init(self, descriptor, widths, default_width)
# PDF spec says descent should be negative
@@ -74,9 +91,13 @@ def PDFFont__init__(self, descriptor, widths, default_width=None):
# to misposition text.
if self.descent > 0:
self.descent = -self.descent
PDFFont.__init__ = PDFFont__init__
original_PDFSimpleFont_init = PDFSimpleFont.__init__
def PDFSimpleFont__init__(self, descriptor, widths, spec):
# Font encoding is specified either by a name of
# built-in encoding or a dictionary that describes
@@ -87,20 +108,25 @@ def PDFSimpleFont__init__(self, descriptor, widths, spec):
if not self.unicode_map and 'Encoding' not in spec:
self.cid2unicode = {}
return
PDFSimpleFont.__init__ = PDFSimpleFont__init__
#
# pdfminer patches when creator is PScript5.dll
#
def PDFType3Font__PScript5_get_height(self):
h = self.bbox[3]-self.bbox[1]
h = self.bbox[3] - self.bbox[1]
if h == 0:
h = self.ascent - self.descent
return h * copysign(1.0, self.vscale)
def PDFType3Font__PScript5_get_descent(self):
return self.descent * copysign(1.0, self.vscale)
def PDFType3Font__PScript5_get_ascent(self):
return self.ascent * copysign(1.0, self.vscale)
@@ -109,14 +135,38 @@ class LTStateAwareChar(LTChar):
"""A subclass of LTChar that tracks text render mode at time of drawing"""
__slots__ = (
'rendermode', '_text', 'matrix', 'fontname', 'adv', 'upright', 'size',
'width', 'height', 'bbox', 'x0', 'x1', 'y0', 'y1'
'rendermode',
'_text',
'matrix',
'fontname',
'adv',
'upright',
'size',
'width',
'height',
'bbox',
'x0',
'x1',
'y0',
'y1',
)
def __init__(self, matrix, font, fontsize, scaling, rise, text, textwidth,
textdisp, textstate, *args):
super().__init__(matrix, font, fontsize, scaling, rise, text, textwidth,
textdisp, *args)
def __init__(
self,
matrix,
font,
fontsize,
scaling,
rise,
text,
textwidth,
textdisp,
textstate,
*args,
):
super().__init__(
matrix, font, fontsize, scaling, rise, text, textwidth, textdisp, *args
)
self.rendermode = textstate.render
def is_compatible(self, obj):
@@ -126,8 +176,7 @@ class LTStateAwareChar(LTChar):
- the Unicode mapping is known, and both have the same render mode
- the Unicode mapping is unknown but both are part of the same font
"""
both_unicode_mapped = (isinstance(self._text, str) and
isinstance(obj._text, str))
both_unicode_mapped = isinstance(self._text, str) and isinstance(obj._text, str)
try:
if both_unicode_mapped:
return self.rendermode == obj.rendermode
@@ -143,10 +192,15 @@ class LTStateAwareChar(LTChar):
return self._text
def __repr__(self):
return ('<%s %s matrix=%s rendermode=%r font=%r adv=%s text=%r>' %
(self.__class__.__name__, bbox2str(self.bbox),
matrix2str(self.matrix), self.rendermode, self.fontname, self.adv,
self.get_text()))
return '<%s %s matrix=%s rendermode=%r font=%r adv=%s text=%r>' % (
self.__class__.__name__,
bbox2str(self.bbox),
matrix2str(self.matrix),
self.rendermode,
self.fontname,
self.adv,
self.get_text(),
)
class TextPositionTracker(PDFLayoutAnalyzer):
@@ -182,13 +236,22 @@ class TextPositionTracker(PDFLayoutAnalyzer):
textwidth = font.char_width(cid)
textdisp = font.char_disp(cid)
item = LTStateAwareChar(
matrix, font, fontsize, scaling, rise, text,
textwidth, textdisp, self.textstate, *args)
matrix,
font,
fontsize,
scaling,
rise,
text,
textwidth,
textdisp,
self.textstate,
*args,
)
self.cur_item.add(item)
return item.adv
def handle_undefined_char(self, font, cid):
#log.info('undefined: %r, %r', font, cid)
# log.info('undefined: %r, %r', font, cid)
return (font.fontname, cid)
def receive_layout(self, ltpage):
@@ -209,7 +272,7 @@ def get_page_analysis(infile, pageno, pscript5_mode):
spec=True,
get_ascent=PDFType3Font__PScript5_get_ascent,
get_descent=PDFType3Font__PScript5_get_descent,
get_height=PDFType3Font__PScript5_get_height
get_height=PDFType3Font__PScript5_get_height,
)
patcher.start()
tests/conftest.py
+24 -13
View File
@@ -144,27 +144,33 @@ def check_ocrmypdf(input_file, output_file, *args, env=None):
assert p.returncode == 0
assert os.path.exists(str(output_file)), "Output file not created"
assert os.stat(str(output_file)).st_size > 100, "PDF too small or empty"
assert out == "", \
"The following was written to stdout and should not have been: \n" + \
"<stdout>\n" + out + "\n</stdout>"
assert out == "", (
"The following was written to stdout and should not have been: \n"
+ "<stdout>\n"
+ out
+ "\n</stdout>"
)
return output_file
@pytest.helpers.register
def run_ocrmypdf(input_file, output_file, *args, env=None,
universal_newlines=True):
def run_ocrmypdf(input_file, output_file, *args, env=None, universal_newlines=True):
"Run ocrmypdf and let caller deal with results"
if env is None:
env = os.environ
p_args = OCRMYPDF + [str(arg) for arg in args] + \
[str(input_file), str(output_file)]
p_args = OCRMYPDF + [str(arg) for arg in args] + [str(input_file), str(output_file)]
p = Popen(
p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
universal_newlines=universal_newlines, env=env)
p_args,
close_fds=True,
stdout=PIPE,
stderr=PIPE,
universal_newlines=universal_newlines,
env=env,
)
out, err = p.communicate()
#print(err)
# print(err)
return p, out, err
@@ -172,6 +178,7 @@ def run_ocrmypdf(input_file, output_file, *args, env=None,
@pytest.helpers.register
def first_page_dimensions(pdf):
from ocrmypdf import pdfinfo
info = pdfinfo.PdfInfo(pdf)
page0 = info[0]
return (page0.width_inches, page0.height_inches)
@@ -179,9 +186,13 @@ def first_page_dimensions(pdf):
def pytest_addoption(parser):
parser.addoption(
"--runslow", action="store_true", default=False,
help=("run slow tests only useful for development (unlikely to be "
"useful for downstream packagers)")
"--runslow",
action="store_true",
default=False,
help=(
"run slow tests only useful for development (unlikely to be "
"useful for downstream packagers)"
),
)
tests/spoof/gs_feature_elision.py
+1
View File
@@ -53,5 +53,6 @@ def main():
sys.exit(0)
if __name__ == '__main__':
main()
tests/spoof/gs_render_failure.py
-1
View File
@@ -26,7 +26,6 @@ import sys
import os
def real_ghostscript(argv):
gs_args = ['gs'] + argv[1:]
os.execvp("gs", gs_args)
tests/spoof/tesseract_badutf8.py
+5 -2
View File
@@ -63,11 +63,14 @@ def main():
sys.exit(1)
elif sys.argv[-1] == 'stdout':
# input file is at sys.argv[-2] but we don't look at it
print("""Orientation: 0
print(
"""Orientation: 0
Orientation in degrees: 0
Orientation confidence: 100.00
Script: 1
Script confidence: 100.00""", file=sys.stderr)
Script confidence: 100.00""",
file=sys.stderr,
)
else:
print("Spoof doesn't understand arguments", file=sys.stderr)
print(sys.argv, file=sys.stderr)
tests/spoof/tesseract_big_image_error.py
+12 -6
View File
@@ -49,16 +49,22 @@ def main():
print('A parameter list would go here\ntextonly_pdf 0\n', file=sys.stderr)
sys.exit(0)
elif sys.argv[-2] == 'hocr':
print("Image too large: (33830, 14959)\n"
"Error during processing.", file=sys.stderr)
print(
"Image too large: (33830, 14959)\n" "Error during processing.",
file=sys.stderr,
)
sys.exit(1)
elif sys.argv[-2] == 'pdf':
print("Image too large: (33830, 14959)\n"
"Error during processing.", file=sys.stderr)
print(
"Image too large: (33830, 14959)\n" "Error during processing.",
file=sys.stderr,
)
sys.exit(1)
elif sys.argv[-1] == 'stdout':
print("Image too large: (33830, 14959)\n"
"Error during processing.", file=sys.stderr)
print(
"Image too large: (33830, 14959)\n" "Error during processing.",
file=sys.stderr,
)
sys.exit(1)
else:
print("Spoof doesn't understand arguments", file=sys.stderr)
tests/spoof/tesseract_cache.py
+16 -13
View File
@@ -63,12 +63,13 @@ if '_OCRMYPDF_SAVE_PATH' in os.environ:
os.environ['PATH'] = os.environ['_OCRMYPDF_SAVE_PATH']
__version__ = subprocess.check_output(
['tesseract', '--version'],
stderr=subprocess.STDOUT).decode()
['tesseract', '--version'], stderr=subprocess.STDOUT
).decode()
parser = argparse.ArgumentParser(
prog='tesseract-cache', description='cache output of tesseract')
prog='tesseract-cache', description='cache output of tesseract'
)
parser.add_argument('-l', '--language', action='append')
parser.add_argument('imagename')
parser.add_argument('outputbase')
@@ -82,6 +83,7 @@ parser.add_argument('--oem', type=int)
TESTS_ROOT = Path(__file__).resolve().parent.parent
CACHE_ROOT = TESTS_ROOT / 'cache'
def real_tesseract():
tess_args = ['tesseract'] + sys.argv[1:]
os.execvp("tesseract", tess_args)
@@ -89,8 +91,10 @@ def real_tesseract():
def main():
if any(opt in sys.argv[1:] for opt in (
'--print-parameters', '--list-langs', '--version')):
if any(
opt in sys.argv[1:]
for opt in ('--print-parameters', '--list-langs', '--version')
):
real_tesseract() # jump into real tesseract, replacing this process
# Convert non-standard but supported -psm to --psm
@@ -122,8 +126,7 @@ def main():
cache_folder = Path(CACHE_ROOT) / Path(source).stem / argv_slug
cache_folder.mkdir(parents=True, exist_ok=True)
print("Tesseract cache folder {} - ".format(cache_folder), end='',
file=sys.stderr)
print("Tesseract cache folder {} - ".format(cache_folder), end='', file=sys.stderr)
if (cache_folder / 'stderr.bin').exists() and not cache_disabled:
# Cache hit
@@ -138,8 +141,7 @@ def main():
for configfile in args.configfiles:
# cp cache -> output
tessfile = args.outputbase + '.' + configfile
shutil.copy(str(cache_folder / configfile) + '.bin',
tessfile)
shutil.copy(str(cache_folder / configfile) + '.bin', tessfile)
sys.exit(0)
# Cache miss
@@ -148,8 +150,8 @@ def main():
# Call tesseract
print(sys.argv[1:])
p = subprocess.run(
['tesseract'] + sys.argv[1:],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
['tesseract'] + sys.argv[1:], stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
sys.stdout.buffer.write(p.stdout)
sys.stderr.buffer.write(p.stderr)
@@ -179,10 +181,11 @@ def main():
manifest['python'] = platform.python_version()
manifest['argv_slug'] = argv_slug
manifest['sourcefile'] = str(Path(source).relative_to(TESTS_ROOT))
def clean_sys_argv():
for arg in sys.argv[1:]:
yield re.sub(r'.*/com.github.ocrmypdf[^/]+[/](.*)',
r'$TMPDIR/\1', arg)
yield re.sub(r'.*/com.github.ocrmypdf[^/]+[/](.*)', r'$TMPDIR/\1', arg)
manifest['args'] = list(clean_sys_argv())
# pylint: disable=E1101
tests/spoof/tesseract_crash.py
+6 -4
View File
@@ -53,8 +53,7 @@ def main():
print('List of available languages (1):\neng', file=sys.stderr)
sys.exit(0)
elif sys.argv[1] == '--print-parameters':
print('A parameter list would go here\ntextonly_pdf 0\n',
file=sys.stderr)
print('A parameter list would go here\ntextonly_pdf 0\n', file=sys.stderr)
sys.exit(0)
elif sys.argv[-2] == 'hocr':
print("KABOOM! Tesseract failed for some reason", file=sys.stderr)
@@ -63,8 +62,11 @@ def main():
print("KABOOM! Tesseract failed for some reason", file=sys.stderr)
sys.exit(128 + signal.SIGSEGV)
elif sys.argv[-1] == 'stdout':
print("libc++abi.dylib: terminating with uncaught exception of type "
"std::bad_alloc: std::bad_alloc", file=sys.stderr)
print(
"libc++abi.dylib: terminating with uncaught exception of type "
"std::bad_alloc: std::bad_alloc",
file=sys.stderr,
)
sys.exit(128 + signal.SIGABRT)
else:
print("Spoof doesn't understand arguments", file=sys.stderr)
tests/spoof/tesseract_noop.py
+8 -4
View File
@@ -83,8 +83,9 @@ def main():
elif sys.argv[-2] == 'hocr':
inputf = sys.argv[-4]
output = sys.argv[-3]
with Image.open(inputf) as im, \
open(output + '.hocr', 'w', encoding='utf-8') as f:
with Image.open(inputf) as im, open(
output + '.hocr', 'w', encoding='utf-8'
) as f:
w, h = im.size
f.write(HOCR_TEMPLATE.format(str(w), str(h)))
with open(output + '.txt', 'w') as f:
@@ -114,11 +115,14 @@ def main():
f.write('')
elif sys.argv[-1] == 'stdout':
inputf = sys.argv[-2]
print("""Orientation: 0
print(
"""Orientation: 0
Orientation in degrees: 0
Orientation confidence: 100.00
Script: 1
Script confidence: 100.00""", file=sys.stderr)
Script confidence: 100.00""",
file=sys.stderr,
)
else:
print("Spoof doesn't understand arguments", file=sys.stderr)
print(sys.argv, file=sys.stderr)
tests/spoof/unpaper_oldversion.py
+1
View File
@@ -23,6 +23,7 @@
import sys
def main():
if sys.argv[1] == '--version':
print('0.5')
tests/test_hocrtransform.py
+1 -2
View File
@@ -39,7 +39,6 @@ def test_mono_image(blank_hocr, outdir):
im.save(outdir / 'mono.tif', format='TIFF')
hocr = hocrtransform.HocrTransform(str(blank_hocr), 300)
hocr.to_pdf(
str(outdir / 'mono.pdf'), imageFileName=str(outdir / 'mono.tif'))
hocr.to_pdf(str(outdir / 'mono.pdf'), imageFileName=str(outdir / 'mono.tif'))
qpdf.check(str(outdir / 'mono.pdf'))
tests/test_lept.py
+1 -2
View File
@@ -87,5 +87,4 @@ def test_leptonica_compile(tmpdir):
# Compile the library but build it somewhere that won't interfere with
# existing compiled library. Also compile in API mode so that we test
# the interfaces, even though we use it ABI mode.
ffibuilder.compile(tmpdir=fspath(tmpdir),
target=fspath(tmpdir / 'lepttest.*'))
ffibuilder.compile(tmpdir=fspath(tmpdir), target=fspath(tmpdir / 'lepttest.*'))
tests/test_main.py
+401 -266
View File
File diff suppressed because it is too large Load Diff
tests/test_metadata.py
+58 -51
View File
@@ -29,11 +29,7 @@ import pikepdf
from pikepdf.models.metadata import decode_pdf_date
from ocrmypdf.exceptions import ExitCode
from ocrmypdf.pdfa import (
file_claims_pdfa,
generate_pdfa_ps,
SRGB_ICC_PROFILE
)
from ocrmypdf.pdfa import file_claims_pdfa, generate_pdfa_ps, SRGB_ICC_PROFILE
from ocrmypdf.exec import ghostscript
try:
@@ -52,17 +48,17 @@ run_ocrmypdf = pytest.helpers.run_ocrmypdf
spoof = pytest.helpers.spoof
@pytest.mark.parametrize("output_type", [
'pdfa', 'pdf'
])
def test_preserve_metadata(spoof_tesseract_noop, output_type,
resources, outpdf):
@pytest.mark.parametrize("output_type", ['pdfa', 'pdf'])
def test_preserve_metadata(spoof_tesseract_noop, output_type, resources, outpdf):
pdf_before = pikepdf.open(resources / 'graph.pdf')
output = check_ocrmypdf(
resources / 'graph.pdf', outpdf,
'--output-type', output_type,
env=spoof_tesseract_noop)
resources / 'graph.pdf',
outpdf,
'--output-type',
output_type,
env=spoof_tesseract_noop,
)
pdf_after = pikepdf.open(output)
@@ -73,21 +69,23 @@ def test_preserve_metadata(spoof_tesseract_noop, output_type,
assert pdfa_info['output'] == output_type
@pytest.mark.parametrize("output_type", [
'pdfa', 'pdf'
])
def test_override_metadata(spoof_tesseract_noop, output_type, resources,
outpdf):
@pytest.mark.parametrize("output_type", ['pdfa', 'pdf'])
def test_override_metadata(spoof_tesseract_noop, output_type, resources, outpdf):
input_file = resources / 'c02-22.pdf'
german = 'Du siehst den Wald vor lauter Bäumen nicht.'
chinese = '孔子'
p, out, err = run_ocrmypdf(
input_file, outpdf,
'--title', german,
'--author', chinese,
'--output-type', output_type,
env=spoof_tesseract_noop)
input_file,
outpdf,
'--title',
german,
'--author',
chinese,
'--output-type',
output_type,
env=spoof_tesseract_noop,
)
assert p.returncode == ExitCode.ok, err
@@ -114,10 +112,14 @@ def test_high_unicode(spoof_tesseract_noop, resources, no_outpdf):
high_unicode = 'U+1030C is: 𐌌'
p, out, err = run_ocrmypdf(
input_file, no_outpdf,
'--subject', high_unicode,
'--output-type', 'pdfa',
env=spoof_tesseract_noop)
input_file,
no_outpdf,
'--subject',
high_unicode,
'--output-type',
'pdfa',
env=spoof_tesseract_noop,
)
assert p.returncode == ExitCode.bad_args, err
@@ -125,16 +127,20 @@ def test_high_unicode(spoof_tesseract_noop, resources, no_outpdf):
@pytest.mark.skipif(not fitz, reason="test uses fitz")
@pytest.mark.parametrize('ocr_option', ['--skip-text', '--force-ocr'])
@pytest.mark.parametrize('output_type', ['pdf', 'pdfa'])
def test_bookmarks_preserved(spoof_tesseract_noop, output_type, ocr_option,
resources, outpdf):
def test_bookmarks_preserved(
spoof_tesseract_noop, output_type, ocr_option, resources, outpdf
):
input_file = resources / 'toc.pdf'
before_toc = fitz.Document(str(input_file)).getToC()
check_ocrmypdf(
input_file, outpdf,
input_file,
outpdf,
ocr_option,
'--output-type', output_type,
env=spoof_tesseract_noop)
'--output-type',
output_type,
env=spoof_tesseract_noop,
)
after_toc = fitz.Document(str(outpdf)).getToC()
print(before_toc)
@@ -148,13 +154,14 @@ def seconds_between_dates(date1, date2):
@pytest.mark.parametrize('infile', ['trivial.pdf', 'jbig2.pdf'])
@pytest.mark.parametrize('output_type', ['pdf', 'pdfa'])
def test_creation_date_preserved(spoof_tesseract_noop, output_type, resources,
infile, outpdf):
def test_creation_date_preserved(
spoof_tesseract_noop, output_type, resources, infile, outpdf
):
input_file = resources / infile
check_ocrmypdf(
input_file, outpdf, '--output-type', output_type,
env=spoof_tesseract_noop)
input_file, outpdf, '--output-type', output_type, env=spoof_tesseract_noop
)
pdf_before = pikepdf.open(input_file)
pdf_after = pikepdf.open(outpdf)
@@ -172,13 +179,11 @@ def test_creation_date_preserved(spoof_tesseract_noop, output_type, resources,
# We expect that the modified date is quite recent
date_after = decode_pdf_date(str(after['/ModDate']))
assert seconds_between_dates(
date_after, datetime.datetime.now(timezone.utc)) < 1000
assert seconds_between_dates(date_after, datetime.datetime.now(timezone.utc)) < 1000
@pytest.mark.parametrize('output_type', ['pdf', 'pdfa'])
def test_xml_metadata_preserved(spoof_tesseract_noop, output_type,
resources, outpdf):
def test_xml_metadata_preserved(spoof_tesseract_noop, output_type, resources, outpdf):
input_file = resources / 'graph.pdf'
try:
@@ -191,9 +196,8 @@ def test_xml_metadata_preserved(spoof_tesseract_noop, output_type,
before = file_to_dict(str(input_file))
check_ocrmypdf(
input_file, outpdf,
'--output-type', output_type,
env=spoof_tesseract_noop)
input_file, outpdf, '--output-type', output_type, env=spoof_tesseract_noop
)
after = file_to_dict(str(outpdf))
@@ -223,7 +227,7 @@ def test_xml_metadata_preserved(spoof_tesseract_noop, output_type,
'xmp:MetadataDate',
'xmp:CreatorTool',
'xmpMM:DocumentId',
'xmpMM:DnstanceId'
'xmpMM:DnstanceId',
]
# Cleanup messy data structure
@@ -251,8 +255,10 @@ def test_xml_metadata_preserved(spoof_tesseract_noop, output_type,
# of several
propidx = '{}[1]'.format(prop)
if propidx in before:
assert after.get(propidx) == before[propidx] \
or after.get(prop) == before[propidx]
assert (
after.get(propidx) == before[propidx]
or after.get(prop) == before[propidx]
)
def test_srgb_in_unicode_path(tmpdir):
@@ -270,9 +276,8 @@ def test_srgb_in_unicode_path(tmpdir):
def test_kodak_toc(resources, outpdf, spoof_tesseract_noop):
output = check_ocrmypdf(
resources / 'kcs.pdf', outpdf,
'--output-type', 'pdf',
env=spoof_tesseract_noop)
resources / 'kcs.pdf', outpdf, '--output-type', 'pdf', env=spoof_tesseract_noop
)
p = pikepdf.open(outpdf)
@@ -297,7 +302,8 @@ def test_metadata_fixup_warning(resources, outdir):
input_files_groups=input_files,
output_file=outdir / 'out.pdf',
log=log,
context=context)
context=context,
)
log.warning.assert_not_called()
# Now add some metadata that will not be copyable
@@ -312,5 +318,6 @@ def test_metadata_fixup_warning(resources, outdir):
input_files_groups=input_files,
output_file=outdir / 'out.pdf',
log=log,
context=context)
context=context,
)
log.warning.assert_called_once()
tests/test_optimize.py
+40 -13
View File
@@ -46,9 +46,12 @@ def test_mono_not_inverted(resources, outdir):
opt.main(infile, outdir / 'out.pdf', level=3)
rasterize_pdf(
outdir / 'out.pdf', outdir / 'im.png',
xres=10, yres=10, raster_device='pnggray',
log=logging.getLogger(name='test_mono_flip')
outdir / 'out.pdf',
outdir / 'im.png',
xres=10,
yres=10,
raster_device='pnggray',
log=logging.getLogger(name='test_mono_flip'),
)
im = Image.open(fspath(outdir / 'im.png'))
@@ -57,9 +60,17 @@ def test_mono_not_inverted(resources, outdir):
def test_jpg_png_params(resources, outpdf, spoof_tesseract_noop):
check_ocrmypdf(
resources / 'crom.png', outpdf, '--image-dpi', '200',
'--optimize', '3', '--jpg-quality', '50', '--png-quality', '20',
env=spoof_tesseract_noop
resources / 'crom.png',
outpdf,
'--image-dpi',
'200',
'--optimize',
'3',
'--jpg-quality',
'50',
'--png-quality',
'20',
env=spoof_tesseract_noop,
)
@@ -67,8 +78,16 @@ def test_jpg_png_params(resources, outpdf, spoof_tesseract_noop):
@pytest.mark.parametrize('lossy', [False, True])
def test_jbig2_lossy(lossy, resources, outpdf, spoof_tesseract_noop):
args = [
resources / 'ccitt.pdf', outpdf, '--image-dpi', '200',
'--optimize', 3, '--jpg-quality', '50', '--png-quality', '20'
resources / 'ccitt.pdf',
outpdf,
'--image-dpi',
'200',
'--optimize',
3,
'--jpg-quality',
'50',
'--png-quality',
'20',
]
if lossy:
args.append('--jbig2-lossy')
@@ -85,8 +104,10 @@ def test_jbig2_lossy(lossy, resources, outpdf, spoof_tesseract_noop):
assert len(pim.decode_parms) == 0
@pytest.mark.skipif(not jbig2enc.available() or not pngquant.available(),
reason='need jbig2enc and pngquant')
@pytest.mark.skipif(
not jbig2enc.available() or not pngquant.available(),
reason='need jbig2enc and pngquant',
)
def test_flate_to_jbig2(resources, outdir, spoof_tesseract_noop):
# This test requires an image that pngquant is capable of converting to
# to 1bpp - so use an existing 1bpp image, convert up, confirm it can
@@ -97,9 +118,15 @@ def test_flate_to_jbig2(resources, outdir, spoof_tesseract_noop):
im.save(fspath(outdir / 'type8.png'))
check_ocrmypdf(
outdir / 'type8.png', outdir / 'out.pdf',
'--image-dpi', '100', '--png-quality', '10', '--optimize', '3',
env=spoof_tesseract_noop
outdir / 'type8.png',
outdir / 'out.pdf',
'--image-dpi',
'100',
'--png-quality',
'10',
'--optimize',
'3',
env=spoof_tesseract_noop,
)
pdf = pikepdf.open(outdir / 'out.pdf')
tests/test_pdfinfo.py
+9 -13
View File
@@ -33,12 +33,13 @@ import pickle
def test_single_page_text(outdir):
filename = outdir / 'text.pdf'
pdf = Canvas(str(filename), pagesize=(8*72, 6*72))
pdf = Canvas(str(filename), pagesize=(8 * 72, 6 * 72))
text = pdf.beginText()
text.setFont('Helvetica', 12)
text.setTextOrigin(1*72, 3*72)
text.textLine("Methink'st thou art a general offence and every"
" man should beat thee.")
text.setTextOrigin(1 * 72, 3 * 72)
text.textLine(
"Methink'st thou art a general offence and every" " man should beat thee."
)
pdf.drawText(text)
pdf.showPage()
pdf.save()
@@ -66,8 +67,8 @@ def test_single_page_image(outdir):
im_bytes = im_tmp.read_bytes()
pdf_bytes = img2pdf.convert(
im_bytes, producer="img2pdf", with_pdfrw=False,
layout_fun=layout_fun)
im_bytes, producer="img2pdf", with_pdfrw=False, layout_fun=layout_fun
)
filename.write_bytes(pdf_bytes)
info = pdfinfo.PdfInfo(filename)
@@ -89,7 +90,7 @@ def test_single_page_image(outdir):
def test_single_page_inline_image(outdir):
filename = outdir / 'image-mono-inline.pdf'
pdf = Canvas(str(filename), pagesize=(8*72, 6*72))
pdf = Canvas(str(filename), pagesize=(8 * 72, 6 * 72))
with NamedTemporaryFile() as im_tmp:
im = Image.new('1', (8, 8), 0)
for n in range(8):
@@ -157,12 +158,7 @@ def test_regex():
b'<char bbox="0 108 0 108" c=">"/>',
b'<char bbox="0 108 0 108" c="X"/>',
]
must_not_match = [
b'<span stuff="c">',
b'<span>',
b'</span>',
b'</page>'
]
must_not_match = [b'<span stuff="c">', b'<span>', b'</span>', b'</page>']
for s in must_match:
assert rx.match(s)
tests/test_qpdf.py
+1
View File
@@ -19,6 +19,7 @@ import pytest
import ocrmypdf.exec.qpdf as qpdf
def test_qpdf_error(resources):
assert qpdf.check(resources / 'blank.pdf')
assert not qpdf.check(__file__)
tests/test_rotation.py
+83 -47
View File
@@ -36,7 +36,7 @@ from ocrmypdf.exec import ghostscript, tesseract
pytestmark = pytest.mark.skipif(
leptonica.get_leptonica_version() < 'leptonica-1.72',
reason="Leptonica is too old, correlation doesn't work"
reason="Leptonica is too old, correlation doesn't work",
)
check_ocrmypdf = pytest.helpers.check_ocrmypdf
@@ -47,23 +47,29 @@ RENDERERS = ['hocr', 'sandwich']
def check_monochrome_correlation(
outdir,
reference_pdf, reference_pageno, test_pdf, test_pageno):
outdir, reference_pdf, reference_pageno, test_pdf, test_pageno
):
gslog = logging.getLogger()
reference_png = outdir / '{}.ref{:04d}.png'.format(
reference_pdf.name, reference_pageno)
test_png = outdir / '{}.test{:04d}.png'.format(
test_pdf.name, test_pageno)
reference_pdf.name, reference_pageno
)
test_png = outdir / '{}.test{:04d}.png'.format(test_pdf.name, test_pageno)
def rasterize(pdf, pageno, png):
if png.exists():
print(png)
return
ghostscript.rasterize_pdf(
pdf, png, xres=100, yres=100,
raster_device='pngmono', log=gslog, pageno=pageno,
rotation=0)
pdf,
png,
xres=100,
yres=100,
raster_device='pngmono',
log=gslog,
pageno=pageno,
rotation=0,
)
rasterize(reference_pdf, reference_pageno, reference_png)
rasterize(test_pdf, test_pageno, test_png)
@@ -83,7 +89,7 @@ def test_monochrome_correlation(resources, outdir):
reference_pageno=1, # north facing page
test_pdf=resources / 'cardinal.pdf',
test_pageno=3, # south facing page
)
)
assert corr < 0.10
corr = check_monochrome_correlation(
outdir,
@@ -91,7 +97,7 @@ def test_monochrome_correlation(resources, outdir):
reference_pageno=2,
test_pdf=resources / 'cardinal.pdf',
test_pageno=2,
)
)
assert corr > 0.90
@@ -100,35 +106,55 @@ def test_monochrome_correlation(resources, outdir):
def test_autorotate(spoof_tesseract_cache, renderer, resources, outdir):
# cardinal.pdf contains four copies of an image rotated in each cardinal
# direction - these ones are "burned in" not tagged with /Rotate
out = check_ocrmypdf(resources / 'cardinal.pdf', outdir / 'out.pdf',
'-r', '-v', '1', '--pdf-renderer', renderer,
env=spoof_tesseract_cache)
for n in range(1, 4+1):
out = check_ocrmypdf(
resources / 'cardinal.pdf',
outdir / 'out.pdf',
'-r',
'-v',
'1',
'--pdf-renderer',
renderer,
env=spoof_tesseract_cache,
)
for n in range(1, 4 + 1):
correlation = check_monochrome_correlation(
outdir,
reference_pdf=resources / 'cardinal.pdf',
reference_pageno=1,
test_pdf=outdir / 'out.pdf',
test_pageno=n)
test_pageno=n,
)
assert correlation > 0.80
@pytest.mark.parametrize('threshold, correlation_test', [
('1', 'correlation > 0.80'), # Low thresh -> always rotate -> high corr
('99', 'correlation < 0.10'), # High thres -> never rotate -> low corr
])
@pytest.mark.parametrize(
'threshold, correlation_test',
[
('1', 'correlation > 0.80'), # Low thresh -> always rotate -> high corr
('99', 'correlation < 0.10'), # High thres -> never rotate -> low corr
],
)
def test_autorotate_threshold(
spoof_tesseract_cache, threshold, correlation_test, resources, outdir):
out = check_ocrmypdf(resources / 'cardinal.pdf', outdir / 'out.pdf',
'--rotate-pages-threshold', threshold,
'-r', '-v', '1', env=spoof_tesseract_cache)
spoof_tesseract_cache, threshold, correlation_test, resources, outdir
):
out = check_ocrmypdf(
resources / 'cardinal.pdf',
outdir / 'out.pdf',
'--rotate-pages-threshold',
threshold,
'-r',
'-v',
'1',
env=spoof_tesseract_cache,
)
correlation = check_monochrome_correlation(
outdir,
reference_pdf=resources / 'cardinal.pdf',
reference_pageno=1,
test_pdf=outdir / 'out.pdf',
test_pageno=3)
test_pageno=3,
)
assert eval(correlation_test)
@@ -143,27 +169,31 @@ def test_rotated_skew_timeout(resources, outpdf):
input_file = resources / 'rotated_skew.pdf'
in_pageinfo = PdfInfo(input_file)[0]
assert in_pageinfo.height_pixels < in_pageinfo.width_pixels, \
"Expected the input page to be landscape"
assert (
in_pageinfo.height_pixels < in_pageinfo.width_pixels
), "Expected the input page to be landscape"
assert in_pageinfo.rotation == 90, "Expected a rotated page"
out = check_ocrmypdf(
input_file, outpdf,
'--pdf-renderer', 'hocr',
'--deskew', '--tesseract-timeout', '0')
input_file,
outpdf,
'--pdf-renderer',
'hocr',
'--deskew',
'--tesseract-timeout',
'0',
)
out_pageinfo = PdfInfo(out)[0]
w, h = out_pageinfo.width_pixels, out_pageinfo.height_pixels
assert h > w, \
"Expected the output page to be portrait"
assert h > w, "Expected the output page to be portrait"
assert out_pageinfo.rotation == 0, \
"Expected no page rotation for output"
assert out_pageinfo.rotation == 0, "Expected no page rotation for output"
assert in_pageinfo.width_pixels == h and \
in_pageinfo.height_pixels == w, \
"Expected page rotation to be baked in"
assert (
in_pageinfo.width_pixels == h and in_pageinfo.height_pixels == w
), "Expected page rotation to be baked in"
def test_rotate_deskew_timeout(resources, outdir):
@@ -171,10 +201,13 @@ def test_rotate_deskew_timeout(resources, outdir):
resources / 'rotated_skew.pdf',
outdir / 'deskewed.pdf',
'--rotate-pages',
'--rotate-pages-threshold', '0',
'--rotate-pages-threshold',
'0',
'--deskew',
'--tesseract-timeout', '0',
'--pdf-renderer', 'sandwich'
'--tesseract-timeout',
'0',
'--pdf-renderer',
'sandwich',
)
correlation = check_monochrome_correlation(
@@ -182,7 +215,8 @@ def test_rotate_deskew_timeout(resources, outdir):
reference_pdf=resources / 'ccitt.pdf',
reference_pageno=1,
test_pdf=outdir / 'deskewed.pdf',
test_pageno=1)
test_pageno=1,
)
# Confirm that the page still got deskewed
assert correlation > 0.50
@@ -192,7 +226,6 @@ def test_rotate_deskew_timeout(resources, outdir):
@pytest.mark.parametrize('page_angle', (0, 90, 180, 270))
@pytest.mark.parametrize('image_angle', (0, 90, 180, 270))
def test_rotate_page_level(image_angle, page_angle, resources, outdir):
def make_rotate_test(prefix, image_angle, page_angle):
im = Image.open(fspath(resources / 'typewriter.png'))
if image_angle != 0:
@@ -205,7 +238,7 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir):
img2pdf.convert(
memimg.read(),
layout_fun=img2pdf.get_fixed_dpi_layout_fun((200, 200)),
outputstream=mempdf
outputstream=mempdf,
)
mempdf.seek(0)
pike = pikepdf.open(mempdf)
@@ -219,11 +252,13 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir):
out = test.with_suffix('.out.pdf')
p, _, err = run_ocrmypdf(
test, out,
test,
out,
'-O0',
'--rotate-pages',
'--rotate-pages-threshold', '0.001',
universal_newlines=False
'--rotate-pages-threshold',
'0.001',
universal_newlines=False,
)
err = err.decode('utf-8', errors='replace')
assert p.returncode == 0, err
@@ -238,4 +273,5 @@ def test_tesseract_orientation(resources, tmpdir):
log = Mock()
tesseract.get_orientation( # Test results of this are unreliable
tmpdir / '000001.png', engine_mode='3', timeout=10, log=log)
tmpdir / '000001.png', engine_mode='3', timeout=10, log=log
)
tests/test_tess4.py
+33 -15
View File
@@ -84,10 +84,11 @@ def tess4_available():
return False
# Skip all tests in this file if not tesseract 4
pytestmark = pytest.mark.skipif(
not tess4_available(),
reason="tesseract 4.0 with textonly_pdf feature required")
not tess4_available(), reason="tesseract 4.0 with textonly_pdf feature required"
)
check_ocrmypdf = pytest.helpers.check_ocrmypdf
run_ocrmypdf = pytest.helpers.run_ocrmypdf
@@ -97,9 +98,13 @@ spoof = pytest.helpers.spoof
def test_textonly_pdf(ensure_tess4, resources, outdir):
check_ocrmypdf(
resources / 'linn.pdf',
outdir / 'linn_textonly.pdf', '--pdf-renderer', 'sandwich',
'--sidecar', outdir / 'foo.txt',
env=ensure_tess4)
outdir / 'linn_textonly.pdf',
'--pdf-renderer',
'sandwich',
'--sidecar',
outdir / 'foo.txt',
env=ensure_tess4,
)
def test_pagesize_consistency_tess4(ensure_tess4, resources, outpdf):
@@ -111,9 +116,15 @@ def test_pagesize_consistency_tess4(ensure_tess4, resources, outpdf):
check_ocrmypdf(
infile,
outpdf, '--pdf-renderer', 'sandwich',
'--clean', '--deskew', '--remove-background', '--clean-final',
env=ensure_tess4)
outpdf,
'--pdf-renderer',
'sandwich',
'--clean',
'--deskew',
'--remove-background',
'--clean-final',
env=ensure_tess4,
)
after_dims = pytest.helpers.first_page_dimensions(outpdf)
@@ -122,16 +133,19 @@ def test_pagesize_consistency_tess4(ensure_tess4, resources, outpdf):
@pytest.mark.parametrize('basename', ['graph_ocred.pdf', 'cardinal.pdf'])
def test_skip_pages_does_not_replicate(
ensure_tess4, resources, basename, outdir):
def test_skip_pages_does_not_replicate(ensure_tess4, resources, basename, outdir):
infile = resources / basename
outpdf = outdir / basename
check_ocrmypdf(
infile,
outpdf, '--pdf-renderer', 'sandwich', '--force-ocr',
'--tesseract-timeout', '0',
env=ensure_tess4
outpdf,
'--pdf-renderer',
'sandwich',
'--force-ocr',
'--tesseract-timeout',
'0',
env=ensure_tess4,
)
info_in = pdfinfo.PdfInfo(infile)
@@ -149,8 +163,12 @@ def test_content_preservation(ensure_tess4, resources, outpdf):
check_ocrmypdf(
infile,
outpdf, '--pdf-renderer', 'sandwich', '--tesseract-timeout', '0',
env=ensure_tess4
outpdf,
'--pdf-renderer',
'sandwich',
'--tesseract-timeout',
'0',
env=ensure_tess4,
)
info = pdfinfo.PdfInfo(outpdf)
tests/test_unpaper.py
+7 -7
View File
@@ -27,27 +27,27 @@ check_ocrmypdf = pytest.helpers.check_ocrmypdf
run_ocrmypdf = pytest.helpers.run_ocrmypdf
spoof = pytest.helpers.spoof
@pytest.fixture(scope='session')
def spoof_unpaper_oldversion(tmpdir_factory):
return spoof(tmpdir_factory, unpaper='unpaper_oldversion.py')
@pytest.mark.skipif(True,
reason="needs new fixture implementation")
@pytest.mark.skipif(True, reason="needs new fixture implementation")
def test_no_unpaper(resources, no_outpdf):
# <disable unpaper here>
p, out, err = run_ocrmypdf(
resources / 'c02-22.pdf', no_outpdf, '--clean', env=os.environ)
resources / 'c02-22.pdf', no_outpdf, '--clean', env=os.environ
)
assert p.returncode == ExitCode.missing_dependency
def test_old_unpaper(spoof_unpaper_oldversion, resources, no_outpdf):
p, out, err = run_ocrmypdf(
resources / 'c02-22.pdf', no_outpdf, '--clean',
env=spoof_unpaper_oldversion)
resources / 'c02-22.pdf', no_outpdf, '--clean', env=spoof_unpaper_oldversion
)
assert p.returncode == ExitCode.missing_dependency
def test_clean(spoof_tesseract_noop, resources, outpdf):
check_ocrmypdf(resources / 'skew.pdf', outpdf, '-c',
env=spoof_tesseract_noop)
check_ocrmypdf(resources / 'skew.pdf', outpdf, '-c', env=spoof_tesseract_noop)
tests/test_userunit.py
+4 -5
View File
@@ -46,14 +46,13 @@ def test_userunit_ghostscript_fails(poster, no_outpdf):
def test_userunit_qpdf_passes(spoof_tesseract_cache, poster, outpdf):
before = PdfInfo(poster)
check_ocrmypdf(poster, outpdf, '--output-type=pdf',
env=spoof_tesseract_cache)
check_ocrmypdf(poster, outpdf, '--output-type=pdf', env=spoof_tesseract_cache)
after = PdfInfo(outpdf)
assert isclose(before[0].width_inches, after[0].width_inches)
def test_rotate_interaction(spoof_tesseract_cache, poster, outpdf):
check_ocrmypdf(poster, outpdf, '--output-type=pdf',
'--rotate-pages',
env=spoof_tesseract_cache)
check_ocrmypdf(
poster, outpdf, '--output-type=pdf', '--rotate-pages', env=spoof_tesseract_cache
)