diff --git a/src/ocrmypdf/__main__.py b/src/ocrmypdf/__main__.py
index f9c6dced..afa3cb20 100755
--- a/src/ocrmypdf/__main__.py
+++ b/src/ocrmypdf/__main__.py
@@ -462,9 +462,9 @@ debugging.add_argument(
action='store_true',
help="Keep temporary files (helpful for debugging)",
)
-debugging.add_argument(
- '--flowchart', type=str, help="Generate the pipeline execution flowchart"
-)
+# debugging.add_argument(
+# '--flowchart', type=str, help="Generate the pipeline execution flowchart"
+# )
def run(args=None):
diff --git a/src/ocrmypdf/_jobcontext.py b/src/ocrmypdf/_jobcontext.py
index c9367ba6..6b059644 100644
--- a/src/ocrmypdf/_jobcontext.py
+++ b/src/ocrmypdf/_jobcontext.py
@@ -17,62 +17,41 @@
import shutil
import sys
+import os
from contextlib import suppress
-from multiprocessing.managers import SyncManager
-
-from .pdfinfo import PdfInfo
-class JobContext:
- """Holds our context for a particular run of the pipeline
+class PDFContext:
+ """Holds our context for a particular run of the pipeline"""
- A multiprocessing manager effectively creates a separate process
- that keeps the master job context object. Other threads access
- job context via multiprocessing proxy objects.
-
- While this would naturally lend itself @property's it seems to make
- a little more sense to use functions to make it explicitly that the
- invocation requires marshalling data across a process boundary.
-
- """
-
- def __init__(self):
- self.pdfinfo = None
- self.options = None
- self.work_folder = None
- self.rotations = {}
-
- def generate_pdfinfo(self, infile):
- self.pdfinfo = PdfInfo(infile)
-
- def get_pdfinfo(self):
- "What we know about the input PDF"
- return self.pdfinfo
-
- def set_pdfinfo(self, pdfinfo):
- self.pdfinfo = pdfinfo
-
- def get_options(self):
- return self.options
-
- def set_options(self, options):
+ def __init__(self, options, work_folder, origin, pdfinfo):
self.options = options
-
- def get_work_folder(self):
- return self.work_folder
-
- def set_work_folder(self, work_folder):
self.work_folder = work_folder
+ self.origin = origin
+ self.pdfinfo = pdfinfo
+ self.log = get_logger(options, '%s: ' % os.path.basename(origin))
- def get_rotation(self, pageno):
- return self.rotations.get(pageno, 0)
+ def get_path(self, name):
+ return os.path.join(self.work_folder, name)
- def set_rotation(self, pageno, value):
- self.rotations[pageno] = value
+ def get_page_contexts(self):
+ npages = len(self.pdfinfo)
+ for n in range(npages):
+ yield PageContext(self, n)
-class JobContextManager(SyncManager):
- pass
+class PageContext:
+ """Holds our context for a page"""
+
+ def __init__(self, pdf_context, pageno):
+ self.pdf_context = pdf_context
+ self.options = pdf_context.options
+ self.pageno = pageno
+ self.pageinfo = pdf_context.pdfinfo[pageno]
+ self.log = get_logger(pdf_context.options, '%s Page %d: ' % (os.path.basename(pdf_context.origin), pageno + 1))
+
+ def get_path(self, name):
+ return os.path.join(self.pdf_context.work_folder, "page_%d_%s" % (self.pageno, name))
def cleanup_working_files(work_folder, options):
@@ -81,3 +60,62 @@ def cleanup_working_files(work_folder, options):
else:
with suppress(FileNotFoundError):
shutil.rmtree(work_folder)
+
+
+def get_logger(options=None, prefix=''):
+ level = INFO # TODO: add option
+ if options is not None and options.output_file == '-' or options.sidecar == '-':
+ return NullLogger()
+ return Logger(prefix, level)
+
+
+ERROR = 40
+WARN = 30
+INFO = 20
+DEBUG = 10
+
+
+class Logger:
+ def __init__(self, prefix, level=INFO):
+ self.prefix = prefix
+ self.level = level
+
+ def debug(self, *args, **kwargs):
+ if self.level <= DEBUG:
+ print('DEBUG', self.prefix, end='')
+ print(*args, **kwargs)
+
+ def info(self, *args, **kwargs):
+ if self.level <= INFO:
+ print('INFO', self.prefix, end='')
+ print(*args, **kwargs)
+
+ def warning(self, *args, **kwargs):
+ self.warn(*args, **kwargs)
+
+ def warn(self, *args, **kwargs):
+ if self.level <= WARN:
+ print('WARN', self.prefix, end='')
+ print(*args, **kwargs)
+
+ def error(self, *args, **kwargs):
+ if self.level <= ERROR:
+ print('ERROR', self.prefix, end='')
+ print(*args, **kwargs)
+
+
+class NullLogger:
+ def debug(self, *args, **kwargs):
+ pass
+
+ def info(self, *args, **kwargs):
+ pass
+
+ def warning(self, *args, **kwargs):
+ pass
+
+ def warn(self, *args, **kwargs):
+ pass
+
+ def error(self, *args, **kwargs):
+ pass
diff --git a/src/ocrmypdf/_pipeline.py b/src/ocrmypdf/_pipeline.py
index d6e0c491..3b553a7e 100644
--- a/src/ocrmypdf/_pipeline.py
+++ b/src/ocrmypdf/_pipeline.py
@@ -18,10 +18,8 @@
import os
import re
import sys
-from contextlib import suppress
from datetime import datetime, timezone
-from pathlib import Path
-from shutil import copyfile, copyfileobj
+from shutil import copyfileobj
import img2pdf
from PIL import Image
@@ -34,14 +32,10 @@ from .exceptions import (
DpiError,
EncryptedPdfError,
InputFileError,
- PriorOcrFoundError,
UnsupportedImageFormatError,
)
from .exec import ghostscript, tesseract
from .helpers import (
- flatten_groups,
- is_iterable_notstr,
- page_number,
re_symlink
)
from .hocrtransform import HocrTransform
@@ -51,10 +45,6 @@ from .pdfinfo import Colorspace, PdfInfo
VECTOR_PAGE_DPI = 400
-#
-# The Pipeline
-#
-
def triage_image_file(input_file, output_file, log, options):
try:
@@ -164,31 +154,28 @@ def triage(input_file, output_file, log, context):
triage_image_file(input_file, output_file, log, options)
-def repair_and_parse_pdf(input_file, output_file, log, context):
- options = context.get_options()
- copyfile(input_file, output_file)
-
- detailed_page_analysis = False
- if options.redo_ocr:
- detailed_page_analysis = True
-
+def get_pdfinfo(input_file, detailed_page_analysis=False):
try:
- pdfinfo = PdfInfo(
- output_file, detailed_page_analysis=detailed_page_analysis, log=log
+ return PdfInfo(
+ input_file, detailed_page_analysis=detailed_page_analysis
)
except pikepdf.PasswordError:
raise EncryptedPdfError()
- except pikepdf.PdfError as e:
- log.error(e)
+ except pikepdf.PdfError:
raise InputFileError()
+
+def validate_pdfinfo_options(context):
+ log = context.log
+ pdfinfo = context.pdfinfo
+ options = context.options
+
if pdfinfo.needs_rendering:
log.error(
"This PDF contains dynamic XFA forms created by Adobe LiveCycle "
"Designer and can only be read by Adobe Acrobat or Adobe Reader."
)
raise InputFileError()
-
if pdfinfo.has_userunit and options.output_type.startswith('pdfa'):
log.error(
"This input file uses a PDF feature that is not supported "
@@ -198,16 +185,15 @@ def repair_and_parse_pdf(input_file, output_file, log, context):
"output these files.) Use --output-type=pdf instead."
)
raise InputFileError()
-
if pdfinfo.has_acroform:
if options.redo_ocr:
log.error(
"This PDF has a user fillable form. --redo-ocr is not "
"currently possible on such files."
)
- raise PriorOcrFoundError()
+ raise InputFileError()
else:
- log.warning(
+ log.warn(
"This PDF has a fillable form. "
"Chances are it is a pure digital "
"document that does not need OCR."
@@ -219,16 +205,6 @@ def repair_and_parse_pdf(input_file, output_file, log, context):
"'flattened' and will no longer be fillable."
)
- context.set_pdfinfo(pdfinfo)
- log.debug(pdfinfo)
-
-
-def get_pageinfo(input_file, context):
- "Get zero-based page info implied by filename, e.g. 000002.pdf -> 1"
- pageno = page_number(input_file) - 1
- pageinfo = context.get_pdfinfo()[pageno]
- return pageinfo
-
def get_page_dpi(pageinfo, options):
"Get the DPI when nonsquare DPI is tolerable"
@@ -272,33 +248,31 @@ def get_canvas_square_dpi(pageinfo, options):
)
-def is_ocr_required(pageinfo, log, options):
- page = pageinfo.pageno + 1
+def is_ocr_required(page_context):
+ pageinfo = page_context.pageinfo
+ options = page_context.options
+ log = page_context.log
+
ocr_required = True
if pageinfo.has_text:
- prefix = f"{page:4d}: page already has text! - "
-
if not options.force_ocr and not (options.skip_text or options.redo_ocr):
- log.error(prefix + "aborting (use --force-ocr to force OCR)")
- raise PriorOcrFoundError()
+ log.error("page already has text! - aborting (use --force-ocr to force OCR)")
+ ocr_required = False
elif options.force_ocr:
- log.info(prefix + "rasterizing text and running OCR anyway")
+ log.info("page already has text! - rasterizing text and running OCR anyway")
ocr_required = True
elif options.redo_ocr:
if pageinfo.has_corrupt_text:
- log.warning(
- prefix + (
- "some text on this page cannot be mapped to characters: "
- "consider using --force-ocr instead",
- )
+ log.warn(
+ "some text on this page cannot be mapped to characters: "
+ "consider using --force-ocr instead",
)
- raise PriorOcrFoundError() # Wrong error but will do for now
else:
- log.info(prefix + "redoing OCR")
+ log.info("redoing OCR")
ocr_required = True
elif options.skip_text:
- log.info(prefix + "skipping all processing on this page")
+ log.info("skipping all processing on this page")
ocr_required = False
elif not pageinfo.images and not options.lossless_reconstruction:
# We found a page with no images and no text. That means it may
@@ -311,14 +285,14 @@ def is_ocr_required(pageinfo, log, options):
if options.force_ocr and options.oversample:
# The user really wants to reprocess this file
log.info(
- f"{page:4d}: page has no images - "
+ "page has no images - "
f"rasterizing at {options.oversample} DPI because "
"--force-ocr --oversample was specified"
)
elif options.force_ocr:
# Warn the user they might not want to do this
- log.warning(
- f"{page:4d}: page has no images - "
+ log.warn(
+ "page has no images - "
"all vector content will be "
f"rasterized at {VECTOR_PAGE_DPI} DPI, losing some resolution and likely "
"increasing file size. Use --oversample to adjust the "
@@ -326,7 +300,7 @@ def is_ocr_required(pageinfo, log, options):
)
else:
log.info(
- f"{page:4d}: page has no images - "
+ "page has no images - "
"skipping all processing on this page to avoid losing detail. "
"Use --force-ocr if you wish to perform OCR on pages that "
"have vector content."
@@ -337,82 +311,31 @@ def is_ocr_required(pageinfo, log, options):
pixel_count = pageinfo.width_pixels * pageinfo.height_pixels
if pixel_count > (options.skip_big * 1_000_000):
ocr_required = False
- log.warning(
- f"{page:4d}: page too big, skipping OCR "
+ log.warn(
+ "page too big, skipping OCR "
f"({(pixel_count / 1_000_000):.1f} MPixels > {options.skip_big:.1f} MPixels --skip-big)"
)
return ocr_required
-def marker_pages(input_files, output_files, log, context):
-
- options = context.get_options()
- work_folder = context.get_work_folder()
-
- if is_iterable_notstr(input_files):
- input_file = input_files[0]
- else:
- input_file = input_files
-
- for oo in output_files:
- with suppress(FileNotFoundError):
- os.unlink(oo)
-
- # If no files were repaired the input will be empty
- if not input_file:
- log.error(f"{options.input_file}: file not found or invalid argument")
- raise InputFileError()
-
- pdfinfo = context.get_pdfinfo()
- npages = len(pdfinfo)
-
- # Ruffus needs to see a file for any task it generates, so make very
- # file a symlink back to the source.
- for n in range(npages):
- page = Path(work_folder) / f'{(n + 1):06d}.marker.pdf'
- page.symlink_to(input_file) # pylint: disable=E1101
-
-
-def ocr_or_skip(input_files, output_files, log, context):
- options = context.get_options()
- work_folder = context.get_work_folder()
- pdfinfo = context.get_pdfinfo()
-
- for input_file in input_files:
- pageno = page_number(input_file) - 1
- pageinfo = pdfinfo[pageno]
- alt_suffix = (
- '.ocr.page.pdf'
- if is_ocr_required(pageinfo, log, options)
- else '.skip.page.pdf'
- )
-
- re_symlink(
- input_file,
- os.path.join(work_folder, os.path.basename(input_file)[0:6] + alt_suffix),
- log,
- )
-
-
-def rasterize_preview(input_file, output_file, log, context):
- pageinfo = get_pageinfo(input_file, context)
- options = context.get_options()
- canvas_dpi = get_canvas_square_dpi(pageinfo, options)
- page_dpi = get_page_square_dpi(pageinfo, options)
-
+def rasterize_preview(input_file, page_context):
+ output_file = page_context.get_path('rasterize_preview.jpg')
+ canvas_dpi = get_canvas_square_dpi(page_context.pageinfo, page_context.options)
+ page_dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
ghostscript.rasterize_pdf(
input_file,
output_file,
xres=canvas_dpi,
yres=canvas_dpi,
raster_device='jpeggray',
- log=log,
+ log=page_context.log,
page_dpi=(page_dpi, page_dpi),
- pageno=page_number(input_file),
+ pageno=page_context.pageinfo.pageno + 1,
)
+ return output_file
-def orient_page(infiles, output_file, log, context):
+def get_orientation_correction(preview, page_context):
"""
Work out orientation correct for each page.
@@ -430,32 +353,22 @@ def orient_page(infiles, output_file, log, context):
"""
- options = context.get_options()
- page_pdf = next(ii for ii in infiles if ii.endswith('.page.pdf'))
-
- if not options.rotate_pages:
- re_symlink(page_pdf, output_file, log)
- return
- preview = next(ii for ii in infiles if ii.endswith('.preview.jpg'))
-
orient_conf = tesseract.get_orientation(
preview,
- engine_mode=options.tesseract_oem,
- timeout=options.tesseract_timeout,
- log=log,
+ engine_mode=page_context.options.tesseract_oem,
+ timeout=page_context.options.tesseract_timeout,
+ log=page_context.log,
)
direction = {0: '⇧', 90: '⇨', 180: '⇩', 270: '⇦'}
- pageno = page_number(page_pdf) - 1
- pdfinfo = context.get_pdfinfo()
- existing_rotation = pdfinfo[pageno].rotation
+ existing_rotation = page_context.pageinfo.rotation
correction = orient_conf.angle % 360
apply_correction = False
action = ''
- if orient_conf.confidence >= options.rotate_pages_threshold:
+ if orient_conf.confidence >= page_context.options.rotate_pages_threshold:
if correction != 0:
apply_correction = True
action = ' - will rotate'
@@ -474,26 +387,25 @@ def orient_page(infiles, output_file, log, context):
)
facing += 'page is facing {}'.format(direction.get(orient_conf.angle, '?'))
- log.info(
+ page_context.log.debug(
'{pagenum:4d}: {facing}, confidence {conf:.2f}{action}'.format(
- pagenum=page_number(preview),
+ pagenum=page_context.pageinfo.pageno,
facing=facing,
conf=orient_conf.confidence,
action=action,
)
)
- re_symlink(page_pdf, output_file, log)
if apply_correction:
- context.set_rotation(pageno, correction)
+ return correction
+ return 0
-def rasterize_with_ghostscript(input_file, output_file, log, context):
- options = context.get_options()
- pageinfo = get_pageinfo(input_file, context)
-
+def rasterize(input_file, page_context, correction=0):
colorspaces = ['pngmono', 'pnggray', 'png256', 'png16m']
device_idx = 0
+ output_file = page_context.get_path('rasterize.png')
+ pageinfo = page_context.pageinfo
def at_least(cs):
return max(device_idx, colorspaces.index(cs))
@@ -511,14 +423,12 @@ def rasterize_with_ghostscript(input_file, output_file, log, context):
device = colorspaces[device_idx]
- log.debug(f"Rasterize {os.path.basename(input_file)} with {device}")
+ page_context.log.debug(f"Rasterize with {device}")
# Produce the page image with square resolution or else deskew and OCR
# will not work properly.
- canvas_dpi = get_canvas_square_dpi(pageinfo, options)
- page_dpi = get_page_square_dpi(pageinfo, options)
-
- correction = context.get_rotation(page_number(input_file) - 1)
+ canvas_dpi = get_canvas_square_dpi(pageinfo, page_context.options)
+ page_dpi = get_page_square_dpi(pageinfo, page_context.options)
ghostscript.rasterize_pdf(
input_file,
@@ -526,64 +436,47 @@ def rasterize_with_ghostscript(input_file, output_file, log, context):
xres=canvas_dpi,
yres=canvas_dpi,
raster_device=device,
- log=log,
+ log=page_context.log,
page_dpi=(page_dpi, page_dpi),
- pageno=page_number(input_file),
+ pageno=pageinfo.pageno + 1,
rotation=correction,
- filter_vector=options.remove_vectors,
+ filter_vector=page_context.options.remove_vectors,
)
+ return output_file
-def preprocess_remove_background(input_file, output_file, log, context):
- options = context.get_options()
- if not options.remove_background:
- re_symlink(input_file, output_file, log)
- return
-
- pageinfo = get_pageinfo(input_file, context)
-
- if any(image.bpc > 1 for image in pageinfo.images):
+def preprocess_remove_background(input_file, page_context):
+ if any(image.bpc > 1 for image in page_context.pageinfo.images):
+ output_file = page_context.get_path('pp_rm_bg.png')
leptonica.remove_background(input_file, output_file)
+ return output_file
else:
- log.info(f"{pageinfo.pageno:4d}: background removal skipped on mono page")
- re_symlink(input_file, output_file, log)
+ page_context.log.info("background removal skipped on mono page")
+ return input_file
-def preprocess_deskew(input_file, output_file, log, context):
- options = context.get_options()
- if not options.deskew:
- re_symlink(input_file, output_file, log)
- return
-
- pageinfo = get_pageinfo(input_file, context)
- dpi = get_page_square_dpi(pageinfo, options)
-
+def preprocess_deskew(input_file, page_context):
+ output_file = page_context.get_path('pp_deskew.png')
+ dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
leptonica.deskew(input_file, output_file, dpi)
+ return output_file
-def preprocess_clean(input_file, output_file, log, context):
- options = context.get_options()
- if not options.clean:
- re_symlink(input_file, output_file, log)
- return
-
+def preprocess_clean(input_file, page_context):
from .exec import unpaper
-
- pageinfo = get_pageinfo(input_file, context)
- dpi = get_page_square_dpi(pageinfo, options)
-
- unpaper.clean(input_file, output_file, dpi, log, options.unpaper_args)
+ output_file = page_context.get_path('pp_clean.png')
+ dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
+ unpaper.clean(input_file, output_file, dpi, page_context.log, page_context.options.unpaper_args)
+ return output_file
-def select_ocr_image(infiles, output_file, log, context):
- """Select the image we send for OCR. May not be the same as the display
+def create_ocr_image(image, page_context):
+ """Create the image we send for OCR. May not be the same as the display
image depending on preprocessing. This image will never be shown to the
user."""
- image = infiles[0]
- options = context.get_options()
- pageinfo = get_pageinfo(image, context)
-
+ output_file = page_context.get_path('ocr.png')
+ options = page_context.options
with Image.open(image) as im:
from PIL import ImageColor
from PIL import ImageDraw
@@ -593,7 +486,7 @@ def select_ocr_image(infiles, output_file, log, context):
draw = ImageDraw.ImageDraw(im)
xres, yres = im.info['dpi']
- log.debug('resolution %r %r', xres, yres)
+ page_context.log.info('resolution %r %r' % (xres, yres))
if not options.force_ocr:
# Do not mask text areas when forcing OCR, because we need to OCR
@@ -602,7 +495,7 @@ def select_ocr_image(infiles, output_file, log, context):
if options.redo_ocr:
mask = True # Mask visible text, but not invisible text
- for textarea in pageinfo.get_textareas(visible=mask, corrupt=None):
+ for textarea in page_context.pageinfo.get_textareas(visible=mask, corrupt=None):
# Calculate resolution based on the image size and page dimensions
# without regard whatever resolution is in pageinfo (may differ or
# be None)
@@ -615,7 +508,7 @@ def select_ocr_image(infiles, output_file, log, context):
im.height - bbox[1] * yscale,
]
pixcoords = [int(round(c)) for c in pixcoords]
- log.debug('blanking %r', pixcoords)
+ print('blanking %r', pixcoords)
draw.rectangle(pixcoords, fill=white)
# draw.rectangle(pixcoords, outline=pink)
@@ -627,7 +520,7 @@ def select_ocr_image(infiles, output_file, log, context):
barcodes = pix.locate_barcodes()
for barcode in barcodes:
decoded, rect = barcode
- log.info('masking barcode %s %r', decoded, rect)
+ print('masking barcode %s %r', decoded, rect)
draw.rectangle(rect, fill=white)
im = pix.topil()
@@ -635,13 +528,16 @@ def select_ocr_image(infiles, output_file, log, context):
# Pillow requires integer DPI
dpi = round(xres), round(yres)
im.save(output_file, dpi=dpi)
+ return output_file
-def ocr_tesseract_hocr(input_file, output_files, log, context):
- options = context.get_options()
+def ocr_tesseract_hocr(input_file, page_context):
+ hocr_out = page_context.get_path('ocr_hocr.hocr')
+ hocr_text_out = page_context.get_path('ocr_hocr.txt')
+ options = page_context.options
tesseract.generate_hocr(
input_file=input_file,
- output_files=output_files,
+ output_files=[hocr_out, hocr_text_out],
language=options.language,
engine_mode=options.tesseract_oem,
tessconfig=options.tesseract_config,
@@ -649,86 +545,57 @@ def ocr_tesseract_hocr(input_file, output_files, log, context):
pagesegmode=options.tesseract_pagesegmode,
user_words=options.user_words,
user_patterns=options.user_patterns,
- log=log,
+ log=page_context.log,
)
+ return (hocr_out, hocr_text_out)
-def select_visible_page_image(infiles, output_file, log, context):
- """Selects a whole page image that we can show the user (if necessary)"""
-
- options = context.get_options()
- if options.clean_final:
- image_suffix = '.pp-clean.png'
- elif options.deskew:
- image_suffix = '.pp-deskew.png'
- elif options.remove_background:
- image_suffix = '.pp-background.png'
- else:
- image_suffix = '.page.png'
- image = next(ii for ii in infiles if ii.endswith(image_suffix))
-
- pageinfo = get_pageinfo(image, context)
- if pageinfo.images and all(im.enc == 'jpeg' for im in pageinfo.images):
- log.debug(f'{page_number(image):4d}: JPEG input -> JPEG output')
- # If all images were JPEGs originally, produce a JPEG as output
- with Image.open(image) as im:
- # At this point the image should be a .png, but deskew, unpaper
- # might have removed the DPI information. In this case, fall back to
- # square DPI used to rasterize. When the preview image was
- # rasterized, it was also converted to square resolution, which is
- # what we want to give tesseract, so keep it square.
- fallback_dpi = get_page_square_dpi(pageinfo, options)
- dpi = im.info.get('dpi', (fallback_dpi, fallback_dpi))
-
- # Pillow requires integer DPI
- dpi = round(dpi[0]), round(dpi[1])
- im.save(output_file, format='JPEG', dpi=dpi)
- else:
- re_symlink(image, output_file, log)
+def should_visible_page_image_use_jpg(pageinfo):
+ # If all images were JPEGs originally, produce a JPEG as output
+ return pageinfo.images and all(im.enc == 'jpeg' for im in pageinfo.images)
-def select_image_layer(infiles, output_file, log, context):
- """Selects the image layer for the output page. If possible this is the
- orientation-corrected input page, or an image of the whole page converted
- to PDF."""
+def create_visible_page_jpg(image, page_context):
+ output_file = page_context.get_path('visible.jpg')
+ with Image.open(image) as im:
+ # At this point the image should be a .png, but deskew, unpaper
+ # might have removed the DPI information. In this case, fall back to
+ # square DPI used to rasterize. When the preview image was
+ # rasterized, it was also converted to square resolution, which is
+ # what we want to give tesseract, so keep it square.
+ fallback_dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
+ dpi = im.info.get('dpi', (fallback_dpi, fallback_dpi))
- options = context.get_options()
- page_pdf = next(ii for ii in infiles if ii.endswith('.ocr.oriented.pdf'))
- image = next(ii for ii in infiles if ii.endswith('.image'))
+ # Pillow requires integer DPI
+ dpi = round(dpi[0]), round(dpi[1])
+ im.save(output_file, format='JPEG', dpi=dpi)
+ return output_file
- if options.lossless_reconstruction:
- log.debug(
- f"{page_number(page_pdf):4d}: page eligible for lossless reconstruction"
- )
- re_symlink(page_pdf, output_file, log) # Still points to multipage
- return
-
- pageinfo = get_pageinfo(image, context)
+def create_pdf_page_from_image(image, page_context):
# We rasterize a square DPI version of each page because most image
# processing tools don't support rectangular DPI. Use the square DPI as it
# accurately describes the image. It would be possible to resample the image
# at this stage back to non-square DPI to more closely resemble the input,
# except that the hocr renderer does not understand non-square DPI. The
# sandwich renderer would be fine.
- dpi = get_page_square_dpi(pageinfo, options)
+ output_file = page_context.get_path('visible.pdf')
+ dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
layout_fun = img2pdf.get_fixed_dpi_layout_fun((dpi, dpi))
# This create a single page PDF
with open(image, 'rb') as imfile, open(output_file, 'wb') as pdf:
- log.debug(f'{page_number(page_pdf):4d}: convert')
+ page_context.log.debug('convert')
img2pdf.convert(
imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf
)
- log.debug(f'{page_number(page_pdf):4d}: convert done')
+ page_context.log.debug('convert done')
+ return output_file
-def render_hocr_page(infiles, output_file, log, context):
- options = context.get_options()
- hocr = next(ii for ii in infiles if ii.endswith('.hocr'))
- pageinfo = get_pageinfo(hocr, context)
- dpi = get_page_square_dpi(pageinfo, options)
-
+def render_hocr_page(hocr, page_context):
+ output_file = page_context.get_path('ocr_hocr.pdf')
+ dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
hocrtransform = HocrTransform(hocr, dpi)
hocrtransform.to_pdf(
output_file,
@@ -737,17 +604,13 @@ def render_hocr_page(infiles, output_file, log, context):
invisibleText=True,
interwordSpaces=True,
)
+ return output_file
-def ocr_tesseract_textonly_pdf(infiles, outfiles, log, context):
- options = context.get_options()
- input_image = next((ii for ii in infiles if ii.endswith('.ocr.png')), '')
- if not input_image:
- raise ValueError("No image rendered?")
-
- output_pdf = next((ii for ii in outfiles if ii.endswith('.pdf')))
- output_text = next((ii for ii in outfiles if ii.endswith('.txt')))
-
+def ocr_tesseract_textonly_pdf(input_image, page_context):
+ output_pdf = page_context.get_path('ocr_tess.pdf')
+ output_text = page_context.get_path('ocr_tess.txt')
+ options = page_context.options
tesseract.generate_pdf(
input_image=input_image,
skip_pdf=None,
@@ -761,8 +624,9 @@ def ocr_tesseract_textonly_pdf(infiles, outfiles, log, context):
pagesegmode=options.tesseract_pagesegmode,
user_words=options.user_words,
user_patterns=options.user_patterns,
- log=log,
+ log=page_context.log,
)
+ return (output_pdf, output_text)
def get_docinfo(base_pdf, options):
@@ -804,63 +668,51 @@ def get_docinfo(base_pdf, options):
return pdfmark
-def generate_postscript_stub(input_file, output_file, log, context):
+def generate_postscript_stub(context):
+ output_file = context.get_path('pdfa.ps')
generate_pdfa_ps(output_file)
+ return output_file
-def convert_to_pdfa(input_files_groups, output_file, log, context):
- options = context.get_options()
- input_pdfinfo = context.get_pdfinfo()
-
- input_files = list(f for f in flatten_groups(input_files_groups))
- layers_file = next(
- (ii for ii in input_files if ii.endswith('layers.rendered.pdf')), None
- )
+def convert_to_pdfa(input_pdf, input_ps_stub, context):
+ options = context.options
+ input_pdfinfo = context.pdfinfo
+ output_file = context.get_path('pdfa.pdf')
# If the DocumentInfo record contains NUL characters, Ghostscript will
# produce XMP metadata which contains invalid XML entities (�).
# NULs in DocumentInfo seem to be common since older Acrobats included them.
# pikepdf can deal with this, but we make the world a better place by
# stamping them out as soon as possible.
- pdf_layers_file = pikepdf.open(layers_file)
- if pdf_layers_file.docinfo:
+ pdf_file = pikepdf.open(input_pdf)
+ if pdf_file.docinfo:
modified = False
- for k, v in pdf_layers_file.docinfo.items():
+ for k, v in pdf_file.docinfo.items():
if b'\x00' in bytes(v):
- pdf_layers_file.docinfo[k] = bytes(v).replace(b'\x00', b'')
+ pdf_file.docinfo[k] = bytes(v).replace(b'\x00', b'')
modified = True
if modified:
- pdf_layers_file.save(layers_file)
- del pdf_layers_file
+ pdf_file.save(input_pdf)
+ del pdf_file
- ps = next((ii for ii in input_files if ii.endswith('.ps')), None)
ghostscript.generate_pdfa(
pdf_version=input_pdfinfo.min_version,
- pdf_pages=[layers_file, ps],
+ pdf_pages=[input_pdf, input_ps_stub],
output_file=output_file,
compression=options.pdfa_image_compression,
- log=log,
+ log=context.log,
threads=options.jobs or 1,
pdfa_part=options.output_type[-1], # is pdfa-1, pdfa-2, or pdfa-3
)
+ return output_file
-def metadata_fixup(input_files_groups, output_file, log, context):
- options = context.get_options()
- input_files = list(f for f in flatten_groups(input_files_groups))
- original_file = next(
- (ii for ii in input_files if ii.endswith('.repaired.pdf')), None
- )
- layers_file = next(
- (ii for ii in input_files if ii.endswith('layers.rendered.pdf')), None
- )
- pdfa_file = next((ii for ii in input_files if ii.endswith('pdfa.pdf')), None)
- original = pikepdf.open(original_file)
+def metadata_fixup(working_file, context):
+ output_file = context.get_path('metafix.pdf')
+ options = context.options
+ original = pikepdf.open(context.origin)
docinfo = get_docinfo(original, options)
-
- working_file = pdfa_file if pdfa_file else layers_file
-
pdf = pikepdf.open(working_file)
with pdf.open_metadata() as meta:
meta.load_from_docinfo(docinfo, delete_missing=False)
@@ -868,16 +720,25 @@ def metadata_fixup(input_files_groups, output_file, log, context):
# match Ghostscript, for consistency
if 'xmp:CreateDate' not in meta:
meta['xmp:CreateDate'] = meta.get('xmp:ModifyDate', '')
- if pdfa_file:
- meta_original = original.open_metadata()
- not_copied = set(meta_original.keys()) - set(meta.keys())
- if not_copied:
- log.warning(
+
+ meta_original = original.open_metadata()
+ not_copied = set(meta_original.keys()) - set(meta.keys())
+ if not_copied:
+ if options.output_type.startswith('pdfa'):
+ context.log.warn(
"Some input metadata could not be copied because it is not "
"permitted in PDF/A. You may wish to examine the output "
"PDF's XMP metadata."
)
- log.debug(
+ context.log.debug(
+ "The following metadata fields were not copied: %r", not_copied
+ )
+ else:
+ context.log.error(
+ "Some input metadata could not be copied."
+ "You may wish to examine the output PDF's XMP metadata."
+ )
+ context.log.info(
"The following metadata fields were not copied: %r", not_copied
)
@@ -886,23 +747,18 @@ def metadata_fixup(input_files_groups, output_file, log, context):
compress_streams=True,
object_stream_mode=pikepdf.ObjectStreamMode.generate,
)
+ return output_file
-def optimize_pdf(input_file, output_file, log, context):
- optimize(input_file, output_file, log, context)
+def optimize_pdf(input_file, context):
+ output_file = context.get_path('optimize.pdf')
+ optimize(input_file, output_file, context)
+ return output_file
-def merge_sidecars(input_files_groups, output_file, log, context):
- pdfinfo = context.get_pdfinfo()
-
- txt_files = [None] * len(pdfinfo)
-
- for infile in flatten_groups(input_files_groups):
- if infile.endswith('.txt'):
- idx = page_number(infile) - 1
- txt_files[idx] = infile
-
- def write_pages(stream):
+def merge_sidecars(txt_files, context):
+ output_file = context.get_path('sidecar.txt')
+ with open(output_file, 'w', encoding="utf-8") as stream:
for page_num, txt_file in enumerate(txt_files):
if page_num != 0:
stream.write('\f') # Form feed between pages
@@ -920,18 +776,11 @@ def merge_sidecars(input_files_groups, output_file, log, context):
stream.write(txt)
else:
stream.write(f'[OCR skipped on page {(page_num + 1)}]')
-
- if output_file == '-':
- write_pages(sys.stdout)
- sys.stdout.flush()
- else:
- with open(output_file, 'w', encoding="utf-8") as out:
- write_pages(out)
+ return output_file
-def copy_final(input_files, output_file, log, context):
- input_file = next((ii for ii in input_files if ii.endswith('.pdf')))
- log.debug('%s -> %s', input_file, output_file)
+def copy_final(input_file, output_file, context):
+ context.log.debug('%s -> %s', input_file, output_file)
with open(input_file, 'rb') as input_stream:
if output_file == '-':
copyfileobj(input_stream, sys.stdout.buffer)
diff --git a/src/ocrmypdf/_pipeline_simple.py b/src/ocrmypdf/_pipeline_simple.py
deleted file mode 100644
index 3b89df3d..00000000
--- a/src/ocrmypdf/_pipeline_simple.py
+++ /dev/null
@@ -1,784 +0,0 @@
-# © 2016 James R. Barlow: github.com/jbarlow83
-#
-# This file is part of OCRmyPDF.
-#
-# OCRmyPDF is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# OCRmyPDF is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with OCRmyPDF. If not, see .
-
-import os
-import re
-import sys
-from datetime import datetime, timezone
-from shutil import copyfileobj
-
-import img2pdf
-from PIL import Image
-
-import pikepdf
-from pikepdf.models.metadata import encode_pdf_date
-
-from . import PROGRAM_NAME, VERSION, leptonica
-from .exceptions import (
- DpiError,
- EncryptedPdfError,
- InputFileError,
- UnsupportedImageFormatError,
-)
-from .exec import ghostscript, tesseract
-from .helpers import (
- re_symlink
-)
-from .hocrtransform import HocrTransform
-from .optimize import optimize
-from .pdfa import generate_pdfa_ps
-from .pdfinfo import Colorspace, PdfInfo
-
-VECTOR_PAGE_DPI = 400
-
-
-def triage_image_file(input_file, output_file, log, options):
- try:
- log.info("Input file is not a PDF, checking if it is an image...")
- im = Image.open(input_file)
- except EnvironmentError as e:
- msg = str(e)
-
- # Recover the original filename
- realpath = ''
- if os.path.islink(input_file):
- realpath = os.path.realpath(input_file)
- elif os.path.isfile(input_file):
- realpath = ''
- msg = msg.replace(input_file, realpath)
- log.error(msg)
- raise UnsupportedImageFormatError() from e
- else:
- log.info("Input file is an image")
-
- if 'dpi' in im.info:
- if im.info['dpi'] <= (96, 96) and not options.image_dpi:
- log.info("Image size: (%d, %d)" % im.size)
- log.info("Image resolution: (%d, %d)" % im.info['dpi'])
- log.error(
- "Input file is an image, but the resolution (DPI) is "
- "not credible. Estimate the resolution at which the "
- "image was scanned and specify it using --image-dpi."
- )
- raise DpiError()
- elif not options.image_dpi:
- log.info("Image size: (%d, %d)" % im.size)
- log.error(
- "Input file is an image, but has no resolution (DPI) "
- "in its metadata. Estimate the resolution at which "
- "image was scanned and specify it using --image-dpi."
- )
- raise DpiError()
-
- if im.mode in ('RGBA', 'LA'):
- log.error(
- "The input image has an alpha channel. Remove the alpha "
- "channel first."
- )
- raise UnsupportedImageFormatError()
-
- if 'iccprofile' not in im.info:
- if im.mode == 'RGB':
- log.info('Input image has no ICC profile, assuming sRGB')
- elif im.mode == 'CMYK':
- log.info('Input CMYK image has no ICC profile, not usable')
- raise UnsupportedImageFormatError()
- im.close()
-
- try:
- log.info("Image seems valid. Try converting to PDF...")
- layout_fun = img2pdf.default_layout_fun
- if options.image_dpi:
- layout_fun = img2pdf.get_fixed_dpi_layout_fun(
- (options.image_dpi, options.image_dpi)
- )
- with open(output_file, 'wb') as outf:
- img2pdf.convert(
- input_file,
- layout_fun=layout_fun,
- with_pdfrw=False,
- outputstream=outf
- )
- log.info("Successfully converted to PDF, processing...")
- except img2pdf.ImageOpenError as e:
- log.error(e)
- raise UnsupportedImageFormatError() from e
-
-
-def _pdf_guess_version(input_file, search_window=1024):
- """Try to find version signature at start of file.
-
- Not robust enough to deal with appended files.
-
- Returns empty string if not found, indicating file is probably not PDF.
- """
-
- with open(input_file, 'rb') as f:
- signature = f.read(search_window)
- m = re.search(br'%PDF-(\d\.\d)', signature)
- if m:
- return m.group(1)
- return ''
-
-
-def triage(input_file, output_file, log, context):
-
- options = context.get_options()
- try:
- if _pdf_guess_version(input_file):
- if options.image_dpi:
- log.warning(
- "Argument --image-dpi ignored because the "
- "input file is a PDF, not an image."
- )
- re_symlink(input_file, output_file, log)
- return
- except EnvironmentError as e:
- log.error(e)
- raise InputFileError() from e
-
- triage_image_file(input_file, output_file, log, options)
-
-
-def get_pdfinfo(input_file, detailed_page_analysis=False):
- try:
- return PdfInfo(
- input_file, detailed_page_analysis=detailed_page_analysis
- )
- except pikepdf.PasswordError:
- raise EncryptedPdfError()
- except pikepdf.PdfError:
- raise InputFileError()
-
-
-def validate_pdfinfo_options(context):
- log = context.log
- pdfinfo = context.pdfinfo
- options = context.options
-
- if pdfinfo.needs_rendering:
- log.error(
- "This PDF contains dynamic XFA forms created by Adobe LiveCycle "
- "Designer and can only be read by Adobe Acrobat or Adobe Reader."
- )
- raise InputFileError()
- if pdfinfo.has_userunit and options.output_type.startswith('pdfa'):
- log.error(
- "This input file uses a PDF feature that is not supported "
- "by Ghostscript, so you cannot use --output-type=pdfa for this "
- "file. (Specifically, it uses the PDF-1.6 /UserUnit feature to "
- "support very large or small page sizes, and Ghostscript cannot "
- "output these files.) Use --output-type=pdf instead."
- )
- raise InputFileError()
- if pdfinfo.has_acroform:
- if options.redo_ocr:
- log.error(
- "This PDF has a user fillable form. --redo-ocr is not "
- "currently possible on such files."
- )
- raise InputFileError()
- else:
- log.warn(
- "This PDF has a fillable form. "
- "Chances are it is a pure digital "
- "document that does not need OCR."
- )
- if not options.force_ocr:
- log.info(
- "Use the option --force-ocr to produce an image of the "
- "form and all filled form fields. The output PDF will be "
- "'flattened' and will no longer be fillable."
- )
-
-
-def get_page_dpi(pageinfo, options):
- "Get the DPI when nonsquare DPI is tolerable"
- xres = max(
- pageinfo.xres or VECTOR_PAGE_DPI,
- options.oversample or 0,
- VECTOR_PAGE_DPI if pageinfo.has_vector else 0,
- )
- yres = max(
- pageinfo.yres or VECTOR_PAGE_DPI,
- options.oversample or 0,
- VECTOR_PAGE_DPI if pageinfo.has_vector else 0,
- )
- return (float(xres), float(yres))
-
-
-def get_page_square_dpi(pageinfo, options):
- "Get the DPI when we require xres == yres, scaled to physical units"
- xres = pageinfo.xres or 0
- yres = pageinfo.yres or 0
- userunit = pageinfo.userunit or 1
- return float(
- max(
- (xres * userunit) or VECTOR_PAGE_DPI,
- (yres * userunit) or VECTOR_PAGE_DPI,
- VECTOR_PAGE_DPI if pageinfo.has_vector else 0,
- options.oversample or 0,
- )
- )
-
-
-def get_canvas_square_dpi(pageinfo, options):
- """Get the DPI when we require xres == yres, in Postscript units"""
- return float(
- max(
- (pageinfo.xres) or VECTOR_PAGE_DPI,
- (pageinfo.yres) or VECTOR_PAGE_DPI,
- VECTOR_PAGE_DPI if pageinfo.has_vector else 0,
- options.oversample or 0,
- )
- )
-
-
-def is_ocr_required(page_context):
- pageinfo = page_context.pageinfo
- options = page_context.options
- log = page_context.log
-
- ocr_required = True
-
- if pageinfo.has_text:
- if not options.force_ocr and not (options.skip_text or options.redo_ocr):
- log.error("page already has text! - aborting (use --force-ocr to force OCR)")
- ocr_required = False
- elif options.force_ocr:
- log.info("page already has text! - rasterizing text and running OCR anyway")
- ocr_required = True
- elif options.redo_ocr:
- if pageinfo.has_corrupt_text:
- log.warn(
- "some text on this page cannot be mapped to characters: "
- "consider using --force-ocr instead",
- )
- else:
- log.info("redoing OCR")
- ocr_required = True
- elif options.skip_text:
- log.info("skipping all processing on this page")
- ocr_required = False
- elif not pageinfo.images and not options.lossless_reconstruction:
- # We found a page with no images and no text. That means it may
- # have vector art that the user wants to OCR. If we determined
- # lossless reconstruction is not possible then we have to rasterize
- # the image. So if OCR is being forced, take that to mean YES, go
- # ahead and rasterize. If not forced, then pretend there's no text
- # on the page at all so we don't lose anything.
- # This could be made smarter by explicitly searching for vector art.
- if options.force_ocr and options.oversample:
- # The user really wants to reprocess this file
- log.info(
- "page has no images - "
- f"rasterizing at {options.oversample} DPI because "
- "--force-ocr --oversample was specified"
- )
- elif options.force_ocr:
- # Warn the user they might not want to do this
- log.warn(
- "page has no images - "
- "all vector content will be "
- f"rasterized at {VECTOR_PAGE_DPI} DPI, losing some resolution and likely "
- "increasing file size. Use --oversample to adjust the "
- "DPI."
- )
- else:
- log.info(
- "page has no images - "
- "skipping all processing on this page to avoid losing detail. "
- "Use --force-ocr if you wish to perform OCR on pages that "
- "have vector content."
- )
- ocr_required = False
-
- if ocr_required and options.skip_big and pageinfo.images:
- pixel_count = pageinfo.width_pixels * pageinfo.height_pixels
- if pixel_count > (options.skip_big * 1_000_000):
- ocr_required = False
- log.warn(
- "page too big, skipping OCR "
- f"({(pixel_count / 1_000_000):.1f} MPixels > {options.skip_big:.1f} MPixels --skip-big)"
- )
- return ocr_required
-
-
-def rasterize_preview(input_file, page_context):
- output_file = page_context.get_path('rasterize_preview.jpg')
- canvas_dpi = get_canvas_square_dpi(page_context.pageinfo, page_context.options)
- page_dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
- ghostscript.rasterize_pdf(
- input_file,
- output_file,
- xres=canvas_dpi,
- yres=canvas_dpi,
- raster_device='jpeggray',
- log=page_context.log,
- page_dpi=(page_dpi, page_dpi),
- pageno=page_context.pageinfo.pageno + 1,
- )
- return output_file
-
-
-def get_orientation_correction(preview, page_context):
- """
- Work out orientation correct for each page.
-
- We ask Ghostscript to draw a preview page, which will rasterize with the
- current /Rotate applied, and then ask Tesseract which way the page is
- oriented. If the value of /Rotate is correct (e.g., a user already
- manually fixed rotation), then Tesseract will say the page is pointing
- up and the correction is zero. Otherwise, the orientation found by
- Tesseract represents the clockwise rotation, or the counterclockwise
- correction to rotation.
-
- When we draw the real page for OCR, we rotate it by the CCW correction,
- which points it (hopefully) upright. _weave.py takes care of the orienting
- the image and text layers.
-
- """
-
- orient_conf = tesseract.get_orientation(
- preview,
- engine_mode=page_context.options.tesseract_oem,
- timeout=page_context.options.tesseract_timeout,
- log=page_context.log,
- )
-
- direction = {0: '⇧', 90: '⇨', 180: '⇩', 270: '⇦'}
-
- existing_rotation = page_context.pageinfo.rotation
-
- correction = orient_conf.angle % 360
-
- apply_correction = False
- action = ''
- if orient_conf.confidence >= page_context.options.rotate_pages_threshold:
- if correction != 0:
- apply_correction = True
- action = ' - will rotate'
- else:
- action = ' - rotation appears correct'
- else:
- if correction != 0:
- action = ' - confidence too low to rotate'
- else:
- action = ' - no change'
-
- facing = ''
- if existing_rotation != 0:
- facing = 'with existing rotation {}, '.format(
- direction.get(existing_rotation, '?')
- )
- facing += 'page is facing {}'.format(direction.get(orient_conf.angle, '?'))
-
- page_context.log.debug(
- '{pagenum:4d}: {facing}, confidence {conf:.2f}{action}'.format(
- pagenum=page_context.pageinfo.pageno,
- facing=facing,
- conf=orient_conf.confidence,
- action=action,
- )
- )
-
- if apply_correction:
- return correction
- return 0
-
-
-def rasterize(input_file, page_context, correction=0):
- colorspaces = ['pngmono', 'pnggray', 'png256', 'png16m']
- device_idx = 0
- output_file = page_context.get_path('rasterize.png')
- pageinfo = page_context.pageinfo
-
- def at_least(cs):
- return max(device_idx, colorspaces.index(cs))
-
- for image in pageinfo.images:
- if image.type_ != 'image':
- continue # ignore masks
- if image.bpc > 1:
- if image.color == Colorspace.index:
- device_idx = at_least('png256')
- elif image.color == Colorspace.gray:
- device_idx = at_least('pnggray')
- else:
- device_idx = at_least('png16m')
-
- device = colorspaces[device_idx]
-
- page_context.log.debug(f"Rasterize with {device}")
-
- # Produce the page image with square resolution or else deskew and OCR
- # will not work properly.
- canvas_dpi = get_canvas_square_dpi(pageinfo, page_context.options)
- page_dpi = get_page_square_dpi(pageinfo, page_context.options)
-
- ghostscript.rasterize_pdf(
- input_file,
- output_file,
- xres=canvas_dpi,
- yres=canvas_dpi,
- raster_device=device,
- log=page_context.log,
- page_dpi=(page_dpi, page_dpi),
- pageno=pageinfo.pageno + 1,
- rotation=correction,
- filter_vector=page_context.options.remove_vectors,
- )
- return output_file
-
-
-def preprocess_remove_background(input_file, page_context):
- if any(image.bpc > 1 for image in page_context.pageinfo.images):
- output_file = page_context.get_path('pp_rm_bg.png')
- leptonica.remove_background(input_file, output_file)
- return output_file
- else:
- page_context.log.info("background removal skipped on mono page")
- return input_file
-
-
-def preprocess_deskew(input_file, page_context):
- output_file = page_context.get_path('pp_deskew.png')
- dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
- leptonica.deskew(input_file, output_file, dpi)
- return output_file
-
-
-def preprocess_clean(input_file, page_context):
- from .exec import unpaper
- output_file = page_context.get_path('pp_clean.png')
- dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
- unpaper.clean(input_file, output_file, dpi, page_context.log, page_context.options.unpaper_args)
- return output_file
-
-
-def create_ocr_image(image, page_context):
- """Create the image we send for OCR. May not be the same as the display
- image depending on preprocessing. This image will never be shown to the
- user."""
-
- output_file = page_context.get_path('ocr.png')
- options = page_context.options
- with Image.open(image) as im:
- from PIL import ImageColor
- from PIL import ImageDraw
-
- white = ImageColor.getcolor('#ffffff', im.mode)
- # pink = ImageColor.getcolor('#ff0080', im.mode)
- draw = ImageDraw.ImageDraw(im)
-
- xres, yres = im.info['dpi']
- page_context.log.info('resolution %r %r' % (xres, yres))
-
- if not options.force_ocr:
- # Do not mask text areas when forcing OCR, because we need to OCR
- # all text areas
- mask = None # Exclude both visible and invisible text from OCR
- if options.redo_ocr:
- mask = True # Mask visible text, but not invisible text
-
- for textarea in page_context.pageinfo.get_textareas(visible=mask, corrupt=None):
- # Calculate resolution based on the image size and page dimensions
- # without regard whatever resolution is in pageinfo (may differ or
- # be None)
- bbox = [float(v) for v in textarea]
- xscale, yscale = float(xres) / 72.0, float(yres) / 72.0
- pixcoords = [
- bbox[0] * xscale,
- im.height - bbox[3] * yscale,
- bbox[2] * xscale,
- im.height - bbox[1] * yscale,
- ]
- pixcoords = [int(round(c)) for c in pixcoords]
- print('blanking %r', pixcoords)
- draw.rectangle(pixcoords, fill=white)
- # draw.rectangle(pixcoords, outline=pink)
-
- if options.mask_barcodes or options.threshold:
- pix = leptonica.Pix.frompil(im)
- if options.threshold:
- pix = pix.masked_threshold_on_background_norm()
- if options.mask_barcodes:
- barcodes = pix.locate_barcodes()
- for barcode in barcodes:
- decoded, rect = barcode
- print('masking barcode %s %r', decoded, rect)
- draw.rectangle(rect, fill=white)
- im = pix.topil()
-
- del draw
- # Pillow requires integer DPI
- dpi = round(xres), round(yres)
- im.save(output_file, dpi=dpi)
- return output_file
-
-
-def ocr_tesseract_hocr(input_file, page_context):
- hocr_out = page_context.get_path('ocr_hocr.hocr')
- hocr_text_out = page_context.get_path('ocr_hocr.txt')
- options = page_context.options
- tesseract.generate_hocr(
- input_file=input_file,
- output_files=[hocr_out, hocr_text_out],
- language=options.language,
- engine_mode=options.tesseract_oem,
- tessconfig=options.tesseract_config,
- timeout=options.tesseract_timeout,
- pagesegmode=options.tesseract_pagesegmode,
- user_words=options.user_words,
- user_patterns=options.user_patterns,
- log=page_context.log,
- )
- return (hocr_out, hocr_text_out)
-
-
-def should_visible_page_image_use_jpg(pageinfo):
- # If all images were JPEGs originally, produce a JPEG as output
- return pageinfo.images and all(im.enc == 'jpeg' for im in pageinfo.images)
-
-
-def create_visible_page_jpg(image, page_context):
- output_file = page_context.get_path('visible.jpg')
- with Image.open(image) as im:
- # At this point the image should be a .png, but deskew, unpaper
- # might have removed the DPI information. In this case, fall back to
- # square DPI used to rasterize. When the preview image was
- # rasterized, it was also converted to square resolution, which is
- # what we want to give tesseract, so keep it square.
- fallback_dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
- dpi = im.info.get('dpi', (fallback_dpi, fallback_dpi))
-
- # Pillow requires integer DPI
- dpi = round(dpi[0]), round(dpi[1])
- im.save(output_file, format='JPEG', dpi=dpi)
- return output_file
-
-
-def create_pdf_page_from_image(image, page_context):
- # We rasterize a square DPI version of each page because most image
- # processing tools don't support rectangular DPI. Use the square DPI as it
- # accurately describes the image. It would be possible to resample the image
- # at this stage back to non-square DPI to more closely resemble the input,
- # except that the hocr renderer does not understand non-square DPI. The
- # sandwich renderer would be fine.
- output_file = page_context.get_path('visible.pdf')
- dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
- layout_fun = img2pdf.get_fixed_dpi_layout_fun((dpi, dpi))
-
- # This create a single page PDF
- with open(image, 'rb') as imfile, open(output_file, 'wb') as pdf:
- page_context.log.debug('convert')
- img2pdf.convert(
- imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf
- )
- page_context.log.debug('convert done')
- return output_file
-
-
-def render_hocr_page(hocr, page_context):
- output_file = page_context.get_path('ocr_hocr.pdf')
- dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
- hocrtransform = HocrTransform(hocr, dpi)
- hocrtransform.to_pdf(
- output_file,
- imageFileName=None,
- showBoundingboxes=False,
- invisibleText=True,
- interwordSpaces=True,
- )
- return output_file
-
-
-def ocr_tesseract_textonly_pdf(input_image, page_context):
- output_pdf = page_context.get_path('ocr_tess.pdf')
- output_text = page_context.get_path('ocr_tess.txt')
- options = page_context.options
- tesseract.generate_pdf(
- input_image=input_image,
- skip_pdf=None,
- output_pdf=output_pdf,
- output_text=output_text,
- language=options.language,
- engine_mode=options.tesseract_oem,
- text_only=True,
- tessconfig=options.tesseract_config,
- timeout=options.tesseract_timeout,
- pagesegmode=options.tesseract_pagesegmode,
- user_words=options.user_words,
- user_patterns=options.user_patterns,
- log=page_context.log,
- )
- return (output_pdf, output_text)
-
-
-def get_docinfo(base_pdf, options):
- def from_document_info(key):
- try:
- s = base_pdf.docinfo[key]
- return str(s)
- except (KeyError, TypeError):
- return ''
-
- pdfmark = {
- k: from_document_info(k)
- for k in ('/Title', '/Author', '/Keywords', '/Subject', '/CreationDate')
- }
- if options.title:
- pdfmark['/Title'] = options.title
- if options.author:
- pdfmark['/Author'] = options.author
- if options.keywords:
- pdfmark['/Keywords'] = options.keywords
- if options.subject:
- pdfmark['/Subject'] = options.subject
-
- if options.pdf_renderer == 'sandwich':
- renderer_tag = 'OCR-PDF'
- else:
- renderer_tag = 'OCR'
-
- pdfmark['/Creator'] = (
- f'{PROGRAM_NAME} {VERSION} / ' f'Tesseract {renderer_tag} {tesseract.version()}'
- )
- pdfmark['/Producer'] = f'pikepdf {pikepdf.__version__}'
- if 'OCRMYPDF_CREATOR' in os.environ:
- pdfmark['/Creator'] = os.environ['OCRMYPDF_CREATOR']
- if 'OCRMYPDF_PRODUCER' in os.environ:
- pdfmark['/Producer'] = os.environ['OCRMYPDF_PRODUCER']
-
- pdfmark['/ModDate'] = encode_pdf_date(datetime.now(timezone.utc))
- return pdfmark
-
-
-def generate_postscript_stub(context):
- output_file = context.get_path('pdfa.ps')
- generate_pdfa_ps(output_file)
- return output_file
-
-
-def convert_to_pdfa(input_pdf, input_ps_stub, context):
- options = context.options
- input_pdfinfo = context.pdfinfo
- output_file = context.get_path('pdfa.pdf')
-
- # If the DocumentInfo record contains NUL characters, Ghostscript will
- # produce XMP metadata which contains invalid XML entities (�).
- # NULs in DocumentInfo seem to be common since older Acrobats included them.
- # pikepdf can deal with this, but we make the world a better place by
- # stamping them out as soon as possible.
- pdf_file = pikepdf.open(input_pdf)
- if pdf_file.docinfo:
- modified = False
- for k, v in pdf_file.docinfo.items():
- if b'\x00' in bytes(v):
- pdf_file.docinfo[k] = bytes(v).replace(b'\x00', b'')
- modified = True
- if modified:
- pdf_file.save(input_pdf)
- del pdf_file
-
- ghostscript.generate_pdfa(
- pdf_version=input_pdfinfo.min_version,
- pdf_pages=[input_pdf, input_ps_stub],
- output_file=output_file,
- compression=options.pdfa_image_compression,
- log=context.log,
- threads=options.jobs or 1,
- pdfa_part=options.output_type[-1], # is pdfa-1, pdfa-2, or pdfa-3
- )
-
- return output_file
-
-
-def metadata_fixup(working_file, context):
- output_file = context.get_path('metafix.pdf')
- options = context.options
- original = pikepdf.open(context.origin)
- docinfo = get_docinfo(original, options)
- pdf = pikepdf.open(working_file)
- with pdf.open_metadata() as meta:
- meta.load_from_docinfo(docinfo, delete_missing=False)
- # If xmp:CreateDate is missing, set it to the modify date to
- # match Ghostscript, for consistency
- if 'xmp:CreateDate' not in meta:
- meta['xmp:CreateDate'] = meta.get('xmp:ModifyDate', '')
-
- meta_original = original.open_metadata()
- not_copied = set(meta_original.keys()) - set(meta.keys())
- if not_copied:
- context.log.warning(
- "Some input metadata could not be copied because it is not "
- "permitted in PDF/A. You may wish to examine the output "
- "PDF's XMP metadata."
- )
- context.log.debug(
- "The following metadata fields were not copied: %r", not_copied
- )
-
- pdf.save(
- output_file,
- compress_streams=True,
- object_stream_mode=pikepdf.ObjectStreamMode.generate,
- )
- return output_file
-
-
-def optimize_pdf(input_file, context):
- output_file = context.get_path('optimize.pdf')
- optimize(input_file, output_file, context)
- return output_file
-
-
-def merge_sidecars(txt_files, context):
- output_file = context.get_path('sidecar.txt')
- with open(output_file, 'w', encoding="utf-8") as stream:
- for page_num, txt_file in enumerate(txt_files):
- if page_num != 0:
- stream.write('\f') # Form feed between pages
- if txt_file:
- with open(txt_file, 'r', encoding="utf-8") as in_:
- txt = in_.read()
- # Tesseract v4 alpha started adding form feeds in
- # commit aa6eb6b
- # No obvious way to detect what binaries will do this, so
- # for consistency just ignore its form feeds and insert our
- # own
- if txt.endswith('\f'):
- stream.write(txt[:-1])
- else:
- stream.write(txt)
- else:
- stream.write(f'[OCR skipped on page {(page_num + 1)}]')
- return output_file
-
-
-def copy_final(input_file, output_file, context):
- context.log.debug('%s -> %s', input_file, output_file)
- with open(input_file, 'rb') as input_stream:
- if output_file == '-':
- copyfileobj(input_stream, sys.stdout.buffer)
- sys.stdout.flush()
- else:
- # At this point we overwrite the output_file specified by the user
- # use copyfileobj because then we use open() to create the file and
- # get the appropriate umask, ownership, etc.
- with open(output_file, 'wb') as output_stream:
- copyfileobj(input_stream, output_stream)
diff --git a/src/ocrmypdf/_ruffus.py b/src/ocrmypdf/_ruffus.py
deleted file mode 100644
index 4cc69043..00000000
--- a/src/ocrmypdf/_ruffus.py
+++ /dev/null
@@ -1,508 +0,0 @@
-# © 2016 James R. Barlow: github.com/jbarlow83
-#
-# This file is part of OCRmyPDF.
-#
-# OCRmyPDF is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# OCRmyPDF is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with OCRmyPDF. If not, see .
-
-import os
-import re
-import sys
-import atexit
-from tempfile import mkdtemp
-from ruffus import (
- Pipeline,
- formatter,
- regex,
- suffix,
- cmdline,
- proxy_logger,
- ruffus_exceptions
-)
-from .exec import qpdf
-from ._jobcontext import JobContext, JobContextManager, cleanup_working_files
-from ._weave import weave_layers
-from ._pipeline import (
- triage,
- repair_and_parse_pdf,
- marker_pages,
- ocr_or_skip,
- rasterize_preview,
- orient_page,
- rasterize_with_ghostscript,
- preprocess_remove_background,
- preprocess_deskew,
- preprocess_clean,
- select_ocr_image,
- ocr_tesseract_hocr,
- select_visible_page_image,
- select_image_layer,
- render_hocr_page,
- ocr_tesseract_textonly_pdf,
- generate_postscript_stub,
- convert_to_pdfa,
- metadata_fixup,
- merge_sidecars,
- optimize_pdf,
- copy_final
-)
-from . import exceptions as ocrmypdf_exceptions
-from .exceptions import (
- ExitCode,
- ExitCodeException,
-)
-from .helpers import available_cpu_count
-from .pdfa import file_claims_pdfa
-from ._validation import (
- check_closed_streams,
- preamble,
- check_options,
- check_dependency_versions,
- check_environ,
- check_input_file,
- check_requested_output_file,
- report_output_file_size,
- log_page_orientations,
- logging_factory,
-)
-
-
-def cleanup_ruffus_error_message(msg):
- msg = re.sub(r'\s+', r' ', msg)
- msg = re.sub(r"\((.+?)\)", r'\1', msg)
- msg = msg.strip()
- return msg
-
-
-def do_ruffus_exception(ruffus_five_tuple, options, log):
- """Replace the elaborate ruffus stack trace with a user friendly
- description of the error message that occurred."""
- exit_code = None
-
- _task_name, _job_name, exc_name, exc_value, exc_stack = ruffus_five_tuple
-
- if isinstance(exc_name, type):
- # ruffus is full of mystery... sometimes (probably when the process
- # group leader is killed) exc_name is the class object of the exception,
- # rather than a str. So reach into the object and get its name.
- exc_name = exc_name.__name__
-
- if exc_name.startswith('ocrmypdf.exceptions.'):
- base_exc_name = exc_name.replace('ocrmypdf.exceptions.', '')
- exc_class = getattr(ocrmypdf_exceptions, base_exc_name)
- exit_code = getattr(exc_class, 'exit_code', ExitCode.other_error)
- try:
- if isinstance(exc_value, exc_class):
- exc_msg = str(exc_value)
- elif isinstance(exc_value, str):
- exc_msg = exc_value
- else:
- exc_msg = str(exc_class())
- except Exception:
- exc_msg = "Unknown"
-
- if exc_name in ('builtins.SystemExit', 'SystemExit'):
- match = re.search(r"\.(.+?)\)", exc_value)
- exit_code_name = match.groups()[0]
- exit_code = getattr(ExitCode, exit_code_name, 'other_error')
- elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':
- log.error(cleanup_ruffus_error_message(exc_value))
- exit_code = ExitCode.input_file
- elif exc_name in ('builtins.KeyboardInterrupt', 'KeyboardInterrupt'):
- # We have to print in this case because the log daemon might be toast
- print("Interrupted by user", file=sys.stderr)
- exit_code = ExitCode.ctrl_c
- elif exc_name == 'subprocess.CalledProcessError':
- # It's up to the subprocess handler to report something useful
- msg = "Error occurred while running this command:"
- log.error(msg + '\n' + exc_value)
- exit_code = ExitCode.child_process_error
- elif exc_name.startswith('ocrmypdf.exceptions.'):
- if exc_msg:
- log.error(exc_msg)
- elif exc_name == 'PIL.Image.DecompressionBombError':
- msg = cleanup_ruffus_error_message(exc_value)
- msg += (
- "\nUse the --max-image-mpixels argument to set increase the "
- "maximum number of megapixels to accept."
- )
- log.error(msg)
- exit_code = ExitCode.input_file
-
- if exit_code is not None:
- return exit_code
-
- if not options.verbose:
- log.error(exc_stack)
- return ExitCode.other_error
-
-
-def traverse_ruffus_exception(exceptions, options, log):
- """Traverse a RethrownJobError and output the exceptions
-
- Ruffus presents exceptions as 5 element tuples. The RethrownJobException
- has a list of exceptions like
- e.job_exceptions = [(5-tuple), (5-tuple), ...]
-
- ruffus < 2.7.0 had a bug with exception marshalling that would give
- different output whether the main or child process raised the exception.
- We no longer support this.
-
- Attempting to log the exception itself will re-marshall it to the logger
- which is normally running in another process. It's better to avoid re-
- marshalling.
-
- The exit code will be based on this, even if multiple exceptions occurred
- at the same time."""
-
- exit_codes = []
- for exc in exceptions:
- exit_code = do_ruffus_exception(exc, options, log)
- exit_codes.append(exit_code)
-
- return exit_codes[0] # Multiple codes are rare so take the first one
-
-
-def build_pipeline(options, work_folder, log, context):
- main_pipeline = Pipeline.pipelines['main']
-
- # Triage
- task_triage = main_pipeline.transform(
- task_func=triage,
- input=os.path.join(work_folder, 'origin'),
- filter=formatter('(?i)'),
- output=os.path.join(work_folder, 'origin.pdf'),
- extras=[log, context],
- )
-
- task_repair_and_parse_pdf = main_pipeline.transform(
- task_func=repair_and_parse_pdf,
- input=task_triage,
- filter=suffix('.pdf'),
- output='.repaired.pdf',
- output_dir=work_folder,
- extras=[log, context],
- )
-
- # Split (kwargs for split seems to be broken, so pass plain args)
- task_marker_pages = main_pipeline.split(
- marker_pages,
- task_repair_and_parse_pdf,
- os.path.join(work_folder, '*.marker.pdf'),
- extras=[log, context],
- )
-
- task_ocr_or_skip = main_pipeline.split(
- ocr_or_skip,
- task_marker_pages,
- [
- os.path.join(work_folder, '*.ocr.page.pdf'),
- os.path.join(work_folder, '*.skip.page.pdf'),
- ],
- extras=[log, context],
- )
-
- # Rasterize preview
- task_rasterize_preview = main_pipeline.transform(
- task_func=rasterize_preview,
- input=task_ocr_or_skip,
- filter=suffix('.page.pdf'),
- output='.preview.jpg',
- output_dir=work_folder,
- extras=[log, context],
- )
- task_rasterize_preview.active_if(options.rotate_pages)
-
- # Orient
- task_orient_page = main_pipeline.collate(
- task_func=orient_page,
- input=[task_ocr_or_skip, task_rasterize_preview],
- filter=regex(r".*/(\d{6})(\.ocr|\.skip)(?:\.page\.pdf|\.preview\.jpg)"),
- output=os.path.join(work_folder, r'\1\2.oriented.pdf'),
- extras=[log, context],
- )
-
- # Rasterize actual
- task_rasterize_with_ghostscript = main_pipeline.transform(
- task_func=rasterize_with_ghostscript,
- input=task_orient_page,
- filter=suffix('.ocr.oriented.pdf'),
- output='.page.png',
- output_dir=work_folder,
- extras=[log, context],
- )
-
- # Preprocessing subpipeline
- task_preprocess_remove_background = main_pipeline.transform(
- task_func=preprocess_remove_background,
- input=task_rasterize_with_ghostscript,
- filter=suffix(".page.png"),
- output=".pp-background.png",
- extras=[log, context],
- )
-
- task_preprocess_deskew = main_pipeline.transform(
- task_func=preprocess_deskew,
- input=task_preprocess_remove_background,
- filter=suffix(".pp-background.png"),
- output=".pp-deskew.png",
- extras=[log, context],
- )
-
- task_preprocess_clean = main_pipeline.transform(
- task_func=preprocess_clean,
- input=task_preprocess_deskew,
- filter=suffix(".pp-deskew.png"),
- output=".pp-clean.png",
- extras=[log, context],
- )
-
- task_select_ocr_image = main_pipeline.collate(
- task_func=select_ocr_image,
- input=[task_preprocess_clean],
- filter=regex(r".*/(\d{6})(?:\.page|\.pp-.*)\.png"),
- output=os.path.join(work_folder, r"\1.ocr.png"),
- extras=[log, context],
- )
-
- # HOCR OCR
- task_ocr_tesseract_hocr = main_pipeline.transform(
- task_func=ocr_tesseract_hocr,
- input=task_select_ocr_image,
- filter=suffix(".ocr.png"),
- output=[".hocr", ".txt"],
- extras=[log, context],
- )
- task_ocr_tesseract_hocr.graphviz(fillcolor='"#00cc66"')
- task_ocr_tesseract_hocr.active_if(options.pdf_renderer == 'hocr')
-
- task_select_visible_page_image = main_pipeline.collate(
- task_func=select_visible_page_image,
- input=[
- task_rasterize_with_ghostscript,
- task_preprocess_remove_background,
- task_preprocess_deskew,
- task_preprocess_clean,
- ],
- filter=regex(r".*/(\d{6})(?:\.page|\.pp-.*)\.png"),
- output=os.path.join(work_folder, r'\1.image'),
- extras=[log, context],
- )
- task_select_visible_page_image.graphviz(shape='diamond')
-
- task_select_image_layer = main_pipeline.collate(
- task_func=select_image_layer,
- input=[task_select_visible_page_image, task_orient_page],
- filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.oriented\.pdf)"),
- output=os.path.join(work_folder, r'\1.image-layer.pdf'),
- extras=[log, context],
- )
- task_select_image_layer.graphviz(fillcolor='"#00cc66"', shape='diamond')
-
- task_render_hocr_page = main_pipeline.transform(
- task_func=render_hocr_page,
- input=task_ocr_tesseract_hocr,
- filter=regex(r".*/(\d{6})(?:\.hocr)"),
- output=os.path.join(work_folder, r'\1.text.pdf'),
- extras=[log, context],
- )
- task_render_hocr_page.graphviz(fillcolor='"#00cc66"')
- task_render_hocr_page.active_if(options.pdf_renderer == 'hocr')
-
- # Tesseract OCR + text only PDF
- task_ocr_tesseract_textonly_pdf = main_pipeline.collate(
- task_func=ocr_tesseract_textonly_pdf,
- input=[task_select_ocr_image],
- filter=regex(r".*/(\d{6})(?:\.ocr.png)"),
- output=[
- os.path.join(work_folder, r'\1.text.pdf'),
- os.path.join(work_folder, r'\1.text.txt'),
- ],
- extras=[log, context],
- )
- task_ocr_tesseract_textonly_pdf.graphviz(fillcolor='"#ff69b4"')
- task_ocr_tesseract_textonly_pdf.active_if(options.pdf_renderer == 'sandwich')
-
- task_weave_layers = main_pipeline.collate(
- task_func=weave_layers,
- input=[
- task_repair_and_parse_pdf,
- task_render_hocr_page,
- task_ocr_tesseract_textonly_pdf,
- task_select_image_layer,
- ],
- filter=regex(
- r".*/((?:\d{6}(?:\.text\.pdf|\.image-layer\.pdf))|(?:origin\.repaired\.pdf))"
- ),
- output=os.path.join(work_folder, r'layers.rendered.pdf'),
- extras=[log, context],
- )
- task_weave_layers.graphviz(fillcolor='"#00cc66"')
-
- # PDF/A pdfmark
- task_generate_postscript_stub = main_pipeline.transform(
- task_func=generate_postscript_stub,
- input=task_repair_and_parse_pdf,
- filter=formatter(r'\.repaired\.pdf'),
- output=os.path.join(work_folder, 'pdfa.ps'),
- extras=[log, context],
- )
- task_generate_postscript_stub.active_if(options.output_type.startswith('pdfa'))
-
- # PDF/A conversion
- task_convert_to_pdfa = main_pipeline.merge(
- task_func=convert_to_pdfa,
- input=[task_generate_postscript_stub, task_weave_layers],
- output=os.path.join(work_folder, 'pdfa.pdf'),
- extras=[log, context],
- )
- task_convert_to_pdfa.active_if(options.output_type.startswith('pdfa'))
-
- task_metadata_fixup = main_pipeline.merge(
- task_func=metadata_fixup,
- input=[task_repair_and_parse_pdf, task_weave_layers, task_convert_to_pdfa],
- output=os.path.join(work_folder, 'metafix.pdf'),
- extras=[log, context],
- )
-
- task_merge_sidecars = main_pipeline.merge(
- task_func=merge_sidecars,
- input=[task_ocr_tesseract_hocr, task_ocr_tesseract_textonly_pdf],
- output=options.sidecar,
- extras=[log, context],
- )
- task_merge_sidecars.active_if(options.sidecar)
-
- # Optimize
- task_optimize_pdf = main_pipeline.transform(
- task_func=optimize_pdf,
- input=task_metadata_fixup,
- filter=suffix('.pdf'),
- output='.optimized.pdf',
- output_dir=work_folder,
- extras=[log, context],
- )
-
- # Finalize
- main_pipeline.merge(
- task_func=copy_final,
- input=[task_optimize_pdf],
- output=options.output_file,
- extras=[log, context],
- )
-
-
-def run_pipeline(options):
- options.verbose_abbreviated_path = 1
- if os.environ.get('_OCRMYPDF_THREADS'):
- options.use_threads = True
-
- if not check_closed_streams(options):
- return ExitCode.bad_args
-
- logger_args = {'verbose': options.verbose, 'quiet': options.quiet}
-
- _log, _log_mutex = proxy_logger.make_shared_logger_and_proxy(
- logging_factory, __name__, logger_args
- )
- preamble(_log)
- check_code = check_options(options, _log)
- if check_code != ExitCode.ok:
- return check_code
- check_dependency_versions(options, _log)
-
- # Any changes to options will not take effect for options that are already
- # bound to function parameters in the pipeline. (For example
- # options.input_file, options.pdf_renderer are already bound.)
- if not options.jobs:
- options.jobs = available_cpu_count()
-
- # Performance is improved by setting Tesseract to single threaded. In tests
- # this gives better throughput than letting a smaller number of Tesseract
- # jobs run multithreaded. Same story for pngquant. Tess <4 ignores this
- # variable, but harmless to set if ignored.
- os.environ.setdefault('OMP_THREAD_LIMIT', '1')
-
- check_environ(options, _log)
- if os.environ.get('PYTEST_CURRENT_TEST'):
- os.environ['_OCRMYPDF_TEST_INFILE'] = options.input_file
-
- try:
- work_folder = mkdtemp(prefix="com.github.ocrmypdf.")
- options.history_file = os.path.join(work_folder, 'ruffus_history.sqlite')
- start_input_file = os.path.join(work_folder, 'origin')
-
- check_input_file(options, _log, start_input_file)
- check_requested_output_file(options, _log)
-
- manager = JobContextManager()
- manager.register('JobContext', JobContext) # pylint: disable=no-member
- manager.start()
-
- context = manager.JobContext() # pylint: disable=no-member
- context.set_options(options)
- context.set_work_folder(work_folder)
-
- build_pipeline(options, work_folder, _log, context)
- atexit.register(cleanup_working_files, work_folder, options)
- if hasattr(os, 'nice'):
- os.nice(5)
- cmdline.run(options)
- except ruffus_exceptions.RethrownJobError as e:
- if options.verbose:
- _log.debug(str(e)) # stringify exception so logger doesn't have to
- exceptions = e.job_exceptions
- exitcode = traverse_ruffus_exception(exceptions, options, _log)
- if exitcode is None:
- _log.error("Unexpected ruffus exception: " + str(e))
- _log.error(repr(e))
- return ExitCode.other_error
- return exitcode
- except ExitCodeException as e:
- return e.exit_code
- except Exception as e:
- _log.error(str(e))
- return ExitCode.other_error
-
- if options.flowchart:
- _log.info(f"Flowchart saved to {options.flowchart}")
- return ExitCode.ok
- elif options.output_file == '-':
- _log.info("Output sent to stdout")
- elif os.path.samefile(options.output_file, os.devnull):
- pass # Say nothing when sending to dev null
- else:
- if options.output_type.startswith('pdfa'):
- pdfa_info = file_claims_pdfa(options.output_file)
- if pdfa_info['pass']:
- msg = f"Output file is a {pdfa_info['conformance']} (as expected)"
- _log.info(msg)
- else:
- msg = f"Output file is okay but is not PDF/A (seems to be {pdfa_info['conformance']})"
- _log.warning(msg)
- return ExitCode.pdfa_conversion_failed
- if not qpdf.check(options.output_file, _log):
- _log.warning('Output file: The generated PDF is INVALID')
- return ExitCode.invalid_output_pdf
-
- report_output_file_size(options, _log, start_input_file, options.output_file)
-
- pdfinfo = context.get_pdfinfo()
- if options.verbose:
- from pprint import pformat
-
- _log.debug(pformat(pdfinfo))
-
- log_page_orientations(pdfinfo, _log)
-
- return ExitCode.ok
diff --git a/src/ocrmypdf/_sync.py b/src/ocrmypdf/_sync.py
index ec339bf5..4e246544 100644
--- a/src/ocrmypdf/_sync.py
+++ b/src/ocrmypdf/_sync.py
@@ -16,13 +16,11 @@
# along with OCRmyPDF. If not, see .
import os
-# import re
-# import sys
import atexit
from tempfile import mkdtemp
-from ._jobcontext import cleanup_working_files
+from ._jobcontext import PDFContext, get_logger, cleanup_working_files
from ._weave import weave_layers
-from ._pipeline_simple import (
+from ._pipeline import (
get_pdfinfo,
validate_pdfinfo_options,
is_ocr_required,
@@ -48,6 +46,7 @@ from ._pipeline_simple import (
)
from .exceptions import (
ExitCode,
+ ExitCodeException,
)
from .helpers import available_cpu_count
from ._validation import (
@@ -61,52 +60,6 @@ from ._validation import (
)
-class Logger:
- def __init__(self, prefix):
- self.prefix = prefix
-
- def debug(self, *argv):
- print(self.prefix, *argv)
-
- def info(self, *argv):
- print(self.prefix, *argv)
-
- def warn(self, *argv):
- print(self.prefix, *argv)
-
- def error(self, *argv):
- print(self.prefix, *argv)
-
-
-class PageContext:
- def __init__(self, pdf_context, pageno):
- self.pdf_context = pdf_context
- self.options = pdf_context.options
- self.pageno = pageno
- self.pageinfo = pdf_context.pdfinfo[pageno]
- self.log = Logger('%s Page %d: ' % (os.path.basename(pdf_context.origin), pageno + 1))
-
- def get_path(self, name):
- return os.path.join(self.pdf_context.work_folder, "page_%d_%s" % (self.pageno, name))
-
-
-class PDFContext:
- def __init__(self, options, work_folder, origin, pdfinfo):
- self.options = options
- self.work_folder = work_folder
- self.origin = origin
- self.pdfinfo = pdfinfo
- self.log = Logger('%s: ' % os.path.basename(origin))
-
- def get_path(self, name):
- return os.path.join(self.work_folder, name)
-
- def get_page_contexts(self):
- npages = len(self.pdfinfo)
- for n in range(npages):
- yield PageContext(self, n)
-
-
def _exec_pipeline(options, work_folder, origin):
# Gather info of pdf
pdfinfo = get_pdfinfo(origin)
@@ -181,7 +134,7 @@ def run_pipeline(options):
if not check_closed_streams(options):
return ExitCode.bad_args
- log = Logger('Pipeline')
+ log = get_logger(options, 'Pipeline')
preamble(log)
check_code = check_options(options, log)
if check_code != ExitCode.ok:
@@ -213,7 +166,13 @@ def run_pipeline(options):
if hasattr(os, 'nice'):
os.nice(5)
- _exec_pipeline(options, work_folder, start_input_file)
+ try:
+ _exec_pipeline(options, work_folder, start_input_file)
+ except ExitCodeException as e:
+ return e.exit_code
+ except Exception as e:
+ log.error(str(e))
+ return ExitCode.other_error
return ExitCode.ok
diff --git a/src/ocrmypdf/optimize.py b/src/ocrmypdf/optimize.py
index 000f24d8..9cf1e69b 100644
--- a/src/ocrmypdf/optimize.py
+++ b/src/ocrmypdf/optimize.py
@@ -16,7 +16,6 @@
# along with OCRmyPDF. If not, see .
import concurrent.futures
-import logging
import sys
from collections import defaultdict
from os import fspath
@@ -28,7 +27,7 @@ import pikepdf
from pikepdf import Name, Dictionary
from . import leptonica
-from ._jobcontext import JobContext
+from ._jobcontext import PDFContext
from .exec import jbig2enc, pngquant
from .helpers import re_symlink
@@ -82,9 +81,7 @@ def extract_image_jbig2(*, pike, root, log, image, xref, options):
pim, filtdp = result
if (
- pim.bits_per_component == 1
- and filtdp != Name.JBIG2Decode
- and jbig2enc.available()
+ pim.bits_per_component == 1 and filtdp != Name.JBIG2Decode and jbig2enc.available()
):
try:
imgname = Path(root / f'{xref:08d}')
@@ -129,9 +126,7 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
return None
return xref, ext
elif (
- pim.indexed
- and pim.colorspace in pim.SIMPLE_COLORSPACES
- and options.optimize >= 3
+ pim.indexed and pim.colorspace in pim.SIMPLE_COLORSPACES and options.optimize >= 3
):
# Try to improve on indexed images - these are far from low hanging
# fruit in most cases
@@ -492,10 +487,6 @@ def main(infile, outfile, level, jobs=1):
self.jbig2_page_group_size = 0
self.jbig2_lossy = jb2lossy
- logging.basicConfig(level=logging.DEBUG)
- log = logging.getLogger()
-
- ctx = JobContext()
options = OptimizeOptions(
jobs=jobs,
optimize=int(level),
@@ -503,11 +494,11 @@ def main(infile, outfile, level, jobs=1):
png_quality=0,
jb2lossy=False,
)
- ctx.set_options(options)
with TemporaryDirectory() as td:
+ context = PDFContext(options, td, infile, None)
tmpout = Path(td) / 'out.pdf'
- optimize(infile, tmpout, log, ctx)
+ optimize(infile, tmpout, context)
copy(fspath(tmpout), fspath(outfile))
diff --git a/tests/test_multiprocessing.py b/tests/_test_multiprocessing.py
similarity index 100%
rename from tests/test_multiprocessing.py
rename to tests/_test_multiprocessing.py
diff --git a/tests/test_main.py b/tests/test_main.py
index 248a9003..49d93025 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -28,7 +28,7 @@ import pytest
from PIL import Image
from ocrmypdf.exceptions import ExitCode, MissingDependencyError
-from ocrmypdf.exec import ghostscript, qpdf, tesseract, unpaper
+from ocrmypdf.exec import ghostscript, qpdf, tesseract
from ocrmypdf.leptonica import Pix
from ocrmypdf.pdfa import file_claims_pdfa
from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
diff --git a/tests/test_metadata.py b/tests/test_metadata.py
index be1d952c..a4d60b55 100644
--- a/tests/test_metadata.py
+++ b/tests/test_metadata.py
@@ -28,7 +28,7 @@ from unittest.mock import MagicMock, patch
import pytest
import pikepdf
-from ocrmypdf._jobcontext import JobContext
+from ocrmypdf._jobcontext import PDFContext
from ocrmypdf.exceptions import ExitCode
from ocrmypdf.pdfa import SRGB_ICC_PROFILE, file_claims_pdfa, generate_pdfa_ps
from pikepdf.models.metadata import decode_pdf_date
@@ -330,23 +330,15 @@ def test_prevent_gs_invalid_xml(resources, outdir):
from ocrmypdf.pdfinfo import PdfInfo
generate_pdfa_ps(outdir / 'pdfa.ps')
- input_files = [str(outdir / 'layers.rendered.pdf'), str(outdir / 'pdfa.ps')]
copyfile(resources / 'enron1.pdf', outdir / 'layers.rendered.pdf')
- log = logging.getLogger()
- context = JobContext()
options = parser.parse_args(
args=['-j', '1', '--output-type', 'pdfa-2', 'a.pdf', 'b.pdf']
)
- context.options = options
- context.pdfinfo = PdfInfo(resources / 'enron1.pdf')
+ pdfinfo = PdfInfo(resources / 'enron1.pdf')
+ context = PDFContext(options, outdir, resources / 'enron1.pdf', pdfinfo)
- convert_to_pdfa(
- input_files_groups=input_files,
- output_file=outdir / 'pdfa.pdf',
- log=log,
- context=context,
- )
+ convert_to_pdfa(str(outdir / 'layers.rendered.pdf'), str(outdir / 'pdfa.ps'), context)
with open(outdir / 'pdfa.pdf', 'rb') as f:
with mmap.mmap(