From 783a128bd1bf3195a2c96b78690b8a26c0599689 Mon Sep 17 00:00:00 2001 From: mawi Date: Thu, 4 Apr 2019 21:02:38 +0200 Subject: [PATCH] feat: move to sync (none ETL) implementation - remove ruffus --- src/ocrmypdf/__main__.py | 6 +- src/ocrmypdf/_jobcontext.py | 130 ++- src/ocrmypdf/_pipeline.py | 489 ++++------- src/ocrmypdf/_pipeline_simple.py | 784 ------------------ src/ocrmypdf/_ruffus.py | 508 ------------ src/ocrmypdf/_sync.py | 63 +- src/ocrmypdf/optimize.py | 19 +- ...processing.py => _test_multiprocessing.py} | 0 tests/test_main.py | 2 +- tests/test_metadata.py | 16 +- 10 files changed, 277 insertions(+), 1740 deletions(-) delete mode 100644 src/ocrmypdf/_pipeline_simple.py delete mode 100644 src/ocrmypdf/_ruffus.py rename tests/{test_multiprocessing.py => _test_multiprocessing.py} (100%) diff --git a/src/ocrmypdf/__main__.py b/src/ocrmypdf/__main__.py index f9c6dced..afa3cb20 100755 --- a/src/ocrmypdf/__main__.py +++ b/src/ocrmypdf/__main__.py @@ -462,9 +462,9 @@ debugging.add_argument( action='store_true', help="Keep temporary files (helpful for debugging)", ) -debugging.add_argument( - '--flowchart', type=str, help="Generate the pipeline execution flowchart" -) +# debugging.add_argument( +# '--flowchart', type=str, help="Generate the pipeline execution flowchart" +# ) def run(args=None): diff --git a/src/ocrmypdf/_jobcontext.py b/src/ocrmypdf/_jobcontext.py index c9367ba6..6b059644 100644 --- a/src/ocrmypdf/_jobcontext.py +++ b/src/ocrmypdf/_jobcontext.py @@ -17,62 +17,41 @@ import shutil import sys +import os from contextlib import suppress -from multiprocessing.managers import SyncManager - -from .pdfinfo import PdfInfo -class JobContext: - """Holds our context for a particular run of the pipeline +class PDFContext: + """Holds our context for a particular run of the pipeline""" - A multiprocessing manager effectively creates a separate process - that keeps the master job context object. Other threads access - job context via multiprocessing proxy objects. - - While this would naturally lend itself @property's it seems to make - a little more sense to use functions to make it explicitly that the - invocation requires marshalling data across a process boundary. - - """ - - def __init__(self): - self.pdfinfo = None - self.options = None - self.work_folder = None - self.rotations = {} - - def generate_pdfinfo(self, infile): - self.pdfinfo = PdfInfo(infile) - - def get_pdfinfo(self): - "What we know about the input PDF" - return self.pdfinfo - - def set_pdfinfo(self, pdfinfo): - self.pdfinfo = pdfinfo - - def get_options(self): - return self.options - - def set_options(self, options): + def __init__(self, options, work_folder, origin, pdfinfo): self.options = options - - def get_work_folder(self): - return self.work_folder - - def set_work_folder(self, work_folder): self.work_folder = work_folder + self.origin = origin + self.pdfinfo = pdfinfo + self.log = get_logger(options, '%s: ' % os.path.basename(origin)) - def get_rotation(self, pageno): - return self.rotations.get(pageno, 0) + def get_path(self, name): + return os.path.join(self.work_folder, name) - def set_rotation(self, pageno, value): - self.rotations[pageno] = value + def get_page_contexts(self): + npages = len(self.pdfinfo) + for n in range(npages): + yield PageContext(self, n) -class JobContextManager(SyncManager): - pass +class PageContext: + """Holds our context for a page""" + + def __init__(self, pdf_context, pageno): + self.pdf_context = pdf_context + self.options = pdf_context.options + self.pageno = pageno + self.pageinfo = pdf_context.pdfinfo[pageno] + self.log = get_logger(pdf_context.options, '%s Page %d: ' % (os.path.basename(pdf_context.origin), pageno + 1)) + + def get_path(self, name): + return os.path.join(self.pdf_context.work_folder, "page_%d_%s" % (self.pageno, name)) def cleanup_working_files(work_folder, options): @@ -81,3 +60,62 @@ def cleanup_working_files(work_folder, options): else: with suppress(FileNotFoundError): shutil.rmtree(work_folder) + + +def get_logger(options=None, prefix=''): + level = INFO # TODO: add option + if options is not None and options.output_file == '-' or options.sidecar == '-': + return NullLogger() + return Logger(prefix, level) + + +ERROR = 40 +WARN = 30 +INFO = 20 +DEBUG = 10 + + +class Logger: + def __init__(self, prefix, level=INFO): + self.prefix = prefix + self.level = level + + def debug(self, *args, **kwargs): + if self.level <= DEBUG: + print('DEBUG', self.prefix, end='') + print(*args, **kwargs) + + def info(self, *args, **kwargs): + if self.level <= INFO: + print('INFO', self.prefix, end='') + print(*args, **kwargs) + + def warning(self, *args, **kwargs): + self.warn(*args, **kwargs) + + def warn(self, *args, **kwargs): + if self.level <= WARN: + print('WARN', self.prefix, end='') + print(*args, **kwargs) + + def error(self, *args, **kwargs): + if self.level <= ERROR: + print('ERROR', self.prefix, end='') + print(*args, **kwargs) + + +class NullLogger: + def debug(self, *args, **kwargs): + pass + + def info(self, *args, **kwargs): + pass + + def warning(self, *args, **kwargs): + pass + + def warn(self, *args, **kwargs): + pass + + def error(self, *args, **kwargs): + pass diff --git a/src/ocrmypdf/_pipeline.py b/src/ocrmypdf/_pipeline.py index d6e0c491..3b553a7e 100644 --- a/src/ocrmypdf/_pipeline.py +++ b/src/ocrmypdf/_pipeline.py @@ -18,10 +18,8 @@ import os import re import sys -from contextlib import suppress from datetime import datetime, timezone -from pathlib import Path -from shutil import copyfile, copyfileobj +from shutil import copyfileobj import img2pdf from PIL import Image @@ -34,14 +32,10 @@ from .exceptions import ( DpiError, EncryptedPdfError, InputFileError, - PriorOcrFoundError, UnsupportedImageFormatError, ) from .exec import ghostscript, tesseract from .helpers import ( - flatten_groups, - is_iterable_notstr, - page_number, re_symlink ) from .hocrtransform import HocrTransform @@ -51,10 +45,6 @@ from .pdfinfo import Colorspace, PdfInfo VECTOR_PAGE_DPI = 400 -# -# The Pipeline -# - def triage_image_file(input_file, output_file, log, options): try: @@ -164,31 +154,28 @@ def triage(input_file, output_file, log, context): triage_image_file(input_file, output_file, log, options) -def repair_and_parse_pdf(input_file, output_file, log, context): - options = context.get_options() - copyfile(input_file, output_file) - - detailed_page_analysis = False - if options.redo_ocr: - detailed_page_analysis = True - +def get_pdfinfo(input_file, detailed_page_analysis=False): try: - pdfinfo = PdfInfo( - output_file, detailed_page_analysis=detailed_page_analysis, log=log + return PdfInfo( + input_file, detailed_page_analysis=detailed_page_analysis ) except pikepdf.PasswordError: raise EncryptedPdfError() - except pikepdf.PdfError as e: - log.error(e) + except pikepdf.PdfError: raise InputFileError() + +def validate_pdfinfo_options(context): + log = context.log + pdfinfo = context.pdfinfo + options = context.options + if pdfinfo.needs_rendering: log.error( "This PDF contains dynamic XFA forms created by Adobe LiveCycle " "Designer and can only be read by Adobe Acrobat or Adobe Reader." ) raise InputFileError() - if pdfinfo.has_userunit and options.output_type.startswith('pdfa'): log.error( "This input file uses a PDF feature that is not supported " @@ -198,16 +185,15 @@ def repair_and_parse_pdf(input_file, output_file, log, context): "output these files.) Use --output-type=pdf instead." ) raise InputFileError() - if pdfinfo.has_acroform: if options.redo_ocr: log.error( "This PDF has a user fillable form. --redo-ocr is not " "currently possible on such files." ) - raise PriorOcrFoundError() + raise InputFileError() else: - log.warning( + log.warn( "This PDF has a fillable form. " "Chances are it is a pure digital " "document that does not need OCR." @@ -219,16 +205,6 @@ def repair_and_parse_pdf(input_file, output_file, log, context): "'flattened' and will no longer be fillable." ) - context.set_pdfinfo(pdfinfo) - log.debug(pdfinfo) - - -def get_pageinfo(input_file, context): - "Get zero-based page info implied by filename, e.g. 000002.pdf -> 1" - pageno = page_number(input_file) - 1 - pageinfo = context.get_pdfinfo()[pageno] - return pageinfo - def get_page_dpi(pageinfo, options): "Get the DPI when nonsquare DPI is tolerable" @@ -272,33 +248,31 @@ def get_canvas_square_dpi(pageinfo, options): ) -def is_ocr_required(pageinfo, log, options): - page = pageinfo.pageno + 1 +def is_ocr_required(page_context): + pageinfo = page_context.pageinfo + options = page_context.options + log = page_context.log + ocr_required = True if pageinfo.has_text: - prefix = f"{page:4d}: page already has text! - " - if not options.force_ocr and not (options.skip_text or options.redo_ocr): - log.error(prefix + "aborting (use --force-ocr to force OCR)") - raise PriorOcrFoundError() + log.error("page already has text! - aborting (use --force-ocr to force OCR)") + ocr_required = False elif options.force_ocr: - log.info(prefix + "rasterizing text and running OCR anyway") + log.info("page already has text! - rasterizing text and running OCR anyway") ocr_required = True elif options.redo_ocr: if pageinfo.has_corrupt_text: - log.warning( - prefix + ( - "some text on this page cannot be mapped to characters: " - "consider using --force-ocr instead", - ) + log.warn( + "some text on this page cannot be mapped to characters: " + "consider using --force-ocr instead", ) - raise PriorOcrFoundError() # Wrong error but will do for now else: - log.info(prefix + "redoing OCR") + log.info("redoing OCR") ocr_required = True elif options.skip_text: - log.info(prefix + "skipping all processing on this page") + log.info("skipping all processing on this page") ocr_required = False elif not pageinfo.images and not options.lossless_reconstruction: # We found a page with no images and no text. That means it may @@ -311,14 +285,14 @@ def is_ocr_required(pageinfo, log, options): if options.force_ocr and options.oversample: # The user really wants to reprocess this file log.info( - f"{page:4d}: page has no images - " + "page has no images - " f"rasterizing at {options.oversample} DPI because " "--force-ocr --oversample was specified" ) elif options.force_ocr: # Warn the user they might not want to do this - log.warning( - f"{page:4d}: page has no images - " + log.warn( + "page has no images - " "all vector content will be " f"rasterized at {VECTOR_PAGE_DPI} DPI, losing some resolution and likely " "increasing file size. Use --oversample to adjust the " @@ -326,7 +300,7 @@ def is_ocr_required(pageinfo, log, options): ) else: log.info( - f"{page:4d}: page has no images - " + "page has no images - " "skipping all processing on this page to avoid losing detail. " "Use --force-ocr if you wish to perform OCR on pages that " "have vector content." @@ -337,82 +311,31 @@ def is_ocr_required(pageinfo, log, options): pixel_count = pageinfo.width_pixels * pageinfo.height_pixels if pixel_count > (options.skip_big * 1_000_000): ocr_required = False - log.warning( - f"{page:4d}: page too big, skipping OCR " + log.warn( + "page too big, skipping OCR " f"({(pixel_count / 1_000_000):.1f} MPixels > {options.skip_big:.1f} MPixels --skip-big)" ) return ocr_required -def marker_pages(input_files, output_files, log, context): - - options = context.get_options() - work_folder = context.get_work_folder() - - if is_iterable_notstr(input_files): - input_file = input_files[0] - else: - input_file = input_files - - for oo in output_files: - with suppress(FileNotFoundError): - os.unlink(oo) - - # If no files were repaired the input will be empty - if not input_file: - log.error(f"{options.input_file}: file not found or invalid argument") - raise InputFileError() - - pdfinfo = context.get_pdfinfo() - npages = len(pdfinfo) - - # Ruffus needs to see a file for any task it generates, so make very - # file a symlink back to the source. - for n in range(npages): - page = Path(work_folder) / f'{(n + 1):06d}.marker.pdf' - page.symlink_to(input_file) # pylint: disable=E1101 - - -def ocr_or_skip(input_files, output_files, log, context): - options = context.get_options() - work_folder = context.get_work_folder() - pdfinfo = context.get_pdfinfo() - - for input_file in input_files: - pageno = page_number(input_file) - 1 - pageinfo = pdfinfo[pageno] - alt_suffix = ( - '.ocr.page.pdf' - if is_ocr_required(pageinfo, log, options) - else '.skip.page.pdf' - ) - - re_symlink( - input_file, - os.path.join(work_folder, os.path.basename(input_file)[0:6] + alt_suffix), - log, - ) - - -def rasterize_preview(input_file, output_file, log, context): - pageinfo = get_pageinfo(input_file, context) - options = context.get_options() - canvas_dpi = get_canvas_square_dpi(pageinfo, options) - page_dpi = get_page_square_dpi(pageinfo, options) - +def rasterize_preview(input_file, page_context): + output_file = page_context.get_path('rasterize_preview.jpg') + canvas_dpi = get_canvas_square_dpi(page_context.pageinfo, page_context.options) + page_dpi = get_page_square_dpi(page_context.pageinfo, page_context.options) ghostscript.rasterize_pdf( input_file, output_file, xres=canvas_dpi, yres=canvas_dpi, raster_device='jpeggray', - log=log, + log=page_context.log, page_dpi=(page_dpi, page_dpi), - pageno=page_number(input_file), + pageno=page_context.pageinfo.pageno + 1, ) + return output_file -def orient_page(infiles, output_file, log, context): +def get_orientation_correction(preview, page_context): """ Work out orientation correct for each page. @@ -430,32 +353,22 @@ def orient_page(infiles, output_file, log, context): """ - options = context.get_options() - page_pdf = next(ii for ii in infiles if ii.endswith('.page.pdf')) - - if not options.rotate_pages: - re_symlink(page_pdf, output_file, log) - return - preview = next(ii for ii in infiles if ii.endswith('.preview.jpg')) - orient_conf = tesseract.get_orientation( preview, - engine_mode=options.tesseract_oem, - timeout=options.tesseract_timeout, - log=log, + engine_mode=page_context.options.tesseract_oem, + timeout=page_context.options.tesseract_timeout, + log=page_context.log, ) direction = {0: '⇧', 90: '⇨', 180: '⇩', 270: '⇦'} - pageno = page_number(page_pdf) - 1 - pdfinfo = context.get_pdfinfo() - existing_rotation = pdfinfo[pageno].rotation + existing_rotation = page_context.pageinfo.rotation correction = orient_conf.angle % 360 apply_correction = False action = '' - if orient_conf.confidence >= options.rotate_pages_threshold: + if orient_conf.confidence >= page_context.options.rotate_pages_threshold: if correction != 0: apply_correction = True action = ' - will rotate' @@ -474,26 +387,25 @@ def orient_page(infiles, output_file, log, context): ) facing += 'page is facing {}'.format(direction.get(orient_conf.angle, '?')) - log.info( + page_context.log.debug( '{pagenum:4d}: {facing}, confidence {conf:.2f}{action}'.format( - pagenum=page_number(preview), + pagenum=page_context.pageinfo.pageno, facing=facing, conf=orient_conf.confidence, action=action, ) ) - re_symlink(page_pdf, output_file, log) if apply_correction: - context.set_rotation(pageno, correction) + return correction + return 0 -def rasterize_with_ghostscript(input_file, output_file, log, context): - options = context.get_options() - pageinfo = get_pageinfo(input_file, context) - +def rasterize(input_file, page_context, correction=0): colorspaces = ['pngmono', 'pnggray', 'png256', 'png16m'] device_idx = 0 + output_file = page_context.get_path('rasterize.png') + pageinfo = page_context.pageinfo def at_least(cs): return max(device_idx, colorspaces.index(cs)) @@ -511,14 +423,12 @@ def rasterize_with_ghostscript(input_file, output_file, log, context): device = colorspaces[device_idx] - log.debug(f"Rasterize {os.path.basename(input_file)} with {device}") + page_context.log.debug(f"Rasterize with {device}") # Produce the page image with square resolution or else deskew and OCR # will not work properly. - canvas_dpi = get_canvas_square_dpi(pageinfo, options) - page_dpi = get_page_square_dpi(pageinfo, options) - - correction = context.get_rotation(page_number(input_file) - 1) + canvas_dpi = get_canvas_square_dpi(pageinfo, page_context.options) + page_dpi = get_page_square_dpi(pageinfo, page_context.options) ghostscript.rasterize_pdf( input_file, @@ -526,64 +436,47 @@ def rasterize_with_ghostscript(input_file, output_file, log, context): xres=canvas_dpi, yres=canvas_dpi, raster_device=device, - log=log, + log=page_context.log, page_dpi=(page_dpi, page_dpi), - pageno=page_number(input_file), + pageno=pageinfo.pageno + 1, rotation=correction, - filter_vector=options.remove_vectors, + filter_vector=page_context.options.remove_vectors, ) + return output_file -def preprocess_remove_background(input_file, output_file, log, context): - options = context.get_options() - if not options.remove_background: - re_symlink(input_file, output_file, log) - return - - pageinfo = get_pageinfo(input_file, context) - - if any(image.bpc > 1 for image in pageinfo.images): +def preprocess_remove_background(input_file, page_context): + if any(image.bpc > 1 for image in page_context.pageinfo.images): + output_file = page_context.get_path('pp_rm_bg.png') leptonica.remove_background(input_file, output_file) + return output_file else: - log.info(f"{pageinfo.pageno:4d}: background removal skipped on mono page") - re_symlink(input_file, output_file, log) + page_context.log.info("background removal skipped on mono page") + return input_file -def preprocess_deskew(input_file, output_file, log, context): - options = context.get_options() - if not options.deskew: - re_symlink(input_file, output_file, log) - return - - pageinfo = get_pageinfo(input_file, context) - dpi = get_page_square_dpi(pageinfo, options) - +def preprocess_deskew(input_file, page_context): + output_file = page_context.get_path('pp_deskew.png') + dpi = get_page_square_dpi(page_context.pageinfo, page_context.options) leptonica.deskew(input_file, output_file, dpi) + return output_file -def preprocess_clean(input_file, output_file, log, context): - options = context.get_options() - if not options.clean: - re_symlink(input_file, output_file, log) - return - +def preprocess_clean(input_file, page_context): from .exec import unpaper - - pageinfo = get_pageinfo(input_file, context) - dpi = get_page_square_dpi(pageinfo, options) - - unpaper.clean(input_file, output_file, dpi, log, options.unpaper_args) + output_file = page_context.get_path('pp_clean.png') + dpi = get_page_square_dpi(page_context.pageinfo, page_context.options) + unpaper.clean(input_file, output_file, dpi, page_context.log, page_context.options.unpaper_args) + return output_file -def select_ocr_image(infiles, output_file, log, context): - """Select the image we send for OCR. May not be the same as the display +def create_ocr_image(image, page_context): + """Create the image we send for OCR. May not be the same as the display image depending on preprocessing. This image will never be shown to the user.""" - image = infiles[0] - options = context.get_options() - pageinfo = get_pageinfo(image, context) - + output_file = page_context.get_path('ocr.png') + options = page_context.options with Image.open(image) as im: from PIL import ImageColor from PIL import ImageDraw @@ -593,7 +486,7 @@ def select_ocr_image(infiles, output_file, log, context): draw = ImageDraw.ImageDraw(im) xres, yres = im.info['dpi'] - log.debug('resolution %r %r', xres, yres) + page_context.log.info('resolution %r %r' % (xres, yres)) if not options.force_ocr: # Do not mask text areas when forcing OCR, because we need to OCR @@ -602,7 +495,7 @@ def select_ocr_image(infiles, output_file, log, context): if options.redo_ocr: mask = True # Mask visible text, but not invisible text - for textarea in pageinfo.get_textareas(visible=mask, corrupt=None): + for textarea in page_context.pageinfo.get_textareas(visible=mask, corrupt=None): # Calculate resolution based on the image size and page dimensions # without regard whatever resolution is in pageinfo (may differ or # be None) @@ -615,7 +508,7 @@ def select_ocr_image(infiles, output_file, log, context): im.height - bbox[1] * yscale, ] pixcoords = [int(round(c)) for c in pixcoords] - log.debug('blanking %r', pixcoords) + print('blanking %r', pixcoords) draw.rectangle(pixcoords, fill=white) # draw.rectangle(pixcoords, outline=pink) @@ -627,7 +520,7 @@ def select_ocr_image(infiles, output_file, log, context): barcodes = pix.locate_barcodes() for barcode in barcodes: decoded, rect = barcode - log.info('masking barcode %s %r', decoded, rect) + print('masking barcode %s %r', decoded, rect) draw.rectangle(rect, fill=white) im = pix.topil() @@ -635,13 +528,16 @@ def select_ocr_image(infiles, output_file, log, context): # Pillow requires integer DPI dpi = round(xres), round(yres) im.save(output_file, dpi=dpi) + return output_file -def ocr_tesseract_hocr(input_file, output_files, log, context): - options = context.get_options() +def ocr_tesseract_hocr(input_file, page_context): + hocr_out = page_context.get_path('ocr_hocr.hocr') + hocr_text_out = page_context.get_path('ocr_hocr.txt') + options = page_context.options tesseract.generate_hocr( input_file=input_file, - output_files=output_files, + output_files=[hocr_out, hocr_text_out], language=options.language, engine_mode=options.tesseract_oem, tessconfig=options.tesseract_config, @@ -649,86 +545,57 @@ def ocr_tesseract_hocr(input_file, output_files, log, context): pagesegmode=options.tesseract_pagesegmode, user_words=options.user_words, user_patterns=options.user_patterns, - log=log, + log=page_context.log, ) + return (hocr_out, hocr_text_out) -def select_visible_page_image(infiles, output_file, log, context): - """Selects a whole page image that we can show the user (if necessary)""" - - options = context.get_options() - if options.clean_final: - image_suffix = '.pp-clean.png' - elif options.deskew: - image_suffix = '.pp-deskew.png' - elif options.remove_background: - image_suffix = '.pp-background.png' - else: - image_suffix = '.page.png' - image = next(ii for ii in infiles if ii.endswith(image_suffix)) - - pageinfo = get_pageinfo(image, context) - if pageinfo.images and all(im.enc == 'jpeg' for im in pageinfo.images): - log.debug(f'{page_number(image):4d}: JPEG input -> JPEG output') - # If all images were JPEGs originally, produce a JPEG as output - with Image.open(image) as im: - # At this point the image should be a .png, but deskew, unpaper - # might have removed the DPI information. In this case, fall back to - # square DPI used to rasterize. When the preview image was - # rasterized, it was also converted to square resolution, which is - # what we want to give tesseract, so keep it square. - fallback_dpi = get_page_square_dpi(pageinfo, options) - dpi = im.info.get('dpi', (fallback_dpi, fallback_dpi)) - - # Pillow requires integer DPI - dpi = round(dpi[0]), round(dpi[1]) - im.save(output_file, format='JPEG', dpi=dpi) - else: - re_symlink(image, output_file, log) +def should_visible_page_image_use_jpg(pageinfo): + # If all images were JPEGs originally, produce a JPEG as output + return pageinfo.images and all(im.enc == 'jpeg' for im in pageinfo.images) -def select_image_layer(infiles, output_file, log, context): - """Selects the image layer for the output page. If possible this is the - orientation-corrected input page, or an image of the whole page converted - to PDF.""" +def create_visible_page_jpg(image, page_context): + output_file = page_context.get_path('visible.jpg') + with Image.open(image) as im: + # At this point the image should be a .png, but deskew, unpaper + # might have removed the DPI information. In this case, fall back to + # square DPI used to rasterize. When the preview image was + # rasterized, it was also converted to square resolution, which is + # what we want to give tesseract, so keep it square. + fallback_dpi = get_page_square_dpi(page_context.pageinfo, page_context.options) + dpi = im.info.get('dpi', (fallback_dpi, fallback_dpi)) - options = context.get_options() - page_pdf = next(ii for ii in infiles if ii.endswith('.ocr.oriented.pdf')) - image = next(ii for ii in infiles if ii.endswith('.image')) + # Pillow requires integer DPI + dpi = round(dpi[0]), round(dpi[1]) + im.save(output_file, format='JPEG', dpi=dpi) + return output_file - if options.lossless_reconstruction: - log.debug( - f"{page_number(page_pdf):4d}: page eligible for lossless reconstruction" - ) - re_symlink(page_pdf, output_file, log) # Still points to multipage - return - - pageinfo = get_pageinfo(image, context) +def create_pdf_page_from_image(image, page_context): # We rasterize a square DPI version of each page because most image # processing tools don't support rectangular DPI. Use the square DPI as it # accurately describes the image. It would be possible to resample the image # at this stage back to non-square DPI to more closely resemble the input, # except that the hocr renderer does not understand non-square DPI. The # sandwich renderer would be fine. - dpi = get_page_square_dpi(pageinfo, options) + output_file = page_context.get_path('visible.pdf') + dpi = get_page_square_dpi(page_context.pageinfo, page_context.options) layout_fun = img2pdf.get_fixed_dpi_layout_fun((dpi, dpi)) # This create a single page PDF with open(image, 'rb') as imfile, open(output_file, 'wb') as pdf: - log.debug(f'{page_number(page_pdf):4d}: convert') + page_context.log.debug('convert') img2pdf.convert( imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf ) - log.debug(f'{page_number(page_pdf):4d}: convert done') + page_context.log.debug('convert done') + return output_file -def render_hocr_page(infiles, output_file, log, context): - options = context.get_options() - hocr = next(ii for ii in infiles if ii.endswith('.hocr')) - pageinfo = get_pageinfo(hocr, context) - dpi = get_page_square_dpi(pageinfo, options) - +def render_hocr_page(hocr, page_context): + output_file = page_context.get_path('ocr_hocr.pdf') + dpi = get_page_square_dpi(page_context.pageinfo, page_context.options) hocrtransform = HocrTransform(hocr, dpi) hocrtransform.to_pdf( output_file, @@ -737,17 +604,13 @@ def render_hocr_page(infiles, output_file, log, context): invisibleText=True, interwordSpaces=True, ) + return output_file -def ocr_tesseract_textonly_pdf(infiles, outfiles, log, context): - options = context.get_options() - input_image = next((ii for ii in infiles if ii.endswith('.ocr.png')), '') - if not input_image: - raise ValueError("No image rendered?") - - output_pdf = next((ii for ii in outfiles if ii.endswith('.pdf'))) - output_text = next((ii for ii in outfiles if ii.endswith('.txt'))) - +def ocr_tesseract_textonly_pdf(input_image, page_context): + output_pdf = page_context.get_path('ocr_tess.pdf') + output_text = page_context.get_path('ocr_tess.txt') + options = page_context.options tesseract.generate_pdf( input_image=input_image, skip_pdf=None, @@ -761,8 +624,9 @@ def ocr_tesseract_textonly_pdf(infiles, outfiles, log, context): pagesegmode=options.tesseract_pagesegmode, user_words=options.user_words, user_patterns=options.user_patterns, - log=log, + log=page_context.log, ) + return (output_pdf, output_text) def get_docinfo(base_pdf, options): @@ -804,63 +668,51 @@ def get_docinfo(base_pdf, options): return pdfmark -def generate_postscript_stub(input_file, output_file, log, context): +def generate_postscript_stub(context): + output_file = context.get_path('pdfa.ps') generate_pdfa_ps(output_file) + return output_file -def convert_to_pdfa(input_files_groups, output_file, log, context): - options = context.get_options() - input_pdfinfo = context.get_pdfinfo() - - input_files = list(f for f in flatten_groups(input_files_groups)) - layers_file = next( - (ii for ii in input_files if ii.endswith('layers.rendered.pdf')), None - ) +def convert_to_pdfa(input_pdf, input_ps_stub, context): + options = context.options + input_pdfinfo = context.pdfinfo + output_file = context.get_path('pdfa.pdf') # If the DocumentInfo record contains NUL characters, Ghostscript will # produce XMP metadata which contains invalid XML entities (�). # NULs in DocumentInfo seem to be common since older Acrobats included them. # pikepdf can deal with this, but we make the world a better place by # stamping them out as soon as possible. - pdf_layers_file = pikepdf.open(layers_file) - if pdf_layers_file.docinfo: + pdf_file = pikepdf.open(input_pdf) + if pdf_file.docinfo: modified = False - for k, v in pdf_layers_file.docinfo.items(): + for k, v in pdf_file.docinfo.items(): if b'\x00' in bytes(v): - pdf_layers_file.docinfo[k] = bytes(v).replace(b'\x00', b'') + pdf_file.docinfo[k] = bytes(v).replace(b'\x00', b'') modified = True if modified: - pdf_layers_file.save(layers_file) - del pdf_layers_file + pdf_file.save(input_pdf) + del pdf_file - ps = next((ii for ii in input_files if ii.endswith('.ps')), None) ghostscript.generate_pdfa( pdf_version=input_pdfinfo.min_version, - pdf_pages=[layers_file, ps], + pdf_pages=[input_pdf, input_ps_stub], output_file=output_file, compression=options.pdfa_image_compression, - log=log, + log=context.log, threads=options.jobs or 1, pdfa_part=options.output_type[-1], # is pdfa-1, pdfa-2, or pdfa-3 ) + return output_file -def metadata_fixup(input_files_groups, output_file, log, context): - options = context.get_options() - input_files = list(f for f in flatten_groups(input_files_groups)) - original_file = next( - (ii for ii in input_files if ii.endswith('.repaired.pdf')), None - ) - layers_file = next( - (ii for ii in input_files if ii.endswith('layers.rendered.pdf')), None - ) - pdfa_file = next((ii for ii in input_files if ii.endswith('pdfa.pdf')), None) - original = pikepdf.open(original_file) +def metadata_fixup(working_file, context): + output_file = context.get_path('metafix.pdf') + options = context.options + original = pikepdf.open(context.origin) docinfo = get_docinfo(original, options) - - working_file = pdfa_file if pdfa_file else layers_file - pdf = pikepdf.open(working_file) with pdf.open_metadata() as meta: meta.load_from_docinfo(docinfo, delete_missing=False) @@ -868,16 +720,25 @@ def metadata_fixup(input_files_groups, output_file, log, context): # match Ghostscript, for consistency if 'xmp:CreateDate' not in meta: meta['xmp:CreateDate'] = meta.get('xmp:ModifyDate', '') - if pdfa_file: - meta_original = original.open_metadata() - not_copied = set(meta_original.keys()) - set(meta.keys()) - if not_copied: - log.warning( + + meta_original = original.open_metadata() + not_copied = set(meta_original.keys()) - set(meta.keys()) + if not_copied: + if options.output_type.startswith('pdfa'): + context.log.warn( "Some input metadata could not be copied because it is not " "permitted in PDF/A. You may wish to examine the output " "PDF's XMP metadata." ) - log.debug( + context.log.debug( + "The following metadata fields were not copied: %r", not_copied + ) + else: + context.log.error( + "Some input metadata could not be copied." + "You may wish to examine the output PDF's XMP metadata." + ) + context.log.info( "The following metadata fields were not copied: %r", not_copied ) @@ -886,23 +747,18 @@ def metadata_fixup(input_files_groups, output_file, log, context): compress_streams=True, object_stream_mode=pikepdf.ObjectStreamMode.generate, ) + return output_file -def optimize_pdf(input_file, output_file, log, context): - optimize(input_file, output_file, log, context) +def optimize_pdf(input_file, context): + output_file = context.get_path('optimize.pdf') + optimize(input_file, output_file, context) + return output_file -def merge_sidecars(input_files_groups, output_file, log, context): - pdfinfo = context.get_pdfinfo() - - txt_files = [None] * len(pdfinfo) - - for infile in flatten_groups(input_files_groups): - if infile.endswith('.txt'): - idx = page_number(infile) - 1 - txt_files[idx] = infile - - def write_pages(stream): +def merge_sidecars(txt_files, context): + output_file = context.get_path('sidecar.txt') + with open(output_file, 'w', encoding="utf-8") as stream: for page_num, txt_file in enumerate(txt_files): if page_num != 0: stream.write('\f') # Form feed between pages @@ -920,18 +776,11 @@ def merge_sidecars(input_files_groups, output_file, log, context): stream.write(txt) else: stream.write(f'[OCR skipped on page {(page_num + 1)}]') - - if output_file == '-': - write_pages(sys.stdout) - sys.stdout.flush() - else: - with open(output_file, 'w', encoding="utf-8") as out: - write_pages(out) + return output_file -def copy_final(input_files, output_file, log, context): - input_file = next((ii for ii in input_files if ii.endswith('.pdf'))) - log.debug('%s -> %s', input_file, output_file) +def copy_final(input_file, output_file, context): + context.log.debug('%s -> %s', input_file, output_file) with open(input_file, 'rb') as input_stream: if output_file == '-': copyfileobj(input_stream, sys.stdout.buffer) diff --git a/src/ocrmypdf/_pipeline_simple.py b/src/ocrmypdf/_pipeline_simple.py deleted file mode 100644 index 3b89df3d..00000000 --- a/src/ocrmypdf/_pipeline_simple.py +++ /dev/null @@ -1,784 +0,0 @@ -# © 2016 James R. Barlow: github.com/jbarlow83 -# -# This file is part of OCRmyPDF. -# -# OCRmyPDF is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# OCRmyPDF is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with OCRmyPDF. If not, see . - -import os -import re -import sys -from datetime import datetime, timezone -from shutil import copyfileobj - -import img2pdf -from PIL import Image - -import pikepdf -from pikepdf.models.metadata import encode_pdf_date - -from . import PROGRAM_NAME, VERSION, leptonica -from .exceptions import ( - DpiError, - EncryptedPdfError, - InputFileError, - UnsupportedImageFormatError, -) -from .exec import ghostscript, tesseract -from .helpers import ( - re_symlink -) -from .hocrtransform import HocrTransform -from .optimize import optimize -from .pdfa import generate_pdfa_ps -from .pdfinfo import Colorspace, PdfInfo - -VECTOR_PAGE_DPI = 400 - - -def triage_image_file(input_file, output_file, log, options): - try: - log.info("Input file is not a PDF, checking if it is an image...") - im = Image.open(input_file) - except EnvironmentError as e: - msg = str(e) - - # Recover the original filename - realpath = '' - if os.path.islink(input_file): - realpath = os.path.realpath(input_file) - elif os.path.isfile(input_file): - realpath = '' - msg = msg.replace(input_file, realpath) - log.error(msg) - raise UnsupportedImageFormatError() from e - else: - log.info("Input file is an image") - - if 'dpi' in im.info: - if im.info['dpi'] <= (96, 96) and not options.image_dpi: - log.info("Image size: (%d, %d)" % im.size) - log.info("Image resolution: (%d, %d)" % im.info['dpi']) - log.error( - "Input file is an image, but the resolution (DPI) is " - "not credible. Estimate the resolution at which the " - "image was scanned and specify it using --image-dpi." - ) - raise DpiError() - elif not options.image_dpi: - log.info("Image size: (%d, %d)" % im.size) - log.error( - "Input file is an image, but has no resolution (DPI) " - "in its metadata. Estimate the resolution at which " - "image was scanned and specify it using --image-dpi." - ) - raise DpiError() - - if im.mode in ('RGBA', 'LA'): - log.error( - "The input image has an alpha channel. Remove the alpha " - "channel first." - ) - raise UnsupportedImageFormatError() - - if 'iccprofile' not in im.info: - if im.mode == 'RGB': - log.info('Input image has no ICC profile, assuming sRGB') - elif im.mode == 'CMYK': - log.info('Input CMYK image has no ICC profile, not usable') - raise UnsupportedImageFormatError() - im.close() - - try: - log.info("Image seems valid. Try converting to PDF...") - layout_fun = img2pdf.default_layout_fun - if options.image_dpi: - layout_fun = img2pdf.get_fixed_dpi_layout_fun( - (options.image_dpi, options.image_dpi) - ) - with open(output_file, 'wb') as outf: - img2pdf.convert( - input_file, - layout_fun=layout_fun, - with_pdfrw=False, - outputstream=outf - ) - log.info("Successfully converted to PDF, processing...") - except img2pdf.ImageOpenError as e: - log.error(e) - raise UnsupportedImageFormatError() from e - - -def _pdf_guess_version(input_file, search_window=1024): - """Try to find version signature at start of file. - - Not robust enough to deal with appended files. - - Returns empty string if not found, indicating file is probably not PDF. - """ - - with open(input_file, 'rb') as f: - signature = f.read(search_window) - m = re.search(br'%PDF-(\d\.\d)', signature) - if m: - return m.group(1) - return '' - - -def triage(input_file, output_file, log, context): - - options = context.get_options() - try: - if _pdf_guess_version(input_file): - if options.image_dpi: - log.warning( - "Argument --image-dpi ignored because the " - "input file is a PDF, not an image." - ) - re_symlink(input_file, output_file, log) - return - except EnvironmentError as e: - log.error(e) - raise InputFileError() from e - - triage_image_file(input_file, output_file, log, options) - - -def get_pdfinfo(input_file, detailed_page_analysis=False): - try: - return PdfInfo( - input_file, detailed_page_analysis=detailed_page_analysis - ) - except pikepdf.PasswordError: - raise EncryptedPdfError() - except pikepdf.PdfError: - raise InputFileError() - - -def validate_pdfinfo_options(context): - log = context.log - pdfinfo = context.pdfinfo - options = context.options - - if pdfinfo.needs_rendering: - log.error( - "This PDF contains dynamic XFA forms created by Adobe LiveCycle " - "Designer and can only be read by Adobe Acrobat or Adobe Reader." - ) - raise InputFileError() - if pdfinfo.has_userunit and options.output_type.startswith('pdfa'): - log.error( - "This input file uses a PDF feature that is not supported " - "by Ghostscript, so you cannot use --output-type=pdfa for this " - "file. (Specifically, it uses the PDF-1.6 /UserUnit feature to " - "support very large or small page sizes, and Ghostscript cannot " - "output these files.) Use --output-type=pdf instead." - ) - raise InputFileError() - if pdfinfo.has_acroform: - if options.redo_ocr: - log.error( - "This PDF has a user fillable form. --redo-ocr is not " - "currently possible on such files." - ) - raise InputFileError() - else: - log.warn( - "This PDF has a fillable form. " - "Chances are it is a pure digital " - "document that does not need OCR." - ) - if not options.force_ocr: - log.info( - "Use the option --force-ocr to produce an image of the " - "form and all filled form fields. The output PDF will be " - "'flattened' and will no longer be fillable." - ) - - -def get_page_dpi(pageinfo, options): - "Get the DPI when nonsquare DPI is tolerable" - xres = max( - pageinfo.xres or VECTOR_PAGE_DPI, - options.oversample or 0, - VECTOR_PAGE_DPI if pageinfo.has_vector else 0, - ) - yres = max( - pageinfo.yres or VECTOR_PAGE_DPI, - options.oversample or 0, - VECTOR_PAGE_DPI if pageinfo.has_vector else 0, - ) - return (float(xres), float(yres)) - - -def get_page_square_dpi(pageinfo, options): - "Get the DPI when we require xres == yres, scaled to physical units" - xres = pageinfo.xres or 0 - yres = pageinfo.yres or 0 - userunit = pageinfo.userunit or 1 - return float( - max( - (xres * userunit) or VECTOR_PAGE_DPI, - (yres * userunit) or VECTOR_PAGE_DPI, - VECTOR_PAGE_DPI if pageinfo.has_vector else 0, - options.oversample or 0, - ) - ) - - -def get_canvas_square_dpi(pageinfo, options): - """Get the DPI when we require xres == yres, in Postscript units""" - return float( - max( - (pageinfo.xres) or VECTOR_PAGE_DPI, - (pageinfo.yres) or VECTOR_PAGE_DPI, - VECTOR_PAGE_DPI if pageinfo.has_vector else 0, - options.oversample or 0, - ) - ) - - -def is_ocr_required(page_context): - pageinfo = page_context.pageinfo - options = page_context.options - log = page_context.log - - ocr_required = True - - if pageinfo.has_text: - if not options.force_ocr and not (options.skip_text or options.redo_ocr): - log.error("page already has text! - aborting (use --force-ocr to force OCR)") - ocr_required = False - elif options.force_ocr: - log.info("page already has text! - rasterizing text and running OCR anyway") - ocr_required = True - elif options.redo_ocr: - if pageinfo.has_corrupt_text: - log.warn( - "some text on this page cannot be mapped to characters: " - "consider using --force-ocr instead", - ) - else: - log.info("redoing OCR") - ocr_required = True - elif options.skip_text: - log.info("skipping all processing on this page") - ocr_required = False - elif not pageinfo.images and not options.lossless_reconstruction: - # We found a page with no images and no text. That means it may - # have vector art that the user wants to OCR. If we determined - # lossless reconstruction is not possible then we have to rasterize - # the image. So if OCR is being forced, take that to mean YES, go - # ahead and rasterize. If not forced, then pretend there's no text - # on the page at all so we don't lose anything. - # This could be made smarter by explicitly searching for vector art. - if options.force_ocr and options.oversample: - # The user really wants to reprocess this file - log.info( - "page has no images - " - f"rasterizing at {options.oversample} DPI because " - "--force-ocr --oversample was specified" - ) - elif options.force_ocr: - # Warn the user they might not want to do this - log.warn( - "page has no images - " - "all vector content will be " - f"rasterized at {VECTOR_PAGE_DPI} DPI, losing some resolution and likely " - "increasing file size. Use --oversample to adjust the " - "DPI." - ) - else: - log.info( - "page has no images - " - "skipping all processing on this page to avoid losing detail. " - "Use --force-ocr if you wish to perform OCR on pages that " - "have vector content." - ) - ocr_required = False - - if ocr_required and options.skip_big and pageinfo.images: - pixel_count = pageinfo.width_pixels * pageinfo.height_pixels - if pixel_count > (options.skip_big * 1_000_000): - ocr_required = False - log.warn( - "page too big, skipping OCR " - f"({(pixel_count / 1_000_000):.1f} MPixels > {options.skip_big:.1f} MPixels --skip-big)" - ) - return ocr_required - - -def rasterize_preview(input_file, page_context): - output_file = page_context.get_path('rasterize_preview.jpg') - canvas_dpi = get_canvas_square_dpi(page_context.pageinfo, page_context.options) - page_dpi = get_page_square_dpi(page_context.pageinfo, page_context.options) - ghostscript.rasterize_pdf( - input_file, - output_file, - xres=canvas_dpi, - yres=canvas_dpi, - raster_device='jpeggray', - log=page_context.log, - page_dpi=(page_dpi, page_dpi), - pageno=page_context.pageinfo.pageno + 1, - ) - return output_file - - -def get_orientation_correction(preview, page_context): - """ - Work out orientation correct for each page. - - We ask Ghostscript to draw a preview page, which will rasterize with the - current /Rotate applied, and then ask Tesseract which way the page is - oriented. If the value of /Rotate is correct (e.g., a user already - manually fixed rotation), then Tesseract will say the page is pointing - up and the correction is zero. Otherwise, the orientation found by - Tesseract represents the clockwise rotation, or the counterclockwise - correction to rotation. - - When we draw the real page for OCR, we rotate it by the CCW correction, - which points it (hopefully) upright. _weave.py takes care of the orienting - the image and text layers. - - """ - - orient_conf = tesseract.get_orientation( - preview, - engine_mode=page_context.options.tesseract_oem, - timeout=page_context.options.tesseract_timeout, - log=page_context.log, - ) - - direction = {0: '⇧', 90: '⇨', 180: '⇩', 270: '⇦'} - - existing_rotation = page_context.pageinfo.rotation - - correction = orient_conf.angle % 360 - - apply_correction = False - action = '' - if orient_conf.confidence >= page_context.options.rotate_pages_threshold: - if correction != 0: - apply_correction = True - action = ' - will rotate' - else: - action = ' - rotation appears correct' - else: - if correction != 0: - action = ' - confidence too low to rotate' - else: - action = ' - no change' - - facing = '' - if existing_rotation != 0: - facing = 'with existing rotation {}, '.format( - direction.get(existing_rotation, '?') - ) - facing += 'page is facing {}'.format(direction.get(orient_conf.angle, '?')) - - page_context.log.debug( - '{pagenum:4d}: {facing}, confidence {conf:.2f}{action}'.format( - pagenum=page_context.pageinfo.pageno, - facing=facing, - conf=orient_conf.confidence, - action=action, - ) - ) - - if apply_correction: - return correction - return 0 - - -def rasterize(input_file, page_context, correction=0): - colorspaces = ['pngmono', 'pnggray', 'png256', 'png16m'] - device_idx = 0 - output_file = page_context.get_path('rasterize.png') - pageinfo = page_context.pageinfo - - def at_least(cs): - return max(device_idx, colorspaces.index(cs)) - - for image in pageinfo.images: - if image.type_ != 'image': - continue # ignore masks - if image.bpc > 1: - if image.color == Colorspace.index: - device_idx = at_least('png256') - elif image.color == Colorspace.gray: - device_idx = at_least('pnggray') - else: - device_idx = at_least('png16m') - - device = colorspaces[device_idx] - - page_context.log.debug(f"Rasterize with {device}") - - # Produce the page image with square resolution or else deskew and OCR - # will not work properly. - canvas_dpi = get_canvas_square_dpi(pageinfo, page_context.options) - page_dpi = get_page_square_dpi(pageinfo, page_context.options) - - ghostscript.rasterize_pdf( - input_file, - output_file, - xres=canvas_dpi, - yres=canvas_dpi, - raster_device=device, - log=page_context.log, - page_dpi=(page_dpi, page_dpi), - pageno=pageinfo.pageno + 1, - rotation=correction, - filter_vector=page_context.options.remove_vectors, - ) - return output_file - - -def preprocess_remove_background(input_file, page_context): - if any(image.bpc > 1 for image in page_context.pageinfo.images): - output_file = page_context.get_path('pp_rm_bg.png') - leptonica.remove_background(input_file, output_file) - return output_file - else: - page_context.log.info("background removal skipped on mono page") - return input_file - - -def preprocess_deskew(input_file, page_context): - output_file = page_context.get_path('pp_deskew.png') - dpi = get_page_square_dpi(page_context.pageinfo, page_context.options) - leptonica.deskew(input_file, output_file, dpi) - return output_file - - -def preprocess_clean(input_file, page_context): - from .exec import unpaper - output_file = page_context.get_path('pp_clean.png') - dpi = get_page_square_dpi(page_context.pageinfo, page_context.options) - unpaper.clean(input_file, output_file, dpi, page_context.log, page_context.options.unpaper_args) - return output_file - - -def create_ocr_image(image, page_context): - """Create the image we send for OCR. May not be the same as the display - image depending on preprocessing. This image will never be shown to the - user.""" - - output_file = page_context.get_path('ocr.png') - options = page_context.options - with Image.open(image) as im: - from PIL import ImageColor - from PIL import ImageDraw - - white = ImageColor.getcolor('#ffffff', im.mode) - # pink = ImageColor.getcolor('#ff0080', im.mode) - draw = ImageDraw.ImageDraw(im) - - xres, yres = im.info['dpi'] - page_context.log.info('resolution %r %r' % (xres, yres)) - - if not options.force_ocr: - # Do not mask text areas when forcing OCR, because we need to OCR - # all text areas - mask = None # Exclude both visible and invisible text from OCR - if options.redo_ocr: - mask = True # Mask visible text, but not invisible text - - for textarea in page_context.pageinfo.get_textareas(visible=mask, corrupt=None): - # Calculate resolution based on the image size and page dimensions - # without regard whatever resolution is in pageinfo (may differ or - # be None) - bbox = [float(v) for v in textarea] - xscale, yscale = float(xres) / 72.0, float(yres) / 72.0 - pixcoords = [ - bbox[0] * xscale, - im.height - bbox[3] * yscale, - bbox[2] * xscale, - im.height - bbox[1] * yscale, - ] - pixcoords = [int(round(c)) for c in pixcoords] - print('blanking %r', pixcoords) - draw.rectangle(pixcoords, fill=white) - # draw.rectangle(pixcoords, outline=pink) - - if options.mask_barcodes or options.threshold: - pix = leptonica.Pix.frompil(im) - if options.threshold: - pix = pix.masked_threshold_on_background_norm() - if options.mask_barcodes: - barcodes = pix.locate_barcodes() - for barcode in barcodes: - decoded, rect = barcode - print('masking barcode %s %r', decoded, rect) - draw.rectangle(rect, fill=white) - im = pix.topil() - - del draw - # Pillow requires integer DPI - dpi = round(xres), round(yres) - im.save(output_file, dpi=dpi) - return output_file - - -def ocr_tesseract_hocr(input_file, page_context): - hocr_out = page_context.get_path('ocr_hocr.hocr') - hocr_text_out = page_context.get_path('ocr_hocr.txt') - options = page_context.options - tesseract.generate_hocr( - input_file=input_file, - output_files=[hocr_out, hocr_text_out], - language=options.language, - engine_mode=options.tesseract_oem, - tessconfig=options.tesseract_config, - timeout=options.tesseract_timeout, - pagesegmode=options.tesseract_pagesegmode, - user_words=options.user_words, - user_patterns=options.user_patterns, - log=page_context.log, - ) - return (hocr_out, hocr_text_out) - - -def should_visible_page_image_use_jpg(pageinfo): - # If all images were JPEGs originally, produce a JPEG as output - return pageinfo.images and all(im.enc == 'jpeg' for im in pageinfo.images) - - -def create_visible_page_jpg(image, page_context): - output_file = page_context.get_path('visible.jpg') - with Image.open(image) as im: - # At this point the image should be a .png, but deskew, unpaper - # might have removed the DPI information. In this case, fall back to - # square DPI used to rasterize. When the preview image was - # rasterized, it was also converted to square resolution, which is - # what we want to give tesseract, so keep it square. - fallback_dpi = get_page_square_dpi(page_context.pageinfo, page_context.options) - dpi = im.info.get('dpi', (fallback_dpi, fallback_dpi)) - - # Pillow requires integer DPI - dpi = round(dpi[0]), round(dpi[1]) - im.save(output_file, format='JPEG', dpi=dpi) - return output_file - - -def create_pdf_page_from_image(image, page_context): - # We rasterize a square DPI version of each page because most image - # processing tools don't support rectangular DPI. Use the square DPI as it - # accurately describes the image. It would be possible to resample the image - # at this stage back to non-square DPI to more closely resemble the input, - # except that the hocr renderer does not understand non-square DPI. The - # sandwich renderer would be fine. - output_file = page_context.get_path('visible.pdf') - dpi = get_page_square_dpi(page_context.pageinfo, page_context.options) - layout_fun = img2pdf.get_fixed_dpi_layout_fun((dpi, dpi)) - - # This create a single page PDF - with open(image, 'rb') as imfile, open(output_file, 'wb') as pdf: - page_context.log.debug('convert') - img2pdf.convert( - imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf - ) - page_context.log.debug('convert done') - return output_file - - -def render_hocr_page(hocr, page_context): - output_file = page_context.get_path('ocr_hocr.pdf') - dpi = get_page_square_dpi(page_context.pageinfo, page_context.options) - hocrtransform = HocrTransform(hocr, dpi) - hocrtransform.to_pdf( - output_file, - imageFileName=None, - showBoundingboxes=False, - invisibleText=True, - interwordSpaces=True, - ) - return output_file - - -def ocr_tesseract_textonly_pdf(input_image, page_context): - output_pdf = page_context.get_path('ocr_tess.pdf') - output_text = page_context.get_path('ocr_tess.txt') - options = page_context.options - tesseract.generate_pdf( - input_image=input_image, - skip_pdf=None, - output_pdf=output_pdf, - output_text=output_text, - language=options.language, - engine_mode=options.tesseract_oem, - text_only=True, - tessconfig=options.tesseract_config, - timeout=options.tesseract_timeout, - pagesegmode=options.tesseract_pagesegmode, - user_words=options.user_words, - user_patterns=options.user_patterns, - log=page_context.log, - ) - return (output_pdf, output_text) - - -def get_docinfo(base_pdf, options): - def from_document_info(key): - try: - s = base_pdf.docinfo[key] - return str(s) - except (KeyError, TypeError): - return '' - - pdfmark = { - k: from_document_info(k) - for k in ('/Title', '/Author', '/Keywords', '/Subject', '/CreationDate') - } - if options.title: - pdfmark['/Title'] = options.title - if options.author: - pdfmark['/Author'] = options.author - if options.keywords: - pdfmark['/Keywords'] = options.keywords - if options.subject: - pdfmark['/Subject'] = options.subject - - if options.pdf_renderer == 'sandwich': - renderer_tag = 'OCR-PDF' - else: - renderer_tag = 'OCR' - - pdfmark['/Creator'] = ( - f'{PROGRAM_NAME} {VERSION} / ' f'Tesseract {renderer_tag} {tesseract.version()}' - ) - pdfmark['/Producer'] = f'pikepdf {pikepdf.__version__}' - if 'OCRMYPDF_CREATOR' in os.environ: - pdfmark['/Creator'] = os.environ['OCRMYPDF_CREATOR'] - if 'OCRMYPDF_PRODUCER' in os.environ: - pdfmark['/Producer'] = os.environ['OCRMYPDF_PRODUCER'] - - pdfmark['/ModDate'] = encode_pdf_date(datetime.now(timezone.utc)) - return pdfmark - - -def generate_postscript_stub(context): - output_file = context.get_path('pdfa.ps') - generate_pdfa_ps(output_file) - return output_file - - -def convert_to_pdfa(input_pdf, input_ps_stub, context): - options = context.options - input_pdfinfo = context.pdfinfo - output_file = context.get_path('pdfa.pdf') - - # If the DocumentInfo record contains NUL characters, Ghostscript will - # produce XMP metadata which contains invalid XML entities (�). - # NULs in DocumentInfo seem to be common since older Acrobats included them. - # pikepdf can deal with this, but we make the world a better place by - # stamping them out as soon as possible. - pdf_file = pikepdf.open(input_pdf) - if pdf_file.docinfo: - modified = False - for k, v in pdf_file.docinfo.items(): - if b'\x00' in bytes(v): - pdf_file.docinfo[k] = bytes(v).replace(b'\x00', b'') - modified = True - if modified: - pdf_file.save(input_pdf) - del pdf_file - - ghostscript.generate_pdfa( - pdf_version=input_pdfinfo.min_version, - pdf_pages=[input_pdf, input_ps_stub], - output_file=output_file, - compression=options.pdfa_image_compression, - log=context.log, - threads=options.jobs or 1, - pdfa_part=options.output_type[-1], # is pdfa-1, pdfa-2, or pdfa-3 - ) - - return output_file - - -def metadata_fixup(working_file, context): - output_file = context.get_path('metafix.pdf') - options = context.options - original = pikepdf.open(context.origin) - docinfo = get_docinfo(original, options) - pdf = pikepdf.open(working_file) - with pdf.open_metadata() as meta: - meta.load_from_docinfo(docinfo, delete_missing=False) - # If xmp:CreateDate is missing, set it to the modify date to - # match Ghostscript, for consistency - if 'xmp:CreateDate' not in meta: - meta['xmp:CreateDate'] = meta.get('xmp:ModifyDate', '') - - meta_original = original.open_metadata() - not_copied = set(meta_original.keys()) - set(meta.keys()) - if not_copied: - context.log.warning( - "Some input metadata could not be copied because it is not " - "permitted in PDF/A. You may wish to examine the output " - "PDF's XMP metadata." - ) - context.log.debug( - "The following metadata fields were not copied: %r", not_copied - ) - - pdf.save( - output_file, - compress_streams=True, - object_stream_mode=pikepdf.ObjectStreamMode.generate, - ) - return output_file - - -def optimize_pdf(input_file, context): - output_file = context.get_path('optimize.pdf') - optimize(input_file, output_file, context) - return output_file - - -def merge_sidecars(txt_files, context): - output_file = context.get_path('sidecar.txt') - with open(output_file, 'w', encoding="utf-8") as stream: - for page_num, txt_file in enumerate(txt_files): - if page_num != 0: - stream.write('\f') # Form feed between pages - if txt_file: - with open(txt_file, 'r', encoding="utf-8") as in_: - txt = in_.read() - # Tesseract v4 alpha started adding form feeds in - # commit aa6eb6b - # No obvious way to detect what binaries will do this, so - # for consistency just ignore its form feeds and insert our - # own - if txt.endswith('\f'): - stream.write(txt[:-1]) - else: - stream.write(txt) - else: - stream.write(f'[OCR skipped on page {(page_num + 1)}]') - return output_file - - -def copy_final(input_file, output_file, context): - context.log.debug('%s -> %s', input_file, output_file) - with open(input_file, 'rb') as input_stream: - if output_file == '-': - copyfileobj(input_stream, sys.stdout.buffer) - sys.stdout.flush() - else: - # At this point we overwrite the output_file specified by the user - # use copyfileobj because then we use open() to create the file and - # get the appropriate umask, ownership, etc. - with open(output_file, 'wb') as output_stream: - copyfileobj(input_stream, output_stream) diff --git a/src/ocrmypdf/_ruffus.py b/src/ocrmypdf/_ruffus.py deleted file mode 100644 index 4cc69043..00000000 --- a/src/ocrmypdf/_ruffus.py +++ /dev/null @@ -1,508 +0,0 @@ -# © 2016 James R. Barlow: github.com/jbarlow83 -# -# This file is part of OCRmyPDF. -# -# OCRmyPDF is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# OCRmyPDF is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with OCRmyPDF. If not, see . - -import os -import re -import sys -import atexit -from tempfile import mkdtemp -from ruffus import ( - Pipeline, - formatter, - regex, - suffix, - cmdline, - proxy_logger, - ruffus_exceptions -) -from .exec import qpdf -from ._jobcontext import JobContext, JobContextManager, cleanup_working_files -from ._weave import weave_layers -from ._pipeline import ( - triage, - repair_and_parse_pdf, - marker_pages, - ocr_or_skip, - rasterize_preview, - orient_page, - rasterize_with_ghostscript, - preprocess_remove_background, - preprocess_deskew, - preprocess_clean, - select_ocr_image, - ocr_tesseract_hocr, - select_visible_page_image, - select_image_layer, - render_hocr_page, - ocr_tesseract_textonly_pdf, - generate_postscript_stub, - convert_to_pdfa, - metadata_fixup, - merge_sidecars, - optimize_pdf, - copy_final -) -from . import exceptions as ocrmypdf_exceptions -from .exceptions import ( - ExitCode, - ExitCodeException, -) -from .helpers import available_cpu_count -from .pdfa import file_claims_pdfa -from ._validation import ( - check_closed_streams, - preamble, - check_options, - check_dependency_versions, - check_environ, - check_input_file, - check_requested_output_file, - report_output_file_size, - log_page_orientations, - logging_factory, -) - - -def cleanup_ruffus_error_message(msg): - msg = re.sub(r'\s+', r' ', msg) - msg = re.sub(r"\((.+?)\)", r'\1', msg) - msg = msg.strip() - return msg - - -def do_ruffus_exception(ruffus_five_tuple, options, log): - """Replace the elaborate ruffus stack trace with a user friendly - description of the error message that occurred.""" - exit_code = None - - _task_name, _job_name, exc_name, exc_value, exc_stack = ruffus_five_tuple - - if isinstance(exc_name, type): - # ruffus is full of mystery... sometimes (probably when the process - # group leader is killed) exc_name is the class object of the exception, - # rather than a str. So reach into the object and get its name. - exc_name = exc_name.__name__ - - if exc_name.startswith('ocrmypdf.exceptions.'): - base_exc_name = exc_name.replace('ocrmypdf.exceptions.', '') - exc_class = getattr(ocrmypdf_exceptions, base_exc_name) - exit_code = getattr(exc_class, 'exit_code', ExitCode.other_error) - try: - if isinstance(exc_value, exc_class): - exc_msg = str(exc_value) - elif isinstance(exc_value, str): - exc_msg = exc_value - else: - exc_msg = str(exc_class()) - except Exception: - exc_msg = "Unknown" - - if exc_name in ('builtins.SystemExit', 'SystemExit'): - match = re.search(r"\.(.+?)\)", exc_value) - exit_code_name = match.groups()[0] - exit_code = getattr(ExitCode, exit_code_name, 'other_error') - elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError': - log.error(cleanup_ruffus_error_message(exc_value)) - exit_code = ExitCode.input_file - elif exc_name in ('builtins.KeyboardInterrupt', 'KeyboardInterrupt'): - # We have to print in this case because the log daemon might be toast - print("Interrupted by user", file=sys.stderr) - exit_code = ExitCode.ctrl_c - elif exc_name == 'subprocess.CalledProcessError': - # It's up to the subprocess handler to report something useful - msg = "Error occurred while running this command:" - log.error(msg + '\n' + exc_value) - exit_code = ExitCode.child_process_error - elif exc_name.startswith('ocrmypdf.exceptions.'): - if exc_msg: - log.error(exc_msg) - elif exc_name == 'PIL.Image.DecompressionBombError': - msg = cleanup_ruffus_error_message(exc_value) - msg += ( - "\nUse the --max-image-mpixels argument to set increase the " - "maximum number of megapixels to accept." - ) - log.error(msg) - exit_code = ExitCode.input_file - - if exit_code is not None: - return exit_code - - if not options.verbose: - log.error(exc_stack) - return ExitCode.other_error - - -def traverse_ruffus_exception(exceptions, options, log): - """Traverse a RethrownJobError and output the exceptions - - Ruffus presents exceptions as 5 element tuples. The RethrownJobException - has a list of exceptions like - e.job_exceptions = [(5-tuple), (5-tuple), ...] - - ruffus < 2.7.0 had a bug with exception marshalling that would give - different output whether the main or child process raised the exception. - We no longer support this. - - Attempting to log the exception itself will re-marshall it to the logger - which is normally running in another process. It's better to avoid re- - marshalling. - - The exit code will be based on this, even if multiple exceptions occurred - at the same time.""" - - exit_codes = [] - for exc in exceptions: - exit_code = do_ruffus_exception(exc, options, log) - exit_codes.append(exit_code) - - return exit_codes[0] # Multiple codes are rare so take the first one - - -def build_pipeline(options, work_folder, log, context): - main_pipeline = Pipeline.pipelines['main'] - - # Triage - task_triage = main_pipeline.transform( - task_func=triage, - input=os.path.join(work_folder, 'origin'), - filter=formatter('(?i)'), - output=os.path.join(work_folder, 'origin.pdf'), - extras=[log, context], - ) - - task_repair_and_parse_pdf = main_pipeline.transform( - task_func=repair_and_parse_pdf, - input=task_triage, - filter=suffix('.pdf'), - output='.repaired.pdf', - output_dir=work_folder, - extras=[log, context], - ) - - # Split (kwargs for split seems to be broken, so pass plain args) - task_marker_pages = main_pipeline.split( - marker_pages, - task_repair_and_parse_pdf, - os.path.join(work_folder, '*.marker.pdf'), - extras=[log, context], - ) - - task_ocr_or_skip = main_pipeline.split( - ocr_or_skip, - task_marker_pages, - [ - os.path.join(work_folder, '*.ocr.page.pdf'), - os.path.join(work_folder, '*.skip.page.pdf'), - ], - extras=[log, context], - ) - - # Rasterize preview - task_rasterize_preview = main_pipeline.transform( - task_func=rasterize_preview, - input=task_ocr_or_skip, - filter=suffix('.page.pdf'), - output='.preview.jpg', - output_dir=work_folder, - extras=[log, context], - ) - task_rasterize_preview.active_if(options.rotate_pages) - - # Orient - task_orient_page = main_pipeline.collate( - task_func=orient_page, - input=[task_ocr_or_skip, task_rasterize_preview], - filter=regex(r".*/(\d{6})(\.ocr|\.skip)(?:\.page\.pdf|\.preview\.jpg)"), - output=os.path.join(work_folder, r'\1\2.oriented.pdf'), - extras=[log, context], - ) - - # Rasterize actual - task_rasterize_with_ghostscript = main_pipeline.transform( - task_func=rasterize_with_ghostscript, - input=task_orient_page, - filter=suffix('.ocr.oriented.pdf'), - output='.page.png', - output_dir=work_folder, - extras=[log, context], - ) - - # Preprocessing subpipeline - task_preprocess_remove_background = main_pipeline.transform( - task_func=preprocess_remove_background, - input=task_rasterize_with_ghostscript, - filter=suffix(".page.png"), - output=".pp-background.png", - extras=[log, context], - ) - - task_preprocess_deskew = main_pipeline.transform( - task_func=preprocess_deskew, - input=task_preprocess_remove_background, - filter=suffix(".pp-background.png"), - output=".pp-deskew.png", - extras=[log, context], - ) - - task_preprocess_clean = main_pipeline.transform( - task_func=preprocess_clean, - input=task_preprocess_deskew, - filter=suffix(".pp-deskew.png"), - output=".pp-clean.png", - extras=[log, context], - ) - - task_select_ocr_image = main_pipeline.collate( - task_func=select_ocr_image, - input=[task_preprocess_clean], - filter=regex(r".*/(\d{6})(?:\.page|\.pp-.*)\.png"), - output=os.path.join(work_folder, r"\1.ocr.png"), - extras=[log, context], - ) - - # HOCR OCR - task_ocr_tesseract_hocr = main_pipeline.transform( - task_func=ocr_tesseract_hocr, - input=task_select_ocr_image, - filter=suffix(".ocr.png"), - output=[".hocr", ".txt"], - extras=[log, context], - ) - task_ocr_tesseract_hocr.graphviz(fillcolor='"#00cc66"') - task_ocr_tesseract_hocr.active_if(options.pdf_renderer == 'hocr') - - task_select_visible_page_image = main_pipeline.collate( - task_func=select_visible_page_image, - input=[ - task_rasterize_with_ghostscript, - task_preprocess_remove_background, - task_preprocess_deskew, - task_preprocess_clean, - ], - filter=regex(r".*/(\d{6})(?:\.page|\.pp-.*)\.png"), - output=os.path.join(work_folder, r'\1.image'), - extras=[log, context], - ) - task_select_visible_page_image.graphviz(shape='diamond') - - task_select_image_layer = main_pipeline.collate( - task_func=select_image_layer, - input=[task_select_visible_page_image, task_orient_page], - filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.oriented\.pdf)"), - output=os.path.join(work_folder, r'\1.image-layer.pdf'), - extras=[log, context], - ) - task_select_image_layer.graphviz(fillcolor='"#00cc66"', shape='diamond') - - task_render_hocr_page = main_pipeline.transform( - task_func=render_hocr_page, - input=task_ocr_tesseract_hocr, - filter=regex(r".*/(\d{6})(?:\.hocr)"), - output=os.path.join(work_folder, r'\1.text.pdf'), - extras=[log, context], - ) - task_render_hocr_page.graphviz(fillcolor='"#00cc66"') - task_render_hocr_page.active_if(options.pdf_renderer == 'hocr') - - # Tesseract OCR + text only PDF - task_ocr_tesseract_textonly_pdf = main_pipeline.collate( - task_func=ocr_tesseract_textonly_pdf, - input=[task_select_ocr_image], - filter=regex(r".*/(\d{6})(?:\.ocr.png)"), - output=[ - os.path.join(work_folder, r'\1.text.pdf'), - os.path.join(work_folder, r'\1.text.txt'), - ], - extras=[log, context], - ) - task_ocr_tesseract_textonly_pdf.graphviz(fillcolor='"#ff69b4"') - task_ocr_tesseract_textonly_pdf.active_if(options.pdf_renderer == 'sandwich') - - task_weave_layers = main_pipeline.collate( - task_func=weave_layers, - input=[ - task_repair_and_parse_pdf, - task_render_hocr_page, - task_ocr_tesseract_textonly_pdf, - task_select_image_layer, - ], - filter=regex( - r".*/((?:\d{6}(?:\.text\.pdf|\.image-layer\.pdf))|(?:origin\.repaired\.pdf))" - ), - output=os.path.join(work_folder, r'layers.rendered.pdf'), - extras=[log, context], - ) - task_weave_layers.graphviz(fillcolor='"#00cc66"') - - # PDF/A pdfmark - task_generate_postscript_stub = main_pipeline.transform( - task_func=generate_postscript_stub, - input=task_repair_and_parse_pdf, - filter=formatter(r'\.repaired\.pdf'), - output=os.path.join(work_folder, 'pdfa.ps'), - extras=[log, context], - ) - task_generate_postscript_stub.active_if(options.output_type.startswith('pdfa')) - - # PDF/A conversion - task_convert_to_pdfa = main_pipeline.merge( - task_func=convert_to_pdfa, - input=[task_generate_postscript_stub, task_weave_layers], - output=os.path.join(work_folder, 'pdfa.pdf'), - extras=[log, context], - ) - task_convert_to_pdfa.active_if(options.output_type.startswith('pdfa')) - - task_metadata_fixup = main_pipeline.merge( - task_func=metadata_fixup, - input=[task_repair_and_parse_pdf, task_weave_layers, task_convert_to_pdfa], - output=os.path.join(work_folder, 'metafix.pdf'), - extras=[log, context], - ) - - task_merge_sidecars = main_pipeline.merge( - task_func=merge_sidecars, - input=[task_ocr_tesseract_hocr, task_ocr_tesseract_textonly_pdf], - output=options.sidecar, - extras=[log, context], - ) - task_merge_sidecars.active_if(options.sidecar) - - # Optimize - task_optimize_pdf = main_pipeline.transform( - task_func=optimize_pdf, - input=task_metadata_fixup, - filter=suffix('.pdf'), - output='.optimized.pdf', - output_dir=work_folder, - extras=[log, context], - ) - - # Finalize - main_pipeline.merge( - task_func=copy_final, - input=[task_optimize_pdf], - output=options.output_file, - extras=[log, context], - ) - - -def run_pipeline(options): - options.verbose_abbreviated_path = 1 - if os.environ.get('_OCRMYPDF_THREADS'): - options.use_threads = True - - if not check_closed_streams(options): - return ExitCode.bad_args - - logger_args = {'verbose': options.verbose, 'quiet': options.quiet} - - _log, _log_mutex = proxy_logger.make_shared_logger_and_proxy( - logging_factory, __name__, logger_args - ) - preamble(_log) - check_code = check_options(options, _log) - if check_code != ExitCode.ok: - return check_code - check_dependency_versions(options, _log) - - # Any changes to options will not take effect for options that are already - # bound to function parameters in the pipeline. (For example - # options.input_file, options.pdf_renderer are already bound.) - if not options.jobs: - options.jobs = available_cpu_count() - - # Performance is improved by setting Tesseract to single threaded. In tests - # this gives better throughput than letting a smaller number of Tesseract - # jobs run multithreaded. Same story for pngquant. Tess <4 ignores this - # variable, but harmless to set if ignored. - os.environ.setdefault('OMP_THREAD_LIMIT', '1') - - check_environ(options, _log) - if os.environ.get('PYTEST_CURRENT_TEST'): - os.environ['_OCRMYPDF_TEST_INFILE'] = options.input_file - - try: - work_folder = mkdtemp(prefix="com.github.ocrmypdf.") - options.history_file = os.path.join(work_folder, 'ruffus_history.sqlite') - start_input_file = os.path.join(work_folder, 'origin') - - check_input_file(options, _log, start_input_file) - check_requested_output_file(options, _log) - - manager = JobContextManager() - manager.register('JobContext', JobContext) # pylint: disable=no-member - manager.start() - - context = manager.JobContext() # pylint: disable=no-member - context.set_options(options) - context.set_work_folder(work_folder) - - build_pipeline(options, work_folder, _log, context) - atexit.register(cleanup_working_files, work_folder, options) - if hasattr(os, 'nice'): - os.nice(5) - cmdline.run(options) - except ruffus_exceptions.RethrownJobError as e: - if options.verbose: - _log.debug(str(e)) # stringify exception so logger doesn't have to - exceptions = e.job_exceptions - exitcode = traverse_ruffus_exception(exceptions, options, _log) - if exitcode is None: - _log.error("Unexpected ruffus exception: " + str(e)) - _log.error(repr(e)) - return ExitCode.other_error - return exitcode - except ExitCodeException as e: - return e.exit_code - except Exception as e: - _log.error(str(e)) - return ExitCode.other_error - - if options.flowchart: - _log.info(f"Flowchart saved to {options.flowchart}") - return ExitCode.ok - elif options.output_file == '-': - _log.info("Output sent to stdout") - elif os.path.samefile(options.output_file, os.devnull): - pass # Say nothing when sending to dev null - else: - if options.output_type.startswith('pdfa'): - pdfa_info = file_claims_pdfa(options.output_file) - if pdfa_info['pass']: - msg = f"Output file is a {pdfa_info['conformance']} (as expected)" - _log.info(msg) - else: - msg = f"Output file is okay but is not PDF/A (seems to be {pdfa_info['conformance']})" - _log.warning(msg) - return ExitCode.pdfa_conversion_failed - if not qpdf.check(options.output_file, _log): - _log.warning('Output file: The generated PDF is INVALID') - return ExitCode.invalid_output_pdf - - report_output_file_size(options, _log, start_input_file, options.output_file) - - pdfinfo = context.get_pdfinfo() - if options.verbose: - from pprint import pformat - - _log.debug(pformat(pdfinfo)) - - log_page_orientations(pdfinfo, _log) - - return ExitCode.ok diff --git a/src/ocrmypdf/_sync.py b/src/ocrmypdf/_sync.py index ec339bf5..4e246544 100644 --- a/src/ocrmypdf/_sync.py +++ b/src/ocrmypdf/_sync.py @@ -16,13 +16,11 @@ # along with OCRmyPDF. If not, see . import os -# import re -# import sys import atexit from tempfile import mkdtemp -from ._jobcontext import cleanup_working_files +from ._jobcontext import PDFContext, get_logger, cleanup_working_files from ._weave import weave_layers -from ._pipeline_simple import ( +from ._pipeline import ( get_pdfinfo, validate_pdfinfo_options, is_ocr_required, @@ -48,6 +46,7 @@ from ._pipeline_simple import ( ) from .exceptions import ( ExitCode, + ExitCodeException, ) from .helpers import available_cpu_count from ._validation import ( @@ -61,52 +60,6 @@ from ._validation import ( ) -class Logger: - def __init__(self, prefix): - self.prefix = prefix - - def debug(self, *argv): - print(self.prefix, *argv) - - def info(self, *argv): - print(self.prefix, *argv) - - def warn(self, *argv): - print(self.prefix, *argv) - - def error(self, *argv): - print(self.prefix, *argv) - - -class PageContext: - def __init__(self, pdf_context, pageno): - self.pdf_context = pdf_context - self.options = pdf_context.options - self.pageno = pageno - self.pageinfo = pdf_context.pdfinfo[pageno] - self.log = Logger('%s Page %d: ' % (os.path.basename(pdf_context.origin), pageno + 1)) - - def get_path(self, name): - return os.path.join(self.pdf_context.work_folder, "page_%d_%s" % (self.pageno, name)) - - -class PDFContext: - def __init__(self, options, work_folder, origin, pdfinfo): - self.options = options - self.work_folder = work_folder - self.origin = origin - self.pdfinfo = pdfinfo - self.log = Logger('%s: ' % os.path.basename(origin)) - - def get_path(self, name): - return os.path.join(self.work_folder, name) - - def get_page_contexts(self): - npages = len(self.pdfinfo) - for n in range(npages): - yield PageContext(self, n) - - def _exec_pipeline(options, work_folder, origin): # Gather info of pdf pdfinfo = get_pdfinfo(origin) @@ -181,7 +134,7 @@ def run_pipeline(options): if not check_closed_streams(options): return ExitCode.bad_args - log = Logger('Pipeline') + log = get_logger(options, 'Pipeline') preamble(log) check_code = check_options(options, log) if check_code != ExitCode.ok: @@ -213,7 +166,13 @@ def run_pipeline(options): if hasattr(os, 'nice'): os.nice(5) - _exec_pipeline(options, work_folder, start_input_file) + try: + _exec_pipeline(options, work_folder, start_input_file) + except ExitCodeException as e: + return e.exit_code + except Exception as e: + log.error(str(e)) + return ExitCode.other_error return ExitCode.ok diff --git a/src/ocrmypdf/optimize.py b/src/ocrmypdf/optimize.py index 000f24d8..9cf1e69b 100644 --- a/src/ocrmypdf/optimize.py +++ b/src/ocrmypdf/optimize.py @@ -16,7 +16,6 @@ # along with OCRmyPDF. If not, see . import concurrent.futures -import logging import sys from collections import defaultdict from os import fspath @@ -28,7 +27,7 @@ import pikepdf from pikepdf import Name, Dictionary from . import leptonica -from ._jobcontext import JobContext +from ._jobcontext import PDFContext from .exec import jbig2enc, pngquant from .helpers import re_symlink @@ -82,9 +81,7 @@ def extract_image_jbig2(*, pike, root, log, image, xref, options): pim, filtdp = result if ( - pim.bits_per_component == 1 - and filtdp != Name.JBIG2Decode - and jbig2enc.available() + pim.bits_per_component == 1 and filtdp != Name.JBIG2Decode and jbig2enc.available() ): try: imgname = Path(root / f'{xref:08d}') @@ -129,9 +126,7 @@ def extract_image_generic(*, pike, root, log, image, xref, options): return None return xref, ext elif ( - pim.indexed - and pim.colorspace in pim.SIMPLE_COLORSPACES - and options.optimize >= 3 + pim.indexed and pim.colorspace in pim.SIMPLE_COLORSPACES and options.optimize >= 3 ): # Try to improve on indexed images - these are far from low hanging # fruit in most cases @@ -492,10 +487,6 @@ def main(infile, outfile, level, jobs=1): self.jbig2_page_group_size = 0 self.jbig2_lossy = jb2lossy - logging.basicConfig(level=logging.DEBUG) - log = logging.getLogger() - - ctx = JobContext() options = OptimizeOptions( jobs=jobs, optimize=int(level), @@ -503,11 +494,11 @@ def main(infile, outfile, level, jobs=1): png_quality=0, jb2lossy=False, ) - ctx.set_options(options) with TemporaryDirectory() as td: + context = PDFContext(options, td, infile, None) tmpout = Path(td) / 'out.pdf' - optimize(infile, tmpout, log, ctx) + optimize(infile, tmpout, context) copy(fspath(tmpout), fspath(outfile)) diff --git a/tests/test_multiprocessing.py b/tests/_test_multiprocessing.py similarity index 100% rename from tests/test_multiprocessing.py rename to tests/_test_multiprocessing.py diff --git a/tests/test_main.py b/tests/test_main.py index 248a9003..49d93025 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -28,7 +28,7 @@ import pytest from PIL import Image from ocrmypdf.exceptions import ExitCode, MissingDependencyError -from ocrmypdf.exec import ghostscript, qpdf, tesseract, unpaper +from ocrmypdf.exec import ghostscript, qpdf, tesseract from ocrmypdf.leptonica import Pix from ocrmypdf.pdfa import file_claims_pdfa from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo diff --git a/tests/test_metadata.py b/tests/test_metadata.py index be1d952c..a4d60b55 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -28,7 +28,7 @@ from unittest.mock import MagicMock, patch import pytest import pikepdf -from ocrmypdf._jobcontext import JobContext +from ocrmypdf._jobcontext import PDFContext from ocrmypdf.exceptions import ExitCode from ocrmypdf.pdfa import SRGB_ICC_PROFILE, file_claims_pdfa, generate_pdfa_ps from pikepdf.models.metadata import decode_pdf_date @@ -330,23 +330,15 @@ def test_prevent_gs_invalid_xml(resources, outdir): from ocrmypdf.pdfinfo import PdfInfo generate_pdfa_ps(outdir / 'pdfa.ps') - input_files = [str(outdir / 'layers.rendered.pdf'), str(outdir / 'pdfa.ps')] copyfile(resources / 'enron1.pdf', outdir / 'layers.rendered.pdf') - log = logging.getLogger() - context = JobContext() options = parser.parse_args( args=['-j', '1', '--output-type', 'pdfa-2', 'a.pdf', 'b.pdf'] ) - context.options = options - context.pdfinfo = PdfInfo(resources / 'enron1.pdf') + pdfinfo = PdfInfo(resources / 'enron1.pdf') + context = PDFContext(options, outdir, resources / 'enron1.pdf', pdfinfo) - convert_to_pdfa( - input_files_groups=input_files, - output_file=outdir / 'pdfa.pdf', - log=log, - context=context, - ) + convert_to_pdfa(str(outdir / 'layers.rendered.pdf'), str(outdir / 'pdfa.ps'), context) with open(outdir / 'pdfa.pdf', 'rb') as f: with mmap.mmap(