menekis/OCRmyPDF
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: move to sync (none ETL) implementation - remove ruffus

Browse Source
This commit is contained in:
mawi
2019-04-04 21:02:38 +02:00
parent b214aa5b38
commit 783a128bd1
10 changed files with 277 additions and 1740 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ocrmypdf/__main__.py
+3 -3
View File
@@ -462,9 +462,9 @@ debugging.add_argument(
action='store_true',
help="Keep temporary files (helpful for debugging)",
)
debugging.add_argument(
'--flowchart', type=str, help="Generate the pipeline execution flowchart"
)
# debugging.add_argument(
# '--flowchart', type=str, help="Generate the pipeline execution flowchart"
# )
def run(args=None):
src/ocrmypdf/_jobcontext.py
+84 -46
View File
@@ -17,62 +17,41 @@
import shutil
import sys
import os
from contextlib import suppress
from multiprocessing.managers import SyncManager
from .pdfinfo import PdfInfo
class JobContext:
"""Holds our context for a particular run of the pipeline
class PDFContext:
"""Holds our context for a particular run of the pipeline"""
A multiprocessing manager effectively creates a separate process
that keeps the master job context object. Other threads access
job context via multiprocessing proxy objects.
While this would naturally lend itself @property's it seems to make
a little more sense to use functions to make it explicitly that the
invocation requires marshalling data across a process boundary.
"""
def __init__(self):
self.pdfinfo = None
self.options = None
self.work_folder = None
self.rotations = {}
def generate_pdfinfo(self, infile):
self.pdfinfo = PdfInfo(infile)
def get_pdfinfo(self):
"What we know about the input PDF"
return self.pdfinfo
def set_pdfinfo(self, pdfinfo):
self.pdfinfo = pdfinfo
def get_options(self):
return self.options
def set_options(self, options):
def __init__(self, options, work_folder, origin, pdfinfo):
self.options = options
def get_work_folder(self):
return self.work_folder
def set_work_folder(self, work_folder):
self.work_folder = work_folder
self.origin = origin
self.pdfinfo = pdfinfo
self.log = get_logger(options, '%s: ' % os.path.basename(origin))
def get_rotation(self, pageno):
return self.rotations.get(pageno, 0)
def get_path(self, name):
return os.path.join(self.work_folder, name)
def set_rotation(self, pageno, value):
self.rotations[pageno] = value
def get_page_contexts(self):
npages = len(self.pdfinfo)
for n in range(npages):
yield PageContext(self, n)
class JobContextManager(SyncManager):
pass
class PageContext:
"""Holds our context for a page"""
def __init__(self, pdf_context, pageno):
self.pdf_context = pdf_context
self.options = pdf_context.options
self.pageno = pageno
self.pageinfo = pdf_context.pdfinfo[pageno]
self.log = get_logger(pdf_context.options, '%s Page %d: ' % (os.path.basename(pdf_context.origin), pageno + 1))
def get_path(self, name):
return os.path.join(self.pdf_context.work_folder, "page_%d_%s" % (self.pageno, name))
def cleanup_working_files(work_folder, options):
@@ -81,3 +60,62 @@ def cleanup_working_files(work_folder, options):
else:
with suppress(FileNotFoundError):
shutil.rmtree(work_folder)
def get_logger(options=None, prefix=''):
level = INFO # TODO: add option
if options is not None and options.output_file == '-' or options.sidecar == '-':
return NullLogger()
return Logger(prefix, level)
ERROR = 40
WARN = 30
INFO = 20
DEBUG = 10
class Logger:
def __init__(self, prefix, level=INFO):
self.prefix = prefix
self.level = level
def debug(self, *args, **kwargs):
if self.level <= DEBUG:
print('DEBUG', self.prefix, end='')
print(*args, **kwargs)
def info(self, *args, **kwargs):
if self.level <= INFO:
print('INFO', self.prefix, end='')
print(*args, **kwargs)
def warning(self, *args, **kwargs):
self.warn(*args, **kwargs)
def warn(self, *args, **kwargs):
if self.level <= WARN:
print('WARN', self.prefix, end='')
print(*args, **kwargs)
def error(self, *args, **kwargs):
if self.level <= ERROR:
print('ERROR', self.prefix, end='')
print(*args, **kwargs)
class NullLogger:
def debug(self, *args, **kwargs):
pass
def info(self, *args, **kwargs):
pass
def warning(self, *args, **kwargs):
pass
def warn(self, *args, **kwargs):
pass
def error(self, *args, **kwargs):
pass
src/ocrmypdf/_pipeline.py
+169 -320
View File
@@ -18,10 +18,8 @@
import os
import re
import sys
from contextlib import suppress
from datetime import datetime, timezone
from pathlib import Path
from shutil import copyfile, copyfileobj
from shutil import copyfileobj
import img2pdf
from PIL import Image
@@ -34,14 +32,10 @@ from .exceptions import (
DpiError,
EncryptedPdfError,
InputFileError,
PriorOcrFoundError,
UnsupportedImageFormatError,
)
from .exec import ghostscript, tesseract
from .helpers import (
flatten_groups,
is_iterable_notstr,
page_number,
re_symlink
)
from .hocrtransform import HocrTransform
@@ -51,10 +45,6 @@ from .pdfinfo import Colorspace, PdfInfo
VECTOR_PAGE_DPI = 400
#
# The Pipeline
#
def triage_image_file(input_file, output_file, log, options):
try:
@@ -164,31 +154,28 @@ def triage(input_file, output_file, log, context):
triage_image_file(input_file, output_file, log, options)
def repair_and_parse_pdf(input_file, output_file, log, context):
options = context.get_options()
copyfile(input_file, output_file)
detailed_page_analysis = False
if options.redo_ocr:
detailed_page_analysis = True
def get_pdfinfo(input_file, detailed_page_analysis=False):
try:
pdfinfo = PdfInfo(
output_file, detailed_page_analysis=detailed_page_analysis, log=log
return PdfInfo(
input_file, detailed_page_analysis=detailed_page_analysis
)
except pikepdf.PasswordError:
raise EncryptedPdfError()
except pikepdf.PdfError as e:
log.error(e)
except pikepdf.PdfError:
raise InputFileError()
def validate_pdfinfo_options(context):
log = context.log
pdfinfo = context.pdfinfo
options = context.options
if pdfinfo.needs_rendering:
log.error(
"This PDF contains dynamic XFA forms created by Adobe LiveCycle "
"Designer and can only be read by Adobe Acrobat or Adobe Reader."
)
raise InputFileError()
if pdfinfo.has_userunit and options.output_type.startswith('pdfa'):
log.error(
"This input file uses a PDF feature that is not supported "
@@ -198,16 +185,15 @@ def repair_and_parse_pdf(input_file, output_file, log, context):
"output these files.) Use --output-type=pdf instead."
)
raise InputFileError()
if pdfinfo.has_acroform:
if options.redo_ocr:
log.error(
"This PDF has a user fillable form. --redo-ocr is not "
"currently possible on such files."
)
raise PriorOcrFoundError()
raise InputFileError()
else:
log.warning(
log.warn(
"This PDF has a fillable form. "
"Chances are it is a pure digital "
"document that does not need OCR."
@@ -219,16 +205,6 @@ def repair_and_parse_pdf(input_file, output_file, log, context):
"'flattened' and will no longer be fillable."
)
context.set_pdfinfo(pdfinfo)
log.debug(pdfinfo)
def get_pageinfo(input_file, context):
"Get zero-based page info implied by filename, e.g. 000002.pdf -> 1"
pageno = page_number(input_file) - 1
pageinfo = context.get_pdfinfo()[pageno]
return pageinfo
def get_page_dpi(pageinfo, options):
"Get the DPI when nonsquare DPI is tolerable"
@@ -272,33 +248,31 @@ def get_canvas_square_dpi(pageinfo, options):
)
def is_ocr_required(pageinfo, log, options):
page = pageinfo.pageno + 1
def is_ocr_required(page_context):
pageinfo = page_context.pageinfo
options = page_context.options
log = page_context.log
ocr_required = True
if pageinfo.has_text:
prefix = f"{page:4d}: page already has text! - "
if not options.force_ocr and not (options.skip_text or options.redo_ocr):
log.error(prefix + "aborting (use --force-ocr to force OCR)")
raise PriorOcrFoundError()
log.error("page already has text! - aborting (use --force-ocr to force OCR)")
ocr_required = False
elif options.force_ocr:
log.info(prefix + "rasterizing text and running OCR anyway")
log.info("page already has text! - rasterizing text and running OCR anyway")
ocr_required = True
elif options.redo_ocr:
if pageinfo.has_corrupt_text:
log.warning(
prefix + (
"some text on this page cannot be mapped to characters: "
"consider using --force-ocr instead",
)
log.warn(
"some text on this page cannot be mapped to characters: "
"consider using --force-ocr instead",
)
raise PriorOcrFoundError() # Wrong error but will do for now
else:
log.info(prefix + "redoing OCR")
log.info("redoing OCR")
ocr_required = True
elif options.skip_text:
log.info(prefix + "skipping all processing on this page")
log.info("skipping all processing on this page")
ocr_required = False
elif not pageinfo.images and not options.lossless_reconstruction:
# We found a page with no images and no text. That means it may
@@ -311,14 +285,14 @@ def is_ocr_required(pageinfo, log, options):
if options.force_ocr and options.oversample:
# The user really wants to reprocess this file
log.info(
f"{page:4d}: page has no images - "
"page has no images - "
f"rasterizing at {options.oversample} DPI because "
"--force-ocr --oversample was specified"
)
elif options.force_ocr:
# Warn the user they might not want to do this
log.warning(
f"{page:4d}: page has no images - "
log.warn(
"page has no images - "
"all vector content will be "
f"rasterized at {VECTOR_PAGE_DPI} DPI, losing some resolution and likely "
"increasing file size. Use --oversample to adjust the "
@@ -326,7 +300,7 @@ def is_ocr_required(pageinfo, log, options):
)
else:
log.info(
f"{page:4d}: page has no images - "
"page has no images - "
"skipping all processing on this page to avoid losing detail. "
"Use --force-ocr if you wish to perform OCR on pages that "
"have vector content."
@@ -337,82 +311,31 @@ def is_ocr_required(pageinfo, log, options):
pixel_count = pageinfo.width_pixels * pageinfo.height_pixels
if pixel_count > (options.skip_big * 1_000_000):
ocr_required = False
log.warning(
f"{page:4d}: page too big, skipping OCR "
log.warn(
"page too big, skipping OCR "
f"({(pixel_count / 1_000_000):.1f} MPixels > {options.skip_big:.1f} MPixels --skip-big)"
)
return ocr_required
def marker_pages(input_files, output_files, log, context):
options = context.get_options()
work_folder = context.get_work_folder()
if is_iterable_notstr(input_files):
input_file = input_files[0]
else:
input_file = input_files
for oo in output_files:
with suppress(FileNotFoundError):
os.unlink(oo)
# If no files were repaired the input will be empty
if not input_file:
log.error(f"{options.input_file}: file not found or invalid argument")
raise InputFileError()
pdfinfo = context.get_pdfinfo()
npages = len(pdfinfo)
# Ruffus needs to see a file for any task it generates, so make very
# file a symlink back to the source.
for n in range(npages):
page = Path(work_folder) / f'{(n + 1):06d}.marker.pdf'
page.symlink_to(input_file) # pylint: disable=E1101
def ocr_or_skip(input_files, output_files, log, context):
options = context.get_options()
work_folder = context.get_work_folder()
pdfinfo = context.get_pdfinfo()
for input_file in input_files:
pageno = page_number(input_file) - 1
pageinfo = pdfinfo[pageno]
alt_suffix = (
'.ocr.page.pdf'
if is_ocr_required(pageinfo, log, options)
else '.skip.page.pdf'
)
re_symlink(
input_file,
os.path.join(work_folder, os.path.basename(input_file)[0:6] + alt_suffix),
log,
)
def rasterize_preview(input_file, output_file, log, context):
pageinfo = get_pageinfo(input_file, context)
options = context.get_options()
canvas_dpi = get_canvas_square_dpi(pageinfo, options)
page_dpi = get_page_square_dpi(pageinfo, options)
def rasterize_preview(input_file, page_context):
output_file = page_context.get_path('rasterize_preview.jpg')
canvas_dpi = get_canvas_square_dpi(page_context.pageinfo, page_context.options)
page_dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
ghostscript.rasterize_pdf(
input_file,
output_file,
xres=canvas_dpi,
yres=canvas_dpi,
raster_device='jpeggray',
log=log,
log=page_context.log,
page_dpi=(page_dpi, page_dpi),
pageno=page_number(input_file),
pageno=page_context.pageinfo.pageno + 1,
)
return output_file
def orient_page(infiles, output_file, log, context):
def get_orientation_correction(preview, page_context):
"""
Work out orientation correct for each page.
@@ -430,32 +353,22 @@ def orient_page(infiles, output_file, log, context):
"""
options = context.get_options()
page_pdf = next(ii for ii in infiles if ii.endswith('.page.pdf'))
if not options.rotate_pages:
re_symlink(page_pdf, output_file, log)
return
preview = next(ii for ii in infiles if ii.endswith('.preview.jpg'))
orient_conf = tesseract.get_orientation(
preview,
engine_mode=options.tesseract_oem,
timeout=options.tesseract_timeout,
log=log,
engine_mode=page_context.options.tesseract_oem,
timeout=page_context.options.tesseract_timeout,
log=page_context.log,
)
direction = {0: '⇧', 90: '⇨', 180: '⇩', 270: '⇦'}
pageno = page_number(page_pdf) - 1
pdfinfo = context.get_pdfinfo()
existing_rotation = pdfinfo[pageno].rotation
existing_rotation = page_context.pageinfo.rotation
correction = orient_conf.angle % 360
apply_correction = False
action = ''
if orient_conf.confidence >= options.rotate_pages_threshold:
if orient_conf.confidence >= page_context.options.rotate_pages_threshold:
if correction != 0:
apply_correction = True
action = ' - will rotate'
@@ -474,26 +387,25 @@ def orient_page(infiles, output_file, log, context):
)
facing += 'page is facing {}'.format(direction.get(orient_conf.angle, '?'))
log.info(
page_context.log.debug(
'{pagenum:4d}: {facing}, confidence {conf:.2f}{action}'.format(
pagenum=page_number(preview),
pagenum=page_context.pageinfo.pageno,
facing=facing,
conf=orient_conf.confidence,
action=action,
)
)
re_symlink(page_pdf, output_file, log)
if apply_correction:
context.set_rotation(pageno, correction)
return correction
return 0
def rasterize_with_ghostscript(input_file, output_file, log, context):
options = context.get_options()
pageinfo = get_pageinfo(input_file, context)
def rasterize(input_file, page_context, correction=0):
colorspaces = ['pngmono', 'pnggray', 'png256', 'png16m']
device_idx = 0
output_file = page_context.get_path('rasterize.png')
pageinfo = page_context.pageinfo
def at_least(cs):
return max(device_idx, colorspaces.index(cs))
@@ -511,14 +423,12 @@ def rasterize_with_ghostscript(input_file, output_file, log, context):
device = colorspaces[device_idx]
log.debug(f"Rasterize {os.path.basename(input_file)} with {device}")
page_context.log.debug(f"Rasterize with {device}")
# Produce the page image with square resolution or else deskew and OCR
# will not work properly.
canvas_dpi = get_canvas_square_dpi(pageinfo, options)
page_dpi = get_page_square_dpi(pageinfo, options)
correction = context.get_rotation(page_number(input_file) - 1)
canvas_dpi = get_canvas_square_dpi(pageinfo, page_context.options)
page_dpi = get_page_square_dpi(pageinfo, page_context.options)
ghostscript.rasterize_pdf(
input_file,
@@ -526,64 +436,47 @@ def rasterize_with_ghostscript(input_file, output_file, log, context):
xres=canvas_dpi,
yres=canvas_dpi,
raster_device=device,
log=log,
log=page_context.log,
page_dpi=(page_dpi, page_dpi),
pageno=page_number(input_file),
pageno=pageinfo.pageno + 1,
rotation=correction,
filter_vector=options.remove_vectors,
filter_vector=page_context.options.remove_vectors,
)
return output_file
def preprocess_remove_background(input_file, output_file, log, context):
options = context.get_options()
if not options.remove_background:
re_symlink(input_file, output_file, log)
return
pageinfo = get_pageinfo(input_file, context)
if any(image.bpc > 1 for image in pageinfo.images):
def preprocess_remove_background(input_file, page_context):
if any(image.bpc > 1 for image in page_context.pageinfo.images):
output_file = page_context.get_path('pp_rm_bg.png')
leptonica.remove_background(input_file, output_file)
return output_file
else:
log.info(f"{pageinfo.pageno:4d}: background removal skipped on mono page")
re_symlink(input_file, output_file, log)
page_context.log.info("background removal skipped on mono page")
return input_file
def preprocess_deskew(input_file, output_file, log, context):
options = context.get_options()
if not options.deskew:
re_symlink(input_file, output_file, log)
return
pageinfo = get_pageinfo(input_file, context)
dpi = get_page_square_dpi(pageinfo, options)
def preprocess_deskew(input_file, page_context):
output_file = page_context.get_path('pp_deskew.png')
dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
leptonica.deskew(input_file, output_file, dpi)
return output_file
def preprocess_clean(input_file, output_file, log, context):
options = context.get_options()
if not options.clean:
re_symlink(input_file, output_file, log)
return
def preprocess_clean(input_file, page_context):
from .exec import unpaper
pageinfo = get_pageinfo(input_file, context)
dpi = get_page_square_dpi(pageinfo, options)
unpaper.clean(input_file, output_file, dpi, log, options.unpaper_args)
output_file = page_context.get_path('pp_clean.png')
dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
unpaper.clean(input_file, output_file, dpi, page_context.log, page_context.options.unpaper_args)
return output_file
def select_ocr_image(infiles, output_file, log, context):
"""Select the image we send for OCR. May not be the same as the display
def create_ocr_image(image, page_context):
"""Create the image we send for OCR. May not be the same as the display
image depending on preprocessing. This image will never be shown to the
user."""
image = infiles[0]
options = context.get_options()
pageinfo = get_pageinfo(image, context)
output_file = page_context.get_path('ocr.png')
options = page_context.options
with Image.open(image) as im:
from PIL import ImageColor
from PIL import ImageDraw
@@ -593,7 +486,7 @@ def select_ocr_image(infiles, output_file, log, context):
draw = ImageDraw.ImageDraw(im)
xres, yres = im.info['dpi']
log.debug('resolution %r %r', xres, yres)
page_context.log.info('resolution %r %r' % (xres, yres))
if not options.force_ocr:
# Do not mask text areas when forcing OCR, because we need to OCR
@@ -602,7 +495,7 @@ def select_ocr_image(infiles, output_file, log, context):
if options.redo_ocr:
mask = True # Mask visible text, but not invisible text
for textarea in pageinfo.get_textareas(visible=mask, corrupt=None):
for textarea in page_context.pageinfo.get_textareas(visible=mask, corrupt=None):
# Calculate resolution based on the image size and page dimensions
# without regard whatever resolution is in pageinfo (may differ or
# be None)
@@ -615,7 +508,7 @@ def select_ocr_image(infiles, output_file, log, context):
im.height - bbox[1] * yscale,
]
pixcoords = [int(round(c)) for c in pixcoords]
log.debug('blanking %r', pixcoords)
print('blanking %r', pixcoords)
draw.rectangle(pixcoords, fill=white)
# draw.rectangle(pixcoords, outline=pink)
@@ -627,7 +520,7 @@ def select_ocr_image(infiles, output_file, log, context):
barcodes = pix.locate_barcodes()
for barcode in barcodes:
decoded, rect = barcode
log.info('masking barcode %s %r', decoded, rect)
print('masking barcode %s %r', decoded, rect)
draw.rectangle(rect, fill=white)
im = pix.topil()
@@ -635,13 +528,16 @@ def select_ocr_image(infiles, output_file, log, context):
# Pillow requires integer DPI
dpi = round(xres), round(yres)
im.save(output_file, dpi=dpi)
return output_file
def ocr_tesseract_hocr(input_file, output_files, log, context):
options = context.get_options()
def ocr_tesseract_hocr(input_file, page_context):
hocr_out = page_context.get_path('ocr_hocr.hocr')
hocr_text_out = page_context.get_path('ocr_hocr.txt')
options = page_context.options
tesseract.generate_hocr(
input_file=input_file,
output_files=output_files,
output_files=[hocr_out, hocr_text_out],
language=options.language,
engine_mode=options.tesseract_oem,
tessconfig=options.tesseract_config,
@@ -649,86 +545,57 @@ def ocr_tesseract_hocr(input_file, output_files, log, context):
pagesegmode=options.tesseract_pagesegmode,
user_words=options.user_words,
user_patterns=options.user_patterns,
log=log,
log=page_context.log,
)
return (hocr_out, hocr_text_out)
def select_visible_page_image(infiles, output_file, log, context):
"""Selects a whole page image that we can show the user (if necessary)"""
options = context.get_options()
if options.clean_final:
image_suffix = '.pp-clean.png'
elif options.deskew:
image_suffix = '.pp-deskew.png'
elif options.remove_background:
image_suffix = '.pp-background.png'
else:
image_suffix = '.page.png'
image = next(ii for ii in infiles if ii.endswith(image_suffix))
pageinfo = get_pageinfo(image, context)
if pageinfo.images and all(im.enc == 'jpeg' for im in pageinfo.images):
log.debug(f'{page_number(image):4d}: JPEG input -> JPEG output')
# If all images were JPEGs originally, produce a JPEG as output
with Image.open(image) as im:
# At this point the image should be a .png, but deskew, unpaper
# might have removed the DPI information. In this case, fall back to
# square DPI used to rasterize. When the preview image was
# rasterized, it was also converted to square resolution, which is
# what we want to give tesseract, so keep it square.
fallback_dpi = get_page_square_dpi(pageinfo, options)
dpi = im.info.get('dpi', (fallback_dpi, fallback_dpi))
# Pillow requires integer DPI
dpi = round(dpi[0]), round(dpi[1])
im.save(output_file, format='JPEG', dpi=dpi)
else:
re_symlink(image, output_file, log)
def should_visible_page_image_use_jpg(pageinfo):
# If all images were JPEGs originally, produce a JPEG as output
return pageinfo.images and all(im.enc == 'jpeg' for im in pageinfo.images)
def select_image_layer(infiles, output_file, log, context):
"""Selects the image layer for the output page. If possible this is the
orientation-corrected input page, or an image of the whole page converted
to PDF."""
def create_visible_page_jpg(image, page_context):
output_file = page_context.get_path('visible.jpg')
with Image.open(image) as im:
# At this point the image should be a .png, but deskew, unpaper
# might have removed the DPI information. In this case, fall back to
# square DPI used to rasterize. When the preview image was
# rasterized, it was also converted to square resolution, which is
# what we want to give tesseract, so keep it square.
fallback_dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
dpi = im.info.get('dpi', (fallback_dpi, fallback_dpi))
options = context.get_options()
page_pdf = next(ii for ii in infiles if ii.endswith('.ocr.oriented.pdf'))
image = next(ii for ii in infiles if ii.endswith('.image'))
# Pillow requires integer DPI
dpi = round(dpi[0]), round(dpi[1])
im.save(output_file, format='JPEG', dpi=dpi)
return output_file
if options.lossless_reconstruction:
log.debug(
f"{page_number(page_pdf):4d}: page eligible for lossless reconstruction"
)
re_symlink(page_pdf, output_file, log) # Still points to multipage
return
pageinfo = get_pageinfo(image, context)
def create_pdf_page_from_image(image, page_context):
# We rasterize a square DPI version of each page because most image
# processing tools don't support rectangular DPI. Use the square DPI as it
# accurately describes the image. It would be possible to resample the image
# at this stage back to non-square DPI to more closely resemble the input,
# except that the hocr renderer does not understand non-square DPI. The
# sandwich renderer would be fine.
dpi = get_page_square_dpi(pageinfo, options)
output_file = page_context.get_path('visible.pdf')
dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
layout_fun = img2pdf.get_fixed_dpi_layout_fun((dpi, dpi))
# This create a single page PDF
with open(image, 'rb') as imfile, open(output_file, 'wb') as pdf:
log.debug(f'{page_number(page_pdf):4d}: convert')
page_context.log.debug('convert')
img2pdf.convert(
imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf
)
log.debug(f'{page_number(page_pdf):4d}: convert done')
page_context.log.debug('convert done')
return output_file
def render_hocr_page(infiles, output_file, log, context):
options = context.get_options()
hocr = next(ii for ii in infiles if ii.endswith('.hocr'))
pageinfo = get_pageinfo(hocr, context)
dpi = get_page_square_dpi(pageinfo, options)
def render_hocr_page(hocr, page_context):
output_file = page_context.get_path('ocr_hocr.pdf')
dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
hocrtransform = HocrTransform(hocr, dpi)
hocrtransform.to_pdf(
output_file,
@@ -737,17 +604,13 @@ def render_hocr_page(infiles, output_file, log, context):
invisibleText=True,
interwordSpaces=True,
)
return output_file
def ocr_tesseract_textonly_pdf(infiles, outfiles, log, context):
options = context.get_options()
input_image = next((ii for ii in infiles if ii.endswith('.ocr.png')), '')
if not input_image:
raise ValueError("No image rendered?")
output_pdf = next((ii for ii in outfiles if ii.endswith('.pdf')))
output_text = next((ii for ii in outfiles if ii.endswith('.txt')))
def ocr_tesseract_textonly_pdf(input_image, page_context):
output_pdf = page_context.get_path('ocr_tess.pdf')
output_text = page_context.get_path('ocr_tess.txt')
options = page_context.options
tesseract.generate_pdf(
input_image=input_image,
skip_pdf=None,
@@ -761,8 +624,9 @@ def ocr_tesseract_textonly_pdf(infiles, outfiles, log, context):
pagesegmode=options.tesseract_pagesegmode,
user_words=options.user_words,
user_patterns=options.user_patterns,
log=log,
log=page_context.log,
)
return (output_pdf, output_text)
def get_docinfo(base_pdf, options):
@@ -804,63 +668,51 @@ def get_docinfo(base_pdf, options):
return pdfmark
def generate_postscript_stub(input_file, output_file, log, context):
def generate_postscript_stub(context):
output_file = context.get_path('pdfa.ps')
generate_pdfa_ps(output_file)
return output_file
def convert_to_pdfa(input_files_groups, output_file, log, context):
options = context.get_options()
input_pdfinfo = context.get_pdfinfo()
input_files = list(f for f in flatten_groups(input_files_groups))
layers_file = next(
(ii for ii in input_files if ii.endswith('layers.rendered.pdf')), None
)
def convert_to_pdfa(input_pdf, input_ps_stub, context):
options = context.options
input_pdfinfo = context.pdfinfo
output_file = context.get_path('pdfa.pdf')
# If the DocumentInfo record contains NUL characters, Ghostscript will
# produce XMP metadata which contains invalid XML entities (&#0;).
# NULs in DocumentInfo seem to be common since older Acrobats included them.
# pikepdf can deal with this, but we make the world a better place by
# stamping them out as soon as possible.
pdf_layers_file = pikepdf.open(layers_file)
if pdf_layers_file.docinfo:
pdf_file = pikepdf.open(input_pdf)
if pdf_file.docinfo:
modified = False
for k, v in pdf_layers_file.docinfo.items():
for k, v in pdf_file.docinfo.items():
if b'\x00' in bytes(v):
pdf_layers_file.docinfo[k] = bytes(v).replace(b'\x00', b'')
pdf_file.docinfo[k] = bytes(v).replace(b'\x00', b'')
modified = True
if modified:
pdf_layers_file.save(layers_file)
del pdf_layers_file
pdf_file.save(input_pdf)
del pdf_file
ps = next((ii for ii in input_files if ii.endswith('.ps')), None)
ghostscript.generate_pdfa(
pdf_version=input_pdfinfo.min_version,
pdf_pages=[layers_file, ps],
pdf_pages=[input_pdf, input_ps_stub],
output_file=output_file,
compression=options.pdfa_image_compression,
log=log,
log=context.log,
threads=options.jobs or 1,
pdfa_part=options.output_type[-1], # is pdfa-1, pdfa-2, or pdfa-3
)
return output_file
def metadata_fixup(input_files_groups, output_file, log, context):
options = context.get_options()
input_files = list(f for f in flatten_groups(input_files_groups))
original_file = next(
(ii for ii in input_files if ii.endswith('.repaired.pdf')), None
)
layers_file = next(
(ii for ii in input_files if ii.endswith('layers.rendered.pdf')), None
)
pdfa_file = next((ii for ii in input_files if ii.endswith('pdfa.pdf')), None)
original = pikepdf.open(original_file)
def metadata_fixup(working_file, context):
output_file = context.get_path('metafix.pdf')
options = context.options
original = pikepdf.open(context.origin)
docinfo = get_docinfo(original, options)
working_file = pdfa_file if pdfa_file else layers_file
pdf = pikepdf.open(working_file)
with pdf.open_metadata() as meta:
meta.load_from_docinfo(docinfo, delete_missing=False)
@@ -868,16 +720,25 @@ def metadata_fixup(input_files_groups, output_file, log, context):
# match Ghostscript, for consistency
if 'xmp:CreateDate' not in meta:
meta['xmp:CreateDate'] = meta.get('xmp:ModifyDate', '')
if pdfa_file:
meta_original = original.open_metadata()
not_copied = set(meta_original.keys()) - set(meta.keys())
if not_copied:
log.warning(
meta_original = original.open_metadata()
not_copied = set(meta_original.keys()) - set(meta.keys())
if not_copied:
if options.output_type.startswith('pdfa'):
context.log.warn(
"Some input metadata could not be copied because it is not "
"permitted in PDF/A. You may wish to examine the output "
"PDF's XMP metadata."
)
log.debug(
context.log.debug(
"The following metadata fields were not copied: %r", not_copied
)
else:
context.log.error(
"Some input metadata could not be copied."
"You may wish to examine the output PDF's XMP metadata."
)
context.log.info(
"The following metadata fields were not copied: %r", not_copied
)
@@ -886,23 +747,18 @@ def metadata_fixup(input_files_groups, output_file, log, context):
compress_streams=True,
object_stream_mode=pikepdf.ObjectStreamMode.generate,
)
return output_file
def optimize_pdf(input_file, output_file, log, context):
optimize(input_file, output_file, log, context)
def optimize_pdf(input_file, context):
output_file = context.get_path('optimize.pdf')
optimize(input_file, output_file, context)
return output_file
def merge_sidecars(input_files_groups, output_file, log, context):
pdfinfo = context.get_pdfinfo()
txt_files = [None] * len(pdfinfo)
for infile in flatten_groups(input_files_groups):
if infile.endswith('.txt'):
idx = page_number(infile) - 1
txt_files[idx] = infile
def write_pages(stream):
def merge_sidecars(txt_files, context):
output_file = context.get_path('sidecar.txt')
with open(output_file, 'w', encoding="utf-8") as stream:
for page_num, txt_file in enumerate(txt_files):
if page_num != 0:
stream.write('\f') # Form feed between pages
@@ -920,18 +776,11 @@ def merge_sidecars(input_files_groups, output_file, log, context):
stream.write(txt)
else:
stream.write(f'[OCR skipped on page {(page_num + 1)}]')
if output_file == '-':
write_pages(sys.stdout)
sys.stdout.flush()
else:
with open(output_file, 'w', encoding="utf-8") as out:
write_pages(out)
return output_file
def copy_final(input_files, output_file, log, context):
input_file = next((ii for ii in input_files if ii.endswith('.pdf')))
log.debug('%s -> %s', input_file, output_file)
def copy_final(input_file, output_file, context):
context.log.debug('%s -> %s', input_file, output_file)
with open(input_file, 'rb') as input_stream:
if output_file == '-':
copyfileobj(input_stream, sys.stdout.buffer)
src/ocrmypdf/_pipeline_simple.py
-784
View File
@@ -1,784 +0,0 @@
# © 2016 James R. Barlow: github.com/jbarlow83
#
# This file is part of OCRmyPDF.
#
# OCRmyPDF is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OCRmyPDF is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
import os
import re
import sys
from datetime import datetime, timezone
from shutil import copyfileobj
import img2pdf
from PIL import Image
import pikepdf
from pikepdf.models.metadata import encode_pdf_date
from . import PROGRAM_NAME, VERSION, leptonica
from .exceptions import (
DpiError,
EncryptedPdfError,
InputFileError,
UnsupportedImageFormatError,
)
from .exec import ghostscript, tesseract
from .helpers import (
re_symlink
)
from .hocrtransform import HocrTransform
from .optimize import optimize
from .pdfa import generate_pdfa_ps
from .pdfinfo import Colorspace, PdfInfo
VECTOR_PAGE_DPI = 400
def triage_image_file(input_file, output_file, log, options):
try:
log.info("Input file is not a PDF, checking if it is an image...")
im = Image.open(input_file)
except EnvironmentError as e:
msg = str(e)
# Recover the original filename
realpath = ''
if os.path.islink(input_file):
realpath = os.path.realpath(input_file)
elif os.path.isfile(input_file):
realpath = '<stdin>'
msg = msg.replace(input_file, realpath)
log.error(msg)
raise UnsupportedImageFormatError() from e
else:
log.info("Input file is an image")
if 'dpi' in im.info:
if im.info['dpi'] <= (96, 96) and not options.image_dpi:
log.info("Image size: (%d, %d)" % im.size)
log.info("Image resolution: (%d, %d)" % im.info['dpi'])
log.error(
"Input file is an image, but the resolution (DPI) is "
"not credible. Estimate the resolution at which the "
"image was scanned and specify it using --image-dpi."
)
raise DpiError()
elif not options.image_dpi:
log.info("Image size: (%d, %d)" % im.size)
log.error(
"Input file is an image, but has no resolution (DPI) "
"in its metadata. Estimate the resolution at which "
"image was scanned and specify it using --image-dpi."
)
raise DpiError()
if im.mode in ('RGBA', 'LA'):
log.error(
"The input image has an alpha channel. Remove the alpha "
"channel first."
)
raise UnsupportedImageFormatError()
if 'iccprofile' not in im.info:
if im.mode == 'RGB':
log.info('Input image has no ICC profile, assuming sRGB')
elif im.mode == 'CMYK':
log.info('Input CMYK image has no ICC profile, not usable')
raise UnsupportedImageFormatError()
im.close()
try:
log.info("Image seems valid. Try converting to PDF...")
layout_fun = img2pdf.default_layout_fun
if options.image_dpi:
layout_fun = img2pdf.get_fixed_dpi_layout_fun(
(options.image_dpi, options.image_dpi)
)
with open(output_file, 'wb') as outf:
img2pdf.convert(
input_file,
layout_fun=layout_fun,
with_pdfrw=False,
outputstream=outf
)
log.info("Successfully converted to PDF, processing...")
except img2pdf.ImageOpenError as e:
log.error(e)
raise UnsupportedImageFormatError() from e
def _pdf_guess_version(input_file, search_window=1024):
"""Try to find version signature at start of file.
Not robust enough to deal with appended files.
Returns empty string if not found, indicating file is probably not PDF.
"""
with open(input_file, 'rb') as f:
signature = f.read(search_window)
m = re.search(br'%PDF-(\d\.\d)', signature)
if m:
return m.group(1)
return ''
def triage(input_file, output_file, log, context):
options = context.get_options()
try:
if _pdf_guess_version(input_file):
if options.image_dpi:
log.warning(
"Argument --image-dpi ignored because the "
"input file is a PDF, not an image."
)
re_symlink(input_file, output_file, log)
return
except EnvironmentError as e:
log.error(e)
raise InputFileError() from e
triage_image_file(input_file, output_file, log, options)
def get_pdfinfo(input_file, detailed_page_analysis=False):
try:
return PdfInfo(
input_file, detailed_page_analysis=detailed_page_analysis
)
except pikepdf.PasswordError:
raise EncryptedPdfError()
except pikepdf.PdfError:
raise InputFileError()
def validate_pdfinfo_options(context):
log = context.log
pdfinfo = context.pdfinfo
options = context.options
if pdfinfo.needs_rendering:
log.error(
"This PDF contains dynamic XFA forms created by Adobe LiveCycle "
"Designer and can only be read by Adobe Acrobat or Adobe Reader."
)
raise InputFileError()
if pdfinfo.has_userunit and options.output_type.startswith('pdfa'):
log.error(
"This input file uses a PDF feature that is not supported "
"by Ghostscript, so you cannot use --output-type=pdfa for this "
"file. (Specifically, it uses the PDF-1.6 /UserUnit feature to "
"support very large or small page sizes, and Ghostscript cannot "
"output these files.) Use --output-type=pdf instead."
)
raise InputFileError()
if pdfinfo.has_acroform:
if options.redo_ocr:
log.error(
"This PDF has a user fillable form. --redo-ocr is not "
"currently possible on such files."
)
raise InputFileError()
else:
log.warn(
"This PDF has a fillable form. "
"Chances are it is a pure digital "
"document that does not need OCR."
)
if not options.force_ocr:
log.info(
"Use the option --force-ocr to produce an image of the "
"form and all filled form fields. The output PDF will be "
"'flattened' and will no longer be fillable."
)
def get_page_dpi(pageinfo, options):
"Get the DPI when nonsquare DPI is tolerable"
xres = max(
pageinfo.xres or VECTOR_PAGE_DPI,
options.oversample or 0,
VECTOR_PAGE_DPI if pageinfo.has_vector else 0,
)
yres = max(
pageinfo.yres or VECTOR_PAGE_DPI,
options.oversample or 0,
VECTOR_PAGE_DPI if pageinfo.has_vector else 0,
)
return (float(xres), float(yres))
def get_page_square_dpi(pageinfo, options):
"Get the DPI when we require xres == yres, scaled to physical units"
xres = pageinfo.xres or 0
yres = pageinfo.yres or 0
userunit = pageinfo.userunit or 1
return float(
max(
(xres * userunit) or VECTOR_PAGE_DPI,
(yres * userunit) or VECTOR_PAGE_DPI,
VECTOR_PAGE_DPI if pageinfo.has_vector else 0,
options.oversample or 0,
)
)
def get_canvas_square_dpi(pageinfo, options):
"""Get the DPI when we require xres == yres, in Postscript units"""
return float(
max(
(pageinfo.xres) or VECTOR_PAGE_DPI,
(pageinfo.yres) or VECTOR_PAGE_DPI,
VECTOR_PAGE_DPI if pageinfo.has_vector else 0,
options.oversample or 0,
)
)
def is_ocr_required(page_context):
pageinfo = page_context.pageinfo
options = page_context.options
log = page_context.log
ocr_required = True
if pageinfo.has_text:
if not options.force_ocr and not (options.skip_text or options.redo_ocr):
log.error("page already has text! - aborting (use --force-ocr to force OCR)")
ocr_required = False
elif options.force_ocr:
log.info("page already has text! - rasterizing text and running OCR anyway")
ocr_required = True
elif options.redo_ocr:
if pageinfo.has_corrupt_text:
log.warn(
"some text on this page cannot be mapped to characters: "
"consider using --force-ocr instead",
)
else:
log.info("redoing OCR")
ocr_required = True
elif options.skip_text:
log.info("skipping all processing on this page")
ocr_required = False
elif not pageinfo.images and not options.lossless_reconstruction:
# We found a page with no images and no text. That means it may
# have vector art that the user wants to OCR. If we determined
# lossless reconstruction is not possible then we have to rasterize
# the image. So if OCR is being forced, take that to mean YES, go
# ahead and rasterize. If not forced, then pretend there's no text
# on the page at all so we don't lose anything.
# This could be made smarter by explicitly searching for vector art.
if options.force_ocr and options.oversample:
# The user really wants to reprocess this file
log.info(
"page has no images - "
f"rasterizing at {options.oversample} DPI because "
"--force-ocr --oversample was specified"
)
elif options.force_ocr:
# Warn the user they might not want to do this
log.warn(
"page has no images - "
"all vector content will be "
f"rasterized at {VECTOR_PAGE_DPI} DPI, losing some resolution and likely "
"increasing file size. Use --oversample to adjust the "
"DPI."
)
else:
log.info(
"page has no images - "
"skipping all processing on this page to avoid losing detail. "
"Use --force-ocr if you wish to perform OCR on pages that "
"have vector content."
)
ocr_required = False
if ocr_required and options.skip_big and pageinfo.images:
pixel_count = pageinfo.width_pixels * pageinfo.height_pixels
if pixel_count > (options.skip_big * 1_000_000):
ocr_required = False
log.warn(
"page too big, skipping OCR "
f"({(pixel_count / 1_000_000):.1f} MPixels > {options.skip_big:.1f} MPixels --skip-big)"
)
return ocr_required
def rasterize_preview(input_file, page_context):
output_file = page_context.get_path('rasterize_preview.jpg')
canvas_dpi = get_canvas_square_dpi(page_context.pageinfo, page_context.options)
page_dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
ghostscript.rasterize_pdf(
input_file,
output_file,
xres=canvas_dpi,
yres=canvas_dpi,
raster_device='jpeggray',
log=page_context.log,
page_dpi=(page_dpi, page_dpi),
pageno=page_context.pageinfo.pageno + 1,
)
return output_file
def get_orientation_correction(preview, page_context):
"""
Work out orientation correct for each page.
We ask Ghostscript to draw a preview page, which will rasterize with the
current /Rotate applied, and then ask Tesseract which way the page is
oriented. If the value of /Rotate is correct (e.g., a user already
manually fixed rotation), then Tesseract will say the page is pointing
up and the correction is zero. Otherwise, the orientation found by
Tesseract represents the clockwise rotation, or the counterclockwise
correction to rotation.
When we draw the real page for OCR, we rotate it by the CCW correction,
which points it (hopefully) upright. _weave.py takes care of the orienting
the image and text layers.
"""
orient_conf = tesseract.get_orientation(
preview,
engine_mode=page_context.options.tesseract_oem,
timeout=page_context.options.tesseract_timeout,
log=page_context.log,
)
direction = {0: '⇧', 90: '⇨', 180: '⇩', 270: '⇦'}
existing_rotation = page_context.pageinfo.rotation
correction = orient_conf.angle % 360
apply_correction = False
action = ''
if orient_conf.confidence >= page_context.options.rotate_pages_threshold:
if correction != 0:
apply_correction = True
action = ' - will rotate'
else:
action = ' - rotation appears correct'
else:
if correction != 0:
action = ' - confidence too low to rotate'
else:
action = ' - no change'
facing = ''
if existing_rotation != 0:
facing = 'with existing rotation {}, '.format(
direction.get(existing_rotation, '?')
)
facing += 'page is facing {}'.format(direction.get(orient_conf.angle, '?'))
page_context.log.debug(
'{pagenum:4d}: {facing}, confidence {conf:.2f}{action}'.format(
pagenum=page_context.pageinfo.pageno,
facing=facing,
conf=orient_conf.confidence,
action=action,
)
)
if apply_correction:
return correction
return 0
def rasterize(input_file, page_context, correction=0):
colorspaces = ['pngmono', 'pnggray', 'png256', 'png16m']
device_idx = 0
output_file = page_context.get_path('rasterize.png')
pageinfo = page_context.pageinfo
def at_least(cs):
return max(device_idx, colorspaces.index(cs))
for image in pageinfo.images:
if image.type_ != 'image':
continue # ignore masks
if image.bpc > 1:
if image.color == Colorspace.index:
device_idx = at_least('png256')
elif image.color == Colorspace.gray:
device_idx = at_least('pnggray')
else:
device_idx = at_least('png16m')
device = colorspaces[device_idx]
page_context.log.debug(f"Rasterize with {device}")
# Produce the page image with square resolution or else deskew and OCR
# will not work properly.
canvas_dpi = get_canvas_square_dpi(pageinfo, page_context.options)
page_dpi = get_page_square_dpi(pageinfo, page_context.options)
ghostscript.rasterize_pdf(
input_file,
output_file,
xres=canvas_dpi,
yres=canvas_dpi,
raster_device=device,
log=page_context.log,
page_dpi=(page_dpi, page_dpi),
pageno=pageinfo.pageno + 1,
rotation=correction,
filter_vector=page_context.options.remove_vectors,
)
return output_file
def preprocess_remove_background(input_file, page_context):
if any(image.bpc > 1 for image in page_context.pageinfo.images):
output_file = page_context.get_path('pp_rm_bg.png')
leptonica.remove_background(input_file, output_file)
return output_file
else:
page_context.log.info("background removal skipped on mono page")
return input_file
def preprocess_deskew(input_file, page_context):
output_file = page_context.get_path('pp_deskew.png')
dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
leptonica.deskew(input_file, output_file, dpi)
return output_file
def preprocess_clean(input_file, page_context):
from .exec import unpaper
output_file = page_context.get_path('pp_clean.png')
dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
unpaper.clean(input_file, output_file, dpi, page_context.log, page_context.options.unpaper_args)
return output_file
def create_ocr_image(image, page_context):
"""Create the image we send for OCR. May not be the same as the display
image depending on preprocessing. This image will never be shown to the
user."""
output_file = page_context.get_path('ocr.png')
options = page_context.options
with Image.open(image) as im:
from PIL import ImageColor
from PIL import ImageDraw
white = ImageColor.getcolor('#ffffff', im.mode)
# pink = ImageColor.getcolor('#ff0080', im.mode)
draw = ImageDraw.ImageDraw(im)
xres, yres = im.info['dpi']
page_context.log.info('resolution %r %r' % (xres, yres))
if not options.force_ocr:
# Do not mask text areas when forcing OCR, because we need to OCR
# all text areas
mask = None # Exclude both visible and invisible text from OCR
if options.redo_ocr:
mask = True # Mask visible text, but not invisible text
for textarea in page_context.pageinfo.get_textareas(visible=mask, corrupt=None):
# Calculate resolution based on the image size and page dimensions
# without regard whatever resolution is in pageinfo (may differ or
# be None)
bbox = [float(v) for v in textarea]
xscale, yscale = float(xres) / 72.0, float(yres) / 72.0
pixcoords = [
bbox[0] * xscale,
im.height - bbox[3] * yscale,
bbox[2] * xscale,
im.height - bbox[1] * yscale,
]
pixcoords = [int(round(c)) for c in pixcoords]
print('blanking %r', pixcoords)
draw.rectangle(pixcoords, fill=white)
# draw.rectangle(pixcoords, outline=pink)
if options.mask_barcodes or options.threshold:
pix = leptonica.Pix.frompil(im)
if options.threshold:
pix = pix.masked_threshold_on_background_norm()
if options.mask_barcodes:
barcodes = pix.locate_barcodes()
for barcode in barcodes:
decoded, rect = barcode
print('masking barcode %s %r', decoded, rect)
draw.rectangle(rect, fill=white)
im = pix.topil()
del draw
# Pillow requires integer DPI
dpi = round(xres), round(yres)
im.save(output_file, dpi=dpi)
return output_file
def ocr_tesseract_hocr(input_file, page_context):
hocr_out = page_context.get_path('ocr_hocr.hocr')
hocr_text_out = page_context.get_path('ocr_hocr.txt')
options = page_context.options
tesseract.generate_hocr(
input_file=input_file,
output_files=[hocr_out, hocr_text_out],
language=options.language,
engine_mode=options.tesseract_oem,
tessconfig=options.tesseract_config,
timeout=options.tesseract_timeout,
pagesegmode=options.tesseract_pagesegmode,
user_words=options.user_words,
user_patterns=options.user_patterns,
log=page_context.log,
)
return (hocr_out, hocr_text_out)
def should_visible_page_image_use_jpg(pageinfo):
# If all images were JPEGs originally, produce a JPEG as output
return pageinfo.images and all(im.enc == 'jpeg' for im in pageinfo.images)
def create_visible_page_jpg(image, page_context):
output_file = page_context.get_path('visible.jpg')
with Image.open(image) as im:
# At this point the image should be a .png, but deskew, unpaper
# might have removed the DPI information. In this case, fall back to
# square DPI used to rasterize. When the preview image was
# rasterized, it was also converted to square resolution, which is
# what we want to give tesseract, so keep it square.
fallback_dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
dpi = im.info.get('dpi', (fallback_dpi, fallback_dpi))
# Pillow requires integer DPI
dpi = round(dpi[0]), round(dpi[1])
im.save(output_file, format='JPEG', dpi=dpi)
return output_file
def create_pdf_page_from_image(image, page_context):
# We rasterize a square DPI version of each page because most image
# processing tools don't support rectangular DPI. Use the square DPI as it
# accurately describes the image. It would be possible to resample the image
# at this stage back to non-square DPI to more closely resemble the input,
# except that the hocr renderer does not understand non-square DPI. The
# sandwich renderer would be fine.
output_file = page_context.get_path('visible.pdf')
dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
layout_fun = img2pdf.get_fixed_dpi_layout_fun((dpi, dpi))
# This create a single page PDF
with open(image, 'rb') as imfile, open(output_file, 'wb') as pdf:
page_context.log.debug('convert')
img2pdf.convert(
imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf
)
page_context.log.debug('convert done')
return output_file
def render_hocr_page(hocr, page_context):
output_file = page_context.get_path('ocr_hocr.pdf')
dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
hocrtransform = HocrTransform(hocr, dpi)
hocrtransform.to_pdf(
output_file,
imageFileName=None,
showBoundingboxes=False,
invisibleText=True,
interwordSpaces=True,
)
return output_file
def ocr_tesseract_textonly_pdf(input_image, page_context):
output_pdf = page_context.get_path('ocr_tess.pdf')
output_text = page_context.get_path('ocr_tess.txt')
options = page_context.options
tesseract.generate_pdf(
input_image=input_image,
skip_pdf=None,
output_pdf=output_pdf,
output_text=output_text,
language=options.language,
engine_mode=options.tesseract_oem,
text_only=True,
tessconfig=options.tesseract_config,
timeout=options.tesseract_timeout,
pagesegmode=options.tesseract_pagesegmode,
user_words=options.user_words,
user_patterns=options.user_patterns,
log=page_context.log,
)
return (output_pdf, output_text)
def get_docinfo(base_pdf, options):
def from_document_info(key):
try:
s = base_pdf.docinfo[key]
return str(s)
except (KeyError, TypeError):
return ''
pdfmark = {
k: from_document_info(k)
for k in ('/Title', '/Author', '/Keywords', '/Subject', '/CreationDate')
}
if options.title:
pdfmark['/Title'] = options.title
if options.author:
pdfmark['/Author'] = options.author
if options.keywords:
pdfmark['/Keywords'] = options.keywords
if options.subject:
pdfmark['/Subject'] = options.subject
if options.pdf_renderer == 'sandwich':
renderer_tag = 'OCR-PDF'
else:
renderer_tag = 'OCR'
pdfmark['/Creator'] = (
f'{PROGRAM_NAME} {VERSION} / ' f'Tesseract {renderer_tag} {tesseract.version()}'
)
pdfmark['/Producer'] = f'pikepdf {pikepdf.__version__}'
if 'OCRMYPDF_CREATOR' in os.environ:
pdfmark['/Creator'] = os.environ['OCRMYPDF_CREATOR']
if 'OCRMYPDF_PRODUCER' in os.environ:
pdfmark['/Producer'] = os.environ['OCRMYPDF_PRODUCER']
pdfmark['/ModDate'] = encode_pdf_date(datetime.now(timezone.utc))
return pdfmark
def generate_postscript_stub(context):
output_file = context.get_path('pdfa.ps')
generate_pdfa_ps(output_file)
return output_file
def convert_to_pdfa(input_pdf, input_ps_stub, context):
options = context.options
input_pdfinfo = context.pdfinfo
output_file = context.get_path('pdfa.pdf')
# If the DocumentInfo record contains NUL characters, Ghostscript will
# produce XMP metadata which contains invalid XML entities (&#0;).
# NULs in DocumentInfo seem to be common since older Acrobats included them.
# pikepdf can deal with this, but we make the world a better place by
# stamping them out as soon as possible.
pdf_file = pikepdf.open(input_pdf)
if pdf_file.docinfo:
modified = False
for k, v in pdf_file.docinfo.items():
if b'\x00' in bytes(v):
pdf_file.docinfo[k] = bytes(v).replace(b'\x00', b'')
modified = True
if modified:
pdf_file.save(input_pdf)
del pdf_file
ghostscript.generate_pdfa(
pdf_version=input_pdfinfo.min_version,
pdf_pages=[input_pdf, input_ps_stub],
output_file=output_file,
compression=options.pdfa_image_compression,
log=context.log,
threads=options.jobs or 1,
pdfa_part=options.output_type[-1], # is pdfa-1, pdfa-2, or pdfa-3
)
return output_file
def metadata_fixup(working_file, context):
output_file = context.get_path('metafix.pdf')
options = context.options
original = pikepdf.open(context.origin)
docinfo = get_docinfo(original, options)
pdf = pikepdf.open(working_file)
with pdf.open_metadata() as meta:
meta.load_from_docinfo(docinfo, delete_missing=False)
# If xmp:CreateDate is missing, set it to the modify date to
# match Ghostscript, for consistency
if 'xmp:CreateDate' not in meta:
meta['xmp:CreateDate'] = meta.get('xmp:ModifyDate', '')
meta_original = original.open_metadata()
not_copied = set(meta_original.keys()) - set(meta.keys())
if not_copied:
context.log.warning(
"Some input metadata could not be copied because it is not "
"permitted in PDF/A. You may wish to examine the output "
"PDF's XMP metadata."
)
context.log.debug(
"The following metadata fields were not copied: %r", not_copied
)
pdf.save(
output_file,
compress_streams=True,
object_stream_mode=pikepdf.ObjectStreamMode.generate,
)
return output_file
def optimize_pdf(input_file, context):
output_file = context.get_path('optimize.pdf')
optimize(input_file, output_file, context)
return output_file
def merge_sidecars(txt_files, context):
output_file = context.get_path('sidecar.txt')
with open(output_file, 'w', encoding="utf-8") as stream:
for page_num, txt_file in enumerate(txt_files):
if page_num != 0:
stream.write('\f') # Form feed between pages
if txt_file:
with open(txt_file, 'r', encoding="utf-8") as in_:
txt = in_.read()
# Tesseract v4 alpha started adding form feeds in
# commit aa6eb6b
# No obvious way to detect what binaries will do this, so
# for consistency just ignore its form feeds and insert our
# own
if txt.endswith('\f'):
stream.write(txt[:-1])
else:
stream.write(txt)
else:
stream.write(f'[OCR skipped on page {(page_num + 1)}]')
return output_file
def copy_final(input_file, output_file, context):
context.log.debug('%s -> %s', input_file, output_file)
with open(input_file, 'rb') as input_stream:
if output_file == '-':
copyfileobj(input_stream, sys.stdout.buffer)
sys.stdout.flush()
else:
# At this point we overwrite the output_file specified by the user
# use copyfileobj because then we use open() to create the file and
# get the appropriate umask, ownership, etc.
with open(output_file, 'wb') as output_stream:
copyfileobj(input_stream, output_stream)
src/ocrmypdf/_ruffus.py
-508
View File
@@ -1,508 +0,0 @@
# © 2016 James R. Barlow: github.com/jbarlow83
#
# This file is part of OCRmyPDF.
#
# OCRmyPDF is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OCRmyPDF is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
import os
import re
import sys
import atexit
from tempfile import mkdtemp
from ruffus import (
Pipeline,
formatter,
regex,
suffix,
cmdline,
proxy_logger,
ruffus_exceptions
)
from .exec import qpdf
from ._jobcontext import JobContext, JobContextManager, cleanup_working_files
from ._weave import weave_layers
from ._pipeline import (
triage,
repair_and_parse_pdf,
marker_pages,
ocr_or_skip,
rasterize_preview,
orient_page,
rasterize_with_ghostscript,
preprocess_remove_background,
preprocess_deskew,
preprocess_clean,
select_ocr_image,
ocr_tesseract_hocr,
select_visible_page_image,
select_image_layer,
render_hocr_page,
ocr_tesseract_textonly_pdf,
generate_postscript_stub,
convert_to_pdfa,
metadata_fixup,
merge_sidecars,
optimize_pdf,
copy_final
)
from . import exceptions as ocrmypdf_exceptions
from .exceptions import (
ExitCode,
ExitCodeException,
)
from .helpers import available_cpu_count
from .pdfa import file_claims_pdfa
from ._validation import (
check_closed_streams,
preamble,
check_options,
check_dependency_versions,
check_environ,
check_input_file,
check_requested_output_file,
report_output_file_size,
log_page_orientations,
logging_factory,
)
def cleanup_ruffus_error_message(msg):
msg = re.sub(r'\s+', r' ', msg)
msg = re.sub(r"\((.+?)\)", r'\1', msg)
msg = msg.strip()
return msg
def do_ruffus_exception(ruffus_five_tuple, options, log):
"""Replace the elaborate ruffus stack trace with a user friendly
description of the error message that occurred."""
exit_code = None
_task_name, _job_name, exc_name, exc_value, exc_stack = ruffus_five_tuple
if isinstance(exc_name, type):
# ruffus is full of mystery... sometimes (probably when the process
# group leader is killed) exc_name is the class object of the exception,
# rather than a str. So reach into the object and get its name.
exc_name = exc_name.__name__
if exc_name.startswith('ocrmypdf.exceptions.'):
base_exc_name = exc_name.replace('ocrmypdf.exceptions.', '')
exc_class = getattr(ocrmypdf_exceptions, base_exc_name)
exit_code = getattr(exc_class, 'exit_code', ExitCode.other_error)
try:
if isinstance(exc_value, exc_class):
exc_msg = str(exc_value)
elif isinstance(exc_value, str):
exc_msg = exc_value
else:
exc_msg = str(exc_class())
except Exception:
exc_msg = "Unknown"
if exc_name in ('builtins.SystemExit', 'SystemExit'):
match = re.search(r"\.(.+?)\)", exc_value)
exit_code_name = match.groups()[0]
exit_code = getattr(ExitCode, exit_code_name, 'other_error')
elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':
log.error(cleanup_ruffus_error_message(exc_value))
exit_code = ExitCode.input_file
elif exc_name in ('builtins.KeyboardInterrupt', 'KeyboardInterrupt'):
# We have to print in this case because the log daemon might be toast
print("Interrupted by user", file=sys.stderr)
exit_code = ExitCode.ctrl_c
elif exc_name == 'subprocess.CalledProcessError':
# It's up to the subprocess handler to report something useful
msg = "Error occurred while running this command:"
log.error(msg + '\n' + exc_value)
exit_code = ExitCode.child_process_error
elif exc_name.startswith('ocrmypdf.exceptions.'):
if exc_msg:
log.error(exc_msg)
elif exc_name == 'PIL.Image.DecompressionBombError':
msg = cleanup_ruffus_error_message(exc_value)
msg += (
"\nUse the --max-image-mpixels argument to set increase the "
"maximum number of megapixels to accept."
)
log.error(msg)
exit_code = ExitCode.input_file
if exit_code is not None:
return exit_code
if not options.verbose:
log.error(exc_stack)
return ExitCode.other_error
def traverse_ruffus_exception(exceptions, options, log):
"""Traverse a RethrownJobError and output the exceptions
Ruffus presents exceptions as 5 element tuples. The RethrownJobException
has a list of exceptions like
e.job_exceptions = [(5-tuple), (5-tuple), ...]
ruffus < 2.7.0 had a bug with exception marshalling that would give
different output whether the main or child process raised the exception.
We no longer support this.
Attempting to log the exception itself will re-marshall it to the logger
which is normally running in another process. It's better to avoid re-
marshalling.
The exit code will be based on this, even if multiple exceptions occurred
at the same time."""
exit_codes = []
for exc in exceptions:
exit_code = do_ruffus_exception(exc, options, log)
exit_codes.append(exit_code)
return exit_codes[0] # Multiple codes are rare so take the first one
def build_pipeline(options, work_folder, log, context):
main_pipeline = Pipeline.pipelines['main']
# Triage
task_triage = main_pipeline.transform(
task_func=triage,
input=os.path.join(work_folder, 'origin'),
filter=formatter('(?i)'),
output=os.path.join(work_folder, 'origin.pdf'),
extras=[log, context],
)
task_repair_and_parse_pdf = main_pipeline.transform(
task_func=repair_and_parse_pdf,
input=task_triage,
filter=suffix('.pdf'),
output='.repaired.pdf',
output_dir=work_folder,
extras=[log, context],
)
# Split (kwargs for split seems to be broken, so pass plain args)
task_marker_pages = main_pipeline.split(
marker_pages,
task_repair_and_parse_pdf,
os.path.join(work_folder, '*.marker.pdf'),
extras=[log, context],
)
task_ocr_or_skip = main_pipeline.split(
ocr_or_skip,
task_marker_pages,
[
os.path.join(work_folder, '*.ocr.page.pdf'),
os.path.join(work_folder, '*.skip.page.pdf'),
],
extras=[log, context],
)
# Rasterize preview
task_rasterize_preview = main_pipeline.transform(
task_func=rasterize_preview,
input=task_ocr_or_skip,
filter=suffix('.page.pdf'),
output='.preview.jpg',
output_dir=work_folder,
extras=[log, context],
)
task_rasterize_preview.active_if(options.rotate_pages)
# Orient
task_orient_page = main_pipeline.collate(
task_func=orient_page,
input=[task_ocr_or_skip, task_rasterize_preview],
filter=regex(r".*/(\d{6})(\.ocr|\.skip)(?:\.page\.pdf|\.preview\.jpg)"),
output=os.path.join(work_folder, r'\1\2.oriented.pdf'),
extras=[log, context],
)
# Rasterize actual
task_rasterize_with_ghostscript = main_pipeline.transform(
task_func=rasterize_with_ghostscript,
input=task_orient_page,
filter=suffix('.ocr.oriented.pdf'),
output='.page.png',
output_dir=work_folder,
extras=[log, context],
)
# Preprocessing subpipeline
task_preprocess_remove_background = main_pipeline.transform(
task_func=preprocess_remove_background,
input=task_rasterize_with_ghostscript,
filter=suffix(".page.png"),
output=".pp-background.png",
extras=[log, context],
)
task_preprocess_deskew = main_pipeline.transform(
task_func=preprocess_deskew,
input=task_preprocess_remove_background,
filter=suffix(".pp-background.png"),
output=".pp-deskew.png",
extras=[log, context],
)
task_preprocess_clean = main_pipeline.transform(
task_func=preprocess_clean,
input=task_preprocess_deskew,
filter=suffix(".pp-deskew.png"),
output=".pp-clean.png",
extras=[log, context],
)
task_select_ocr_image = main_pipeline.collate(
task_func=select_ocr_image,
input=[task_preprocess_clean],
filter=regex(r".*/(\d{6})(?:\.page|\.pp-.*)\.png"),
output=os.path.join(work_folder, r"\1.ocr.png"),
extras=[log, context],
)
# HOCR OCR
task_ocr_tesseract_hocr = main_pipeline.transform(
task_func=ocr_tesseract_hocr,
input=task_select_ocr_image,
filter=suffix(".ocr.png"),
output=[".hocr", ".txt"],
extras=[log, context],
)
task_ocr_tesseract_hocr.graphviz(fillcolor='"#00cc66"')
task_ocr_tesseract_hocr.active_if(options.pdf_renderer == 'hocr')
task_select_visible_page_image = main_pipeline.collate(
task_func=select_visible_page_image,
input=[
task_rasterize_with_ghostscript,
task_preprocess_remove_background,
task_preprocess_deskew,
task_preprocess_clean,
],
filter=regex(r".*/(\d{6})(?:\.page|\.pp-.*)\.png"),
output=os.path.join(work_folder, r'\1.image'),
extras=[log, context],
)
task_select_visible_page_image.graphviz(shape='diamond')
task_select_image_layer = main_pipeline.collate(
task_func=select_image_layer,
input=[task_select_visible_page_image, task_orient_page],
filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.oriented\.pdf)"),
output=os.path.join(work_folder, r'\1.image-layer.pdf'),
extras=[log, context],
)
task_select_image_layer.graphviz(fillcolor='"#00cc66"', shape='diamond')
task_render_hocr_page = main_pipeline.transform(
task_func=render_hocr_page,
input=task_ocr_tesseract_hocr,
filter=regex(r".*/(\d{6})(?:\.hocr)"),
output=os.path.join(work_folder, r'\1.text.pdf'),
extras=[log, context],
)
task_render_hocr_page.graphviz(fillcolor='"#00cc66"')
task_render_hocr_page.active_if(options.pdf_renderer == 'hocr')
# Tesseract OCR + text only PDF
task_ocr_tesseract_textonly_pdf = main_pipeline.collate(
task_func=ocr_tesseract_textonly_pdf,
input=[task_select_ocr_image],
filter=regex(r".*/(\d{6})(?:\.ocr.png)"),
output=[
os.path.join(work_folder, r'\1.text.pdf'),
os.path.join(work_folder, r'\1.text.txt'),
],
extras=[log, context],
)
task_ocr_tesseract_textonly_pdf.graphviz(fillcolor='"#ff69b4"')
task_ocr_tesseract_textonly_pdf.active_if(options.pdf_renderer == 'sandwich')
task_weave_layers = main_pipeline.collate(
task_func=weave_layers,
input=[
task_repair_and_parse_pdf,
task_render_hocr_page,
task_ocr_tesseract_textonly_pdf,
task_select_image_layer,
],
filter=regex(
r".*/((?:\d{6}(?:\.text\.pdf|\.image-layer\.pdf))|(?:origin\.repaired\.pdf))"
),
output=os.path.join(work_folder, r'layers.rendered.pdf'),
extras=[log, context],
)
task_weave_layers.graphviz(fillcolor='"#00cc66"')
# PDF/A pdfmark
task_generate_postscript_stub = main_pipeline.transform(
task_func=generate_postscript_stub,
input=task_repair_and_parse_pdf,
filter=formatter(r'\.repaired\.pdf'),
output=os.path.join(work_folder, 'pdfa.ps'),
extras=[log, context],
)
task_generate_postscript_stub.active_if(options.output_type.startswith('pdfa'))
# PDF/A conversion
task_convert_to_pdfa = main_pipeline.merge(
task_func=convert_to_pdfa,
input=[task_generate_postscript_stub, task_weave_layers],
output=os.path.join(work_folder, 'pdfa.pdf'),
extras=[log, context],
)
task_convert_to_pdfa.active_if(options.output_type.startswith('pdfa'))
task_metadata_fixup = main_pipeline.merge(
task_func=metadata_fixup,
input=[task_repair_and_parse_pdf, task_weave_layers, task_convert_to_pdfa],
output=os.path.join(work_folder, 'metafix.pdf'),
extras=[log, context],
)
task_merge_sidecars = main_pipeline.merge(
task_func=merge_sidecars,
input=[task_ocr_tesseract_hocr, task_ocr_tesseract_textonly_pdf],
output=options.sidecar,
extras=[log, context],
)
task_merge_sidecars.active_if(options.sidecar)
# Optimize
task_optimize_pdf = main_pipeline.transform(
task_func=optimize_pdf,
input=task_metadata_fixup,
filter=suffix('.pdf'),
output='.optimized.pdf',
output_dir=work_folder,
extras=[log, context],
)
# Finalize
main_pipeline.merge(
task_func=copy_final,
input=[task_optimize_pdf],
output=options.output_file,
extras=[log, context],
)
def run_pipeline(options):
options.verbose_abbreviated_path = 1
if os.environ.get('_OCRMYPDF_THREADS'):
options.use_threads = True
if not check_closed_streams(options):
return ExitCode.bad_args
logger_args = {'verbose': options.verbose, 'quiet': options.quiet}
_log, _log_mutex = proxy_logger.make_shared_logger_and_proxy(
logging_factory, __name__, logger_args
)
preamble(_log)
check_code = check_options(options, _log)
if check_code != ExitCode.ok:
return check_code
check_dependency_versions(options, _log)
# Any changes to options will not take effect for options that are already
# bound to function parameters in the pipeline. (For example
# options.input_file, options.pdf_renderer are already bound.)
if not options.jobs:
options.jobs = available_cpu_count()
# Performance is improved by setting Tesseract to single threaded. In tests
# this gives better throughput than letting a smaller number of Tesseract
# jobs run multithreaded. Same story for pngquant. Tess <4 ignores this
# variable, but harmless to set if ignored.
os.environ.setdefault('OMP_THREAD_LIMIT', '1')
check_environ(options, _log)
if os.environ.get('PYTEST_CURRENT_TEST'):
os.environ['_OCRMYPDF_TEST_INFILE'] = options.input_file
try:
work_folder = mkdtemp(prefix="com.github.ocrmypdf.")
options.history_file = os.path.join(work_folder, 'ruffus_history.sqlite')
start_input_file = os.path.join(work_folder, 'origin')
check_input_file(options, _log, start_input_file)
check_requested_output_file(options, _log)
manager = JobContextManager()
manager.register('JobContext', JobContext) # pylint: disable=no-member
manager.start()
context = manager.JobContext() # pylint: disable=no-member
context.set_options(options)
context.set_work_folder(work_folder)
build_pipeline(options, work_folder, _log, context)
atexit.register(cleanup_working_files, work_folder, options)
if hasattr(os, 'nice'):
os.nice(5)
cmdline.run(options)
except ruffus_exceptions.RethrownJobError as e:
if options.verbose:
_log.debug(str(e)) # stringify exception so logger doesn't have to
exceptions = e.job_exceptions
exitcode = traverse_ruffus_exception(exceptions, options, _log)
if exitcode is None:
_log.error("Unexpected ruffus exception: " + str(e))
_log.error(repr(e))
return ExitCode.other_error
return exitcode
except ExitCodeException as e:
return e.exit_code
except Exception as e:
_log.error(str(e))
return ExitCode.other_error
if options.flowchart:
_log.info(f"Flowchart saved to {options.flowchart}")
return ExitCode.ok
elif options.output_file == '-':
_log.info("Output sent to stdout")
elif os.path.samefile(options.output_file, os.devnull):
pass # Say nothing when sending to dev null
else:
if options.output_type.startswith('pdfa'):
pdfa_info = file_claims_pdfa(options.output_file)
if pdfa_info['pass']:
msg = f"Output file is a {pdfa_info['conformance']} (as expected)"
_log.info(msg)
else:
msg = f"Output file is okay but is not PDF/A (seems to be {pdfa_info['conformance']})"
_log.warning(msg)
return ExitCode.pdfa_conversion_failed
if not qpdf.check(options.output_file, _log):
_log.warning('Output file: The generated PDF is INVALID')
return ExitCode.invalid_output_pdf
report_output_file_size(options, _log, start_input_file, options.output_file)
pdfinfo = context.get_pdfinfo()
if options.verbose:
from pprint import pformat
_log.debug(pformat(pdfinfo))
log_page_orientations(pdfinfo, _log)
return ExitCode.ok
src/ocrmypdf/_sync.py
+11 -52
View File
@@ -16,13 +16,11 @@
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
import os
# import re
# import sys
import atexit
from tempfile import mkdtemp
from ._jobcontext import cleanup_working_files
from ._jobcontext import PDFContext, get_logger, cleanup_working_files
from ._weave import weave_layers
from ._pipeline_simple import (
from ._pipeline import (
get_pdfinfo,
validate_pdfinfo_options,
is_ocr_required,
@@ -48,6 +46,7 @@ from ._pipeline_simple import (
)
from .exceptions import (
ExitCode,
ExitCodeException,
)
from .helpers import available_cpu_count
from ._validation import (
@@ -61,52 +60,6 @@ from ._validation import (
)
class Logger:
def __init__(self, prefix):
self.prefix = prefix
def debug(self, *argv):
print(self.prefix, *argv)
def info(self, *argv):
print(self.prefix, *argv)
def warn(self, *argv):
print(self.prefix, *argv)
def error(self, *argv):
print(self.prefix, *argv)
class PageContext:
def __init__(self, pdf_context, pageno):
self.pdf_context = pdf_context
self.options = pdf_context.options
self.pageno = pageno
self.pageinfo = pdf_context.pdfinfo[pageno]
self.log = Logger('%s Page %d: ' % (os.path.basename(pdf_context.origin), pageno + 1))
def get_path(self, name):
return os.path.join(self.pdf_context.work_folder, "page_%d_%s" % (self.pageno, name))
class PDFContext:
def __init__(self, options, work_folder, origin, pdfinfo):
self.options = options
self.work_folder = work_folder
self.origin = origin
self.pdfinfo = pdfinfo
self.log = Logger('%s: ' % os.path.basename(origin))
def get_path(self, name):
return os.path.join(self.work_folder, name)
def get_page_contexts(self):
npages = len(self.pdfinfo)
for n in range(npages):
yield PageContext(self, n)
def _exec_pipeline(options, work_folder, origin):
# Gather info of pdf
pdfinfo = get_pdfinfo(origin)
@@ -181,7 +134,7 @@ def run_pipeline(options):
if not check_closed_streams(options):
return ExitCode.bad_args
log = Logger('Pipeline')
log = get_logger(options, 'Pipeline')
preamble(log)
check_code = check_options(options, log)
if check_code != ExitCode.ok:
@@ -213,7 +166,13 @@ def run_pipeline(options):
if hasattr(os, 'nice'):
os.nice(5)
_exec_pipeline(options, work_folder, start_input_file)
try:
_exec_pipeline(options, work_folder, start_input_file)
except ExitCodeException as e:
return e.exit_code
except Exception as e:
log.error(str(e))
return ExitCode.other_error
return ExitCode.ok
src/ocrmypdf/optimize.py
+5 -14
View File
@@ -16,7 +16,6 @@
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
import concurrent.futures
import logging
import sys
from collections import defaultdict
from os import fspath
@@ -28,7 +27,7 @@ import pikepdf
from pikepdf import Name, Dictionary
from . import leptonica
from ._jobcontext import JobContext
from ._jobcontext import PDFContext
from .exec import jbig2enc, pngquant
from .helpers import re_symlink
@@ -82,9 +81,7 @@ def extract_image_jbig2(*, pike, root, log, image, xref, options):
pim, filtdp = result
if (
pim.bits_per_component == 1
and filtdp != Name.JBIG2Decode
and jbig2enc.available()
pim.bits_per_component == 1 and filtdp != Name.JBIG2Decode and jbig2enc.available()
):
try:
imgname = Path(root / f'{xref:08d}')
@@ -129,9 +126,7 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
return None
return xref, ext
elif (
pim.indexed
and pim.colorspace in pim.SIMPLE_COLORSPACES
and options.optimize >= 3
pim.indexed and pim.colorspace in pim.SIMPLE_COLORSPACES and options.optimize >= 3
):
# Try to improve on indexed images - these are far from low hanging
# fruit in most cases
@@ -492,10 +487,6 @@ def main(infile, outfile, level, jobs=1):
self.jbig2_page_group_size = 0
self.jbig2_lossy = jb2lossy
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger()
ctx = JobContext()
options = OptimizeOptions(
jobs=jobs,
optimize=int(level),
@@ -503,11 +494,11 @@ def main(infile, outfile, level, jobs=1):
png_quality=0,
jb2lossy=False,
)
ctx.set_options(options)
with TemporaryDirectory() as td:
context = PDFContext(options, td, infile, None)
tmpout = Path(td) / 'out.pdf'
optimize(infile, tmpout, log, ctx)
optimize(infile, tmpout, context)
copy(fspath(tmpout), fspath(outfile))
tests/test_multiprocessing.py → tests/_test_multiprocessing.py
View File
tests/test_main.py
+1 -1
View File
@@ -28,7 +28,7 @@ import pytest
from PIL import Image
from ocrmypdf.exceptions import ExitCode, MissingDependencyError
from ocrmypdf.exec import ghostscript, qpdf, tesseract, unpaper
from ocrmypdf.exec import ghostscript, qpdf, tesseract
from ocrmypdf.leptonica import Pix
from ocrmypdf.pdfa import file_claims_pdfa
from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
tests/test_metadata.py
+4 -12
View File
@@ -28,7 +28,7 @@ from unittest.mock import MagicMock, patch
import pytest
import pikepdf
from ocrmypdf._jobcontext import JobContext
from ocrmypdf._jobcontext import PDFContext
from ocrmypdf.exceptions import ExitCode
from ocrmypdf.pdfa import SRGB_ICC_PROFILE, file_claims_pdfa, generate_pdfa_ps
from pikepdf.models.metadata import decode_pdf_date
@@ -330,23 +330,15 @@ def test_prevent_gs_invalid_xml(resources, outdir):
from ocrmypdf.pdfinfo import PdfInfo
generate_pdfa_ps(outdir / 'pdfa.ps')
input_files = [str(outdir / 'layers.rendered.pdf'), str(outdir / 'pdfa.ps')]
copyfile(resources / 'enron1.pdf', outdir / 'layers.rendered.pdf')
log = logging.getLogger()
context = JobContext()
options = parser.parse_args(
args=['-j', '1', '--output-type', 'pdfa-2', 'a.pdf', 'b.pdf']
)
context.options = options
context.pdfinfo = PdfInfo(resources / 'enron1.pdf')
pdfinfo = PdfInfo(resources / 'enron1.pdf')
context = PDFContext(options, outdir, resources / 'enron1.pdf', pdfinfo)
convert_to_pdfa(
input_files_groups=input_files,
output_file=outdir / 'pdfa.pdf',
log=log,
context=context,
)
convert_to_pdfa(str(outdir / 'layers.rendered.pdf'), str(outdir / 'pdfa.ps'), context)
with open(outdir / 'pdfa.pdf', 'rb') as f:
with mmap.mmap(