diff --git a/ocrmypdf/__main__.py b/ocrmypdf/__main__.py index 2b05f230..c83508b7 100755 --- a/ocrmypdf/__main__.py +++ b/ocrmypdf/__main__.py @@ -734,7 +734,7 @@ def run_pipeline(): 180: 's', 270: 'w'} orientations = [] for n, page in enumerate(pdfinfo): - angle = pdfinfo[n].get('rotated', 0) + angle = pdfinfo[n].rotation or 0 if angle != 0: orientations.append('{0}{1}'.format( n + 1, diff --git a/ocrmypdf/exec/ghostscript.py b/ocrmypdf/exec/ghostscript.py index f01707c6..d3399710 100644 --- a/ocrmypdf/exec/ghostscript.py +++ b/ocrmypdf/exec/ghostscript.py @@ -9,6 +9,7 @@ import re import sys from . import get_program from ..exceptions import SubprocessOutputError +from PIL import Image @lru_cache(maxsize=1) @@ -34,7 +35,28 @@ def _gs_error_reported(stream): def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log, - pageno=1): + pageno=1, page_dpi=None): + """ + Rasterize one page of a PDF at resolution (xres, yres) in canvas units. + + The image is sized to match the integer pixels dimensions implied by + (xres, yres) even if those numbers are noninteger. The image's DPI will + be overridden with the values in page_dpi. + + :param input_file: + :param output_file: + :param xres: resolution at which to rasterize page + :param yres: + :param raster_device: + :param log: + :param pageno: page number to rasterize + :param page_dpi: resolution tuple (x, y) overriding output image DPI + :return: + """ + res = xres, yres + int_res = round(xres), round(yres) + if not page_dpi: + page_dpi = res with NamedTemporaryFile(delete=True) as tmp: args_gs = [ get_program('gs'), @@ -46,7 +68,7 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log, '-dFirstPage=%i' % pageno, '-dLastPage=%i' % pageno, '-o', tmp.name, - '-r{0}x{1}'.format(str(round(xres)), str(round(yres))), + '-r{0}x{1}'.format(str(int_res[0]), str(int_res[1])), input_file ] @@ -57,14 +79,29 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log, else: log.debug(p.stdout) - if p.returncode == 0: - copy(tmp.name, output_file) - else: + if p.returncode != 0: log.error('Ghostscript rasterizing failed') raise SubprocessOutputError() + # Ghostscript only accepts integers for output resolution + # if the resolution happens to be fractional, then the discrepancy + # would change the size of the output page, especially if the DPI + # is quite low. Resize the image to the expected size + tmp.seek(0) + with Image.open(tmp) as im: + expected_size = round(im.size[0] / int_res[0] * res[0]), \ + round(im.size[1] / int_res[1] * res[1]) + if expected_size != im.size or page_dpi != (xres, yres): + log.debug( + "Ghostscript: resize output image {} -> {}".format( + im.size, expected_size)) + im.resize(expected_size).save(output_file, dpi=page_dpi) + else: + copy(tmp.name, output_file) -def generate_pdfa(pdf_pages, output_file, compression, log, threads=1): + +def generate_pdfa(pdf_pages, output_file, compression, log, + threads=1, pdf_version='1.5'): compression_args = [] if compression == 'jpeg': compression_args = [ @@ -92,7 +129,8 @@ def generate_pdfa(pdf_pages, output_file, compression, log, threads=1): "-dQUIET", "-dBATCH", "-dNOPAUSE", - '-dNumRenderingThreads=' + str(threads), + "-dCompatibilityLevel=" + str(pdf_version), + "-dNumRenderingThreads=" + str(threads), "-sDEVICE=pdfwrite", "-dAutoRotatePages=/None", "-sColorConversionStrategy=/RGB", diff --git a/ocrmypdf/exec/qpdf.py b/ocrmypdf/exec/qpdf.py index 6f2f3297..2ba8a5b6 100644 --- a/ocrmypdf/exec/qpdf.py +++ b/ocrmypdf/exec/qpdf.py @@ -120,13 +120,18 @@ def split_pages(input_file, work_folder, npages): run(args_qpdf, check=True) -def merge(input_files, output_file): +def merge(input_files, output_file, min_version=None): """Merge the list of input files (all filenames) into the output file. The input files may contain one or more pages. """ + version_arg = ['--min-version={}'.format(min_version)] \ + if min_version else [] + args_qpdf = [ - get_program('qpdf'), input_files[0], '--pages' + get_program('qpdf') + ] + version_arg + [ + input_files[0], '--pages' ] + input_files + ['--', output_file] run(args_qpdf, check=True) diff --git a/ocrmypdf/helpers.py b/ocrmypdf/helpers.py index 655debba..ea39f67d 100644 --- a/ocrmypdf/helpers.py +++ b/ocrmypdf/helpers.py @@ -3,7 +3,7 @@ from functools import partial from collections.abc import Iterable -from contextlib import suppress +from contextlib import suppress, contextmanager import sys import os @@ -77,3 +77,12 @@ def is_file_writable(test_file): with suppress(OSError): os.unlink(test_file) return True + + +@contextmanager +def universal_open(p, *args, **kwargs): + "Work around Python 3.5's inability to open(pathlib.Path())" + try: + yield p.open(*args, **kwargs) + except AttributeError: + yield open(p, *args, **kwargs) \ No newline at end of file diff --git a/ocrmypdf/pageinfo.py b/ocrmypdf/pageinfo.py index b421de1c..1b4cd69f 100644 --- a/ocrmypdf/pageinfo.py +++ b/ocrmypdf/pageinfo.py @@ -8,8 +8,12 @@ import re import sys import PyPDF2 as pypdf from collections import namedtuple +import warnings +warnings.warn("ocrmypdf.pageinfo is deprecated'; use ocrmypdf.pdfinfo", + DeprecationWarning) + matrix_mult = pypdf.pdf.utils.matrixMultiply FRIENDLY_COLORSPACE = { diff --git a/ocrmypdf/pdfinfo.py b/ocrmypdf/pdfinfo.py new file mode 100644 index 00000000..f0f4a460 --- /dev/null +++ b/ocrmypdf/pdfinfo.py @@ -0,0 +1,729 @@ +#!/usr/bin/env python3 +# © 2015 James R. Barlow: github.com/jbarlow83 + +from subprocess import Popen, PIPE +from decimal import Decimal +from math import hypot, isclose +import re +import sys +import PyPDF2 as pypdf +from collections import namedtuple +from collections.abc import MutableMapping, Mapping +import warnings +from pathlib import Path +from enum import Enum +from .helpers import universal_open + + +matrix_mult = pypdf.pdf.utils.matrixMultiply + +Colorspace = Enum('Colorspace', + 'gray rgb cmyk lab icc index sep devn pattern jpeg2000') + +Encoding = Enum('Encoding', + 'ccitt jpeg jpeg2000 jbig2 asciihex ascii85 lzw flate ' + \ + 'runlength') + + +FRIENDLY_COLORSPACE = { + '/DeviceGray': Colorspace.gray, + '/CalGray': Colorspace.gray, + '/DeviceRGB': Colorspace.rgb, + '/CalRGB': Colorspace.rgb, + '/DeviceCMYK': Colorspace.cmyk, + '/Lab': Colorspace.lab, + '/ICCBased': Colorspace.icc, + '/Indexed': Colorspace.index, + '/Separation': Colorspace.sep, + '/DeviceN': Colorspace.devn, + '/Pattern': Colorspace.pattern, + '/G': Colorspace.gray, # Abbreviations permitted in inline images + '/RGB': Colorspace.rgb, + '/CMYK': Colorspace.cmyk, + '/I': Colorspace.index, +} + +FRIENDLY_ENCODING = { + '/CCITTFaxDecode': Encoding.ccitt, + '/DCTDecode': Encoding.jpeg, + '/JPXDecode': Encoding.jpeg2000, + '/JBIG2Decode': Encoding.jbig2, + '/CCF': Encoding.ccitt, # Abbreviations permitted in inline images + '/DCT': Encoding.jpeg, + '/AHx': Encoding.asciihex, + '/A85': Encoding.ascii85, + '/LZW': Encoding.lzw, + '/Fl': Encoding.flate, + '/RL': Encoding.runlength +} + +FRIENDLY_COMP = { + Colorspace.gray: 1, + Colorspace.rgb: 3, + Colorspace.cmyk: 4, + Colorspace.lab: 3, + Colorspace.index: 1 +} + + +UNIT_SQUARE = (1.0, 0.0, 0.0, 1.0, 0.0, 0.0) + + +def _matrix_from_shorthand(shorthand): + """Convert from PDF matrix shorthand to full matrix + + PDF 1.7 spec defines a shorthand for describing the entries of a matrix + since the last column is always (0, 0, 1). + """ + + a, b, c, d, e, f = map(float, shorthand) + return ((a, b, 0), + (c, d, 0), + (e, f, 1)) + + +def _shorthand_from_matrix(matrix): + """Convert from transformation matrix to PDF shorthand.""" + a, b = matrix[0][0], matrix[0][1] + c, d = matrix[1][0], matrix[1][1] + e, f = matrix[2][0], matrix[2][1] + return tuple(map(float, (a, b, c, d, e, f))) + + +def _is_unit_square(shorthand): + values = map(float, shorthand) + pairwise = zip(values, UNIT_SQUARE) + return all([isclose(a, b, rel_tol=1e-3) for a, b in pairwise]) + +XobjectSettings = namedtuple('XobjectSettings', + ['name', 'shorthand', 'stack_depth']) + +InlineSettings = namedtuple('InlineSettings', + ['settings', 'shorthand', 'stack_depth']) + +ContentsInfo = namedtuple('ContentsInfo', ['xobject_settings', 'inline_images']) + + +def _normalize_stack(operations): + """Fix runs of qQ's in the stack + + For some reason PyPDF2 converts runs of qqq, QQ, QQQq, etc. into single + operations. Break this silliness up and issue each stack operation + individually so we don't lose count. + + """ + for operands, command in operations: + if re.match(br'Q*q+$', command): # Zero or more Q, one or more q + for char in command: # Split into individual bytes + yield ([], bytes([char])) # Yield individual bytes + else: + yield (operands, command) + + +def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE): + """Interpret the PDF content stream + + The stack represents the state of the PDF graphics stack. We are only + interested in the current transformation matrix (CTM) so we only track + this object; a full implementation would need to track many other items. + + The CTM is initialized to the mapping from user space to device space. + PDF units are 1/72". In a PDF viewer or printer this matrix is initialized + to the transformation to device space. For example if set to + (1/72, 0, 0, 1/72, 0, 0) then all units would be calculated in inches. + + Images are always considered to be (0, 0) -> (1, 1). Before drawing an + image there should be a 'cm' that sets up an image coordinate system + where drawing from (0, 0) -> (1, 1) will draw on the desired area of the + page. + + PDF units suit our needs so we initialize ctm to the identity matrix. + + PyPDF2 replaces inline images with a fake "INLINE IMAGE" operator. + + """ + + operations = contentstream.operations + stack = [] + ctm = _matrix_from_shorthand(initial_shorthand) + xobject_settings = [] + inline_images = [] + + for n, op in enumerate(_normalize_stack(operations)): + operands, command = op + if command == b'q': + stack.append(ctm) + if len(stack) > 32: + raise RuntimeError( + "PDF graphics stack overflow, command %i" % n) + elif command == b'Q': + try: + ctm = stack.pop() + except IndexError: + raise RuntimeError( + "PDF graphics stack underflow, command %i" % n) + elif command == b'cm': + ctm = matrix_mult( + _matrix_from_shorthand(operands), ctm) + elif command == b'Do': + image_name = operands[0] + settings = XobjectSettings( + name=image_name, shorthand=_shorthand_from_matrix(ctm), + stack_depth=len(stack)) + xobject_settings.append(settings) + elif command == b'INLINE IMAGE': + settings = operands['settings'] + inline = InlineSettings( + settings=settings, shorthand=_shorthand_from_matrix(ctm), + stack_depth=len(stack)) + inline_images.append(inline) + + return ContentsInfo( + xobject_settings=xobject_settings, + inline_images=inline_images) + + +def _get_dpi(ctm_shorthand, image_size): + """Given the transformation matrix and image size, find the image DPI. + + PDFs do not include image resolution information within image data. + Instead, the PDF page content stream describes the location where the + image will be rasterized, and the effective resolution is the ratio of the + pixel size to raster target size. + + Normally a scanned PDF has the paper size set appropriately but this is + not guaranteed. The most common case is a cropped image will change the + page size (/CropBox) without altering the page content stream. That means + it is not sufficient to assume that the image fills the page, even though + that is the most common case. + + A PDF image may be scaled (always), cropped, translated, rotated in place + to an arbitrary angle (rarely) and skewed. Only equal area mappings can + be expressed, that is, it is not necessary to consider distortions where + the effective DPI varies with position. + + To determine the image scale, transform an offset axis vector v0 (0, 0), + width-axis vector v0 (1, 0), height-axis vector vh (0, 1) with the matrix, + which gives the dimensions of the image in PDF units. From there we can + compare to actual image dimensions. PDF uses + row vector * matrix_tranposed unlike the traditional + matrix * column vector. + + The offset, width and height vectors can be combined in a matrix and + multiplied by the transform matrix. Then we want to calculated + magnitude(width_vector - offset_vector) + and + magnitude(height_vector - offset_vector) + + When the above is worked out algebraically, the effect of translation + cancels out, and the vector magnitudes become functions of the nonzero + transformation matrix indices. The results of the derivation are used + in this code. + + pdfimages -list does calculate the DPI in some way that is not completely + naive, but it does not get the DPI of rotated images right, so cannot be + used anymore to validate this. Photoshop works, or using Acrobat to + rotate the image back to normal. + + It does not matter if the image is partially cropped, or even out of the + /MediaBox. + + """ + + a, b, c, d, _, _ = ctm_shorthand + + # Calculate the width and height of the image in PDF units + image_drawn_width = hypot(a, b) + image_drawn_height = hypot(c, d) + + # The scale of the image is pixels per unit of default user space (1/72") + scale_w = image_size[0] / image_drawn_width + scale_h = image_size[1] / image_drawn_height + + # DPI = scale * 72 + dpi_w = scale_w * 72.0 + dpi_h = scale_h * 72.0 + + return dpi_w, dpi_h + + +class ImageInfo: + DPI_PREC = Decimal('1.000') + + def __init__(self, *, name='', pdfimage=None, inline=None, + shorthand=None): + + self._name = name + self._shorthand = shorthand + if inline: + # Fixme does not work for inline images with non abbreviated + # fields + self._origin = 'inline' + self._width = inline.settings['/W'] + self._height = inline.settings['/H'] + self._bpc = inline.settings.get('/BPC', 8) + try: + self._color = FRIENDLY_COLORSPACE[inline.settings['/CS']] + except Exception: + self._color = '-' + self._comp = FRIENDLY_COMP.get(self._color, '?') + if '/F' in inline.settings: + filter_ = inline.settings['/F'] + if isinstance(filter_, pypdf.generic.ArrayObject): + filter_ = filter_[0] + self._enc = FRIENDLY_ENCODING.get(filter_, 'image') + else: + self._enc = 'image' + elif pdfimage: + self._origin = 'xobject' + self._width = pdfimage['/Width'] + self._height = pdfimage['/Height'] + if '/BitsPerComponent' in pdfimage: + self._bpc = pdfimage['/BitsPerComponent'] + else: + self._bpc = 8 + + # Fixme: this is incorrectly treats explicit masks as stencil masks, + # but good enough for now. Explicit masks have /ImageMask true but are + # never called for in content stream, instead are drawn as a /Mask on + # other images. For our purposes finding out the details of /Mask + # will seldom matter. + if '/ImageMask' in pdfimage: + self._type = 'stencil' if pdfimage['/ImageMask'].value \ + else 'image' + else: + self._type = 'image' + if '/Filter' in pdfimage: + filter_ = pdfimage['/Filter'] + if isinstance(filter_, pypdf.generic.ArrayObject): + filter_ = filter_[0] + self._enc = FRIENDLY_ENCODING.get(filter_, 'image') + else: + self._enc = 'image' + if '/ColorSpace' in pdfimage: + cs = pdfimage['/ColorSpace'] + if isinstance(cs, pypdf.generic.ArrayObject): + cs = cs[0] + self._color = FRIENDLY_COLORSPACE.get(cs, '-') + else: + self._color = FRIENDLY_COLORSPACE[Colorspace.jpeg2000] \ + if self._enc == Encoding.jpeg2000 else '?' + + self._comp = FRIENDLY_COMP.get(self._color, '?') + + # Bit of a hack... infer grayscale if component count is uncertain + # but encoding must be monochrome. This happens if a monochrome image + # has an ICC profile attached. Better solution would be to examine + # the ICC profile. + if self._comp == '?' and self._enc in (Encoding.ccitt, 'jbig2'): + self._comp = FRIENDLY_COMP[Colorspace.gray] + + @property + def name(self): + return self._name + + @property + def type_(self): + return self._type + + @property + def width(self): + return self._width + + @property + def height(self): + return self._height + + @property + def bpc(self): + return self._bpc + + @property + def color(self): + return self._color + + @property + def comp(self): + return self._comp + + @property + def enc(self): + return self._enc + + @property + def xres(self): + return _get_dpi(self._shorthand, (self._width, self._height))[0] + + @property + def yres(self): + return _get_dpi(self._shorthand, (self._width, self._height))[1] + + def __getitem__(self, item): + warnings.warn("ImageInfo.__getitem__", DeprecationWarning) + if item in ('name', 'width', 'height', 'bpc', 'color', 'comp', 'enc'): + return getattr(self, item) + elif item == 'dpi_w': + return Decimal(self.xres).quantize(self.DPI_PREC) + elif item == 'dpi_h': + return Decimal(self.yres).quantize(self.DPI_PREC) + elif item == 'dpi': + return Decimal(self.xres * self.yres).sqrt().quantize( + self.DPI_PREC) + else: + raise KeyError(item) + + def __repr__(self): + class_locals = {attr: getattr(self, attr, None) for attr in dir(self) + if not attr.startswith('_')} + return ( + "").format(**class_locals) + + +def _find_inline_images(contentsinfo): + "Find inline images in the contentstream" + + for n, inline in enumerate(contentsinfo.inline_images): + yield ImageInfo(name='inline-%02d' % n, shorthand=inline.shorthand, + inline=inline) + + +def _image_xobjects(container): + """Search for all XObject-based images in the container + + Usually the container is a page, but it could also be a Form XObject + that contains images. Filter out the Form XObjects which are dealt with + elsewhere. + + Generate a sequence of tuples (image, xobj container), where container, + where xobj is the name of the object and image is the object itself, + since the object does not know its own name. + + """ + + if '/Resources' not in container: + return + resources = container['/Resources'] + if '/XObject' not in resources: + return + for xobj in resources['/XObject']: + candidate = resources['/XObject'][xobj] + if candidate['/Subtype'] == '/Image': + pdfimage = candidate + yield (pdfimage, xobj) + + +def _find_regular_images(container, contentsinfo): + """Find images stored in the container's /Resources /XObject + + Usually the container is a page, but it could also be a Form XObject + that contains images. + + Generates images with their DPI at time of drawing. + + """ + + for pdfimage, xobj in _image_xobjects(container): + + # For each image that is drawn on this, check if we drawing the + # current image - yes this is O(n^2), but n == 1 almost always + for draw in contentsinfo.xobject_settings: + if draw.name != xobj: + continue + + if draw.stack_depth == 0 and _is_unit_square(draw.shorthand): + # At least one PDF in the wild (and test suite) draws an image + # when the graphics stack depth is 0, meaning that the image + # gets drawn into a square of 1x1 PDF units (or 1/72", + # or 0.35 mm). The equivalent DPI will be >100,000. Exclude + # these from our DPI calculation for the page. + continue + + yield ImageInfo(name=draw.name, pdfimage=pdfimage, shorthand= + draw.shorthand) + + +def _find_form_xobject_images(pdf, container, contentsinfo): + """Find any images that are in Form XObjects in the container + + The container may be a page, or a parent Form XObject. + + """ + if '/Resources' not in container: + return + resources = container['/Resources'] + if '/XObject' not in resources: + return + for xobj in resources['/XObject']: + candidate = resources['/XObject'][xobj] + if candidate['/Subtype'] != '/Form': + continue + + form_xobject = candidate + for settings in contentsinfo.xobject_settings: + if settings.name != xobj: + continue + + # Find images once for each time this Form XObject is drawn. + # This could be optimized to cache the multiple drawing events + # but in practice both Form XObjects and multiple drawing of the + # same object are both very rare. + ctm_shorthand = settings.shorthand + yield from _find_images( + pdf=pdf, container=form_xobject, shorthand=ctm_shorthand) + + +def _find_images(*, pdf, container, shorthand=None): + """Find all individual instances of images drawn in the container + + Usually the container is a page, but it may also be a Form XObject. + + On a typical page images are stored inline or as regular images + in an XObject. + + Form XObjects may include inline images, XObject images, + and recursively, other Form XObjects; and also vector drawing commands. + + Every instance of an image being drawn somewhere is flattened and + treated as a unique image, since if the same image is drawn multiple times + on one page it may be drawn at differing resolutions, and our objective + is to find the resolution at which the page can be rastered without + downsampling. + + """ + + if container.get('/Type') == '/Page' and '/Contents' in container: + # For a /Page the content stream is attached to the page's /Contents + page = container + contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf) + initial_shorthand = shorthand or UNIT_SQUARE + elif container.get('/Type') == '/XObject' and \ + container['/Subtype'] == '/Form': + # For a Form XObject that content stream is attached to the XObject + contentstream = pypdf.pdf.ContentStream(container, pdf) + + # Set the CTM to the state it was when the "Do" operator was + # encountered that is drawing this instance of the Form XObject + ctm = _matrix_from_shorthand(shorthand or UNIT_SQUARE) + + # A Form XObject may provide its own matrix to map form space into + # user space. Get this if one exists + form_matrix = _matrix_from_shorthand( + container.get('/Matrix', UNIT_SQUARE)) + + # Concatenate form matrix with CTM to ensure CTM is correct for + # drawing this instance of the XObject + ctm = matrix_mult(form_matrix, ctm) + initial_shorthand = _shorthand_from_matrix(ctm) + else: + return + + contentsinfo = _interpret_contents(contentstream, initial_shorthand) + + yield from _find_inline_images(contentsinfo) + yield from _find_regular_images(container, contentsinfo) + yield from _find_form_xobject_images(pdf, container, contentsinfo) + + +def _page_has_text(pdf, page): + if not '/Contents' in page: + return False + + # Simple test + text = page.extractText() + if text.strip() != '': + return True + + # More nuanced test to deal with quirks of Tesseract PDF generation + # Check if there's a Glyphless font + try: + font = page['/Resources']['/Font'] + except KeyError: + pass + else: + font_objects = list(font.keys()) + for font_object in font_objects: + basefont = font[font_object]['/BaseFont'] + if basefont.endswith('GlyphLessFont'): + return True + + return False + + +def _pdf_get_pageinfo(pdf, pageno: int): + pageinfo = {} + pageinfo['pageno'] = pageno + pageinfo['images'] = [] + + if isinstance(pdf, Path): + pdf = pypdf.PdfFileReader(str(pdf)) + elif isinstance(pdf, str): + pdf = pypdf.PdfFileReader(pdf) + + page = pdf.pages[pageno] + + pageinfo['has_text'] = _page_has_text(pdf, page) + + width_pt = page.mediaBox.getWidth() + height_pt = page.mediaBox.getHeight() + + userunit = page.get('/UserUnit', Decimal(1.0)) + pageinfo['userunit'] = userunit + pageinfo['width_inches'] = width_pt * userunit / Decimal(72.0) + pageinfo['height_inches'] = height_pt * userunit / Decimal(72.0) + + try: + pageinfo['rotate'] = int(page['/Rotate']) + except KeyError: + pageinfo['rotate'] = 0 + + userunit_shorthand = (userunit, 0, 0, userunit, 0, 0) + pageinfo['images'] = [im for im in + _find_images(pdf=pdf, container=page, + shorthand=userunit_shorthand)] + if pageinfo['images']: + xres = max(image['dpi_w'] for image in pageinfo['images']) + yres = max(image['dpi_h'] for image in pageinfo['images']) + pageinfo['xres'], pageinfo['yres'] = xres, yres + pageinfo['width_pixels'] = \ + int(round(xres * pageinfo['width_inches'])) + pageinfo['height_pixels'] = \ + int(round(yres * pageinfo['height_inches'])) + + return pageinfo + + +def _pdf_get_all_pageinfo(infile): + with universal_open(infile, 'rb') as f: + pdf = pypdf.PdfFileReader(f) + return [PageInfo(pdf, n) for n in range(pdf.numPages)] + + +class PageInfo: + def __init__(self, pdf, pageno): + self._pageno = pageno + self._pageinfo = _pdf_get_pageinfo(pdf, pageno) + + @property + def pageno(self): + return self._pageno + + @property + def has_text(self): + return self._pageinfo['has_text'] + + @property + def width_inches(self): + return self._pageinfo['width_inches'] + + @property + def height_inches(self): + return self._pageinfo['height_inches'] + + @property + def width_pixels(self): + return int(round(self.width_inches * self.xres)) + + @property + def height_pixels(self): + return int(round(self.height_inches * self.yres)) + + @property + def rotation(self): + return self._pageinfo.get('rotate', None) + + @rotation.setter + def rotation(self, value): + if value in (0, 90, 180, 270, 360, -90, -180, -270): + self._pageinfo['rotate'] = value + else: + raise ValueError("rotation must be a cardinal angle") + + @property + def images(self): + return self._pageinfo['images'] + + @property + def xres(self): + return self._pageinfo.get('xres', None) + + @property + def yres(self): + return self._pageinfo.get('yres', None) + + @property + def userunit(self): + return self._pageinfo.get('userunit', None) + + @property + def min_version(self): + if self.userunit is not None: + return '1.6' + else: + return '1.5' + + @property + def images(self): + return self._pageinfo['images'] + + def __repr__(self): + return ( + '').format( + self.pageno, self.width_inches, self.height_inches, + self.rotation, + self.xres, self.yres, self.has_text + ) + + +class PdfInfo: + """Get summary information about a PDF + + """ + def __init__(self, infile): + self._infile = infile + self._pages = _pdf_get_all_pageinfo(infile) + + @property + def pages(self): + return self._pages + + @property + def min_version(self): + # The minimum PDF is the maximum version that any particular page needs + return max(page.min_version for page in self.pages) + + @property + def has_userunit(self): + return any(page.userunit != 1.0 for page in self.pages) + + def __getitem__(self, item): + return self._pages[item] + + def __len__(self): + return len(self._pages) + + def __repr__(self): + return "".format(len(self)) + + # def __getstate__(self): + # state = {'_infile': self._infile} + # return state + # + # def __setstate__(self, state): + # self._infile = state['_infile'] + # self._pages = _pdf_get_all_pageinfo(self._infile) + + +def main(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('infile') + args = parser.parse_args() + info = _pdf_get_all_pageinfo(args.infile) + from pprint import pprint + pprint(info) + + +if __name__ == '__main__': + main() diff --git a/ocrmypdf/pipeline.py b/ocrmypdf/pipeline.py index b95e76f1..6031fb67 100644 --- a/ocrmypdf/pipeline.py +++ b/ocrmypdf/pipeline.py @@ -22,7 +22,7 @@ from PIL import Image from ruffus import formatter, regex, Pipeline, suffix from .hocrtransform import HocrTransform -from .pageinfo import pdf_get_all_pageinfo +from .pdfinfo import PdfInfo, Encoding, Colorspace from .pdfa import generate_pdfa_ps, file_claims_pdfa from .helpers import re_symlink, is_iterable_notstr, page_number from .exec import ghostscript, tesseract, qpdf @@ -50,7 +50,10 @@ class JobContext: """ def __init__(self): - self.pdfinfo = [] + self.pdfinfo = None + + def generate_pdfinfo(self, infile): + self.pdfinfo = PdfInfo(infile) def get_pdfinfo(self): "What we know about the input PDF" @@ -72,8 +75,8 @@ class JobContext: self.work_folder = work_folder -from multiprocessing.managers import BaseManager -class JobContextManager(BaseManager): +from multiprocessing.managers import SyncManager +class JobContextManager(SyncManager): pass @@ -181,9 +184,19 @@ def repair_pdf( output_file, log, context): - + options = context.get_options() qpdf.repair(input_file, output_file, log) - pdfinfo = pdf_get_all_pageinfo(output_file) + pdfinfo = PdfInfo(output_file) + + if pdfinfo.has_userunit and options.output_type == 'pdfa': + log.error("This input file uses a PDF feature that is not supported " + "by Ghostscript, so you cannot use --output-type=pdfa for this " + "file. (Specifically, it uses the PDF-1.6 /UserUnit feature to " + "support very large or small page sizes, and Ghostscript cannot " + "output these files.) Use --output-type=pdf instead." + ) + raise InputFileError() + context.set_pdfinfo(pdfinfo) log.debug(pdfinfo) @@ -196,23 +209,34 @@ def get_pageinfo(input_file, context): def get_page_dpi(pageinfo, options): "Get the DPI when nonsquare DPI is tolerable" - xres = max(pageinfo.get('xres', VECTOR_PAGE_DPI), options.oversample or 0) - yres = max(pageinfo.get('yres', VECTOR_PAGE_DPI), options.oversample or 0) + xres = max(pageinfo.xres or VECTOR_PAGE_DPI, options.oversample or 0) + yres = max(pageinfo.yres or VECTOR_PAGE_DPI, options.oversample or 0) return (float(xres), float(yres)) def get_page_square_dpi(pageinfo, options): - "Get the DPI when we require xres == yres" + "Get the DPI when we require xres == yres, scaled to physical units" + xres = pageinfo.xres or 0 + yres = pageinfo.yres or 0 + userunit = pageinfo.userunit or 1 return float(max( - pageinfo.get('xres', VECTOR_PAGE_DPI), - pageinfo.get('yres', VECTOR_PAGE_DPI), + (xres * userunit) or VECTOR_PAGE_DPI, + (yres * userunit) or VECTOR_PAGE_DPI, + options.oversample or 0)) + + +def get_canvas_square_dpi(pageinfo, options): + """Get the DPI when we require xres == yres, in Postscript units""" + return float(max( + (pageinfo.xres) or VECTOR_PAGE_DPI, + (pageinfo.yres) or VECTOR_PAGE_DPI, options.oversample or 0)) def is_ocr_required(pageinfo, log, options): - page = pageinfo['pageno'] + 1 + page = pageinfo.pageno + 1 ocr_required = True - if not pageinfo['images']: + if not pageinfo.images: if options.force_ocr and options.oversample: # The user really wants to reprocess this file log.info( @@ -234,7 +258,7 @@ def is_ocr_required(pageinfo, log, options): "skipping all processing on this page".format(page)) ocr_required = False - elif pageinfo['has_text']: + elif pageinfo.has_text: msg = "{0:4d}: page already has text! – {1}" if not options.force_ocr and not options.skip_text: @@ -250,8 +274,8 @@ def is_ocr_required(pageinfo, log, options): "skipping all processing on this page")) ocr_required = False - if ocr_required and options.skip_big and pageinfo['images']: - pixel_count = pageinfo['width_pixels'] * pageinfo['height_pixels'] + if ocr_required and options.skip_big and pageinfo.images: + pixel_count = pageinfo.width_pixels * pageinfo.height_pixels if pixel_count > (options.skip_big * 1000000): ocr_required = False log.warning( @@ -309,13 +333,14 @@ def rasterize_preview( output_file, log, context): + pageinfo = get_pageinfo(input_file, context) + options = context.get_options() + canvas_dpi = get_canvas_square_dpi(pageinfo, options) / 2 + page_dpi = get_page_square_dpi(pageinfo, options) / 2 + ghostscript.rasterize_pdf( - input_file=input_file, - output_file=output_file, - xres=200, - yres=200, - raster_device='jpeggray', - log=log) + input_file, output_file, xres=canvas_dpi, yres=canvas_dpi, + raster_device='jpeggray', log=log, page_dpi=(page_dpi, page_dpi)) def orient_page( @@ -383,7 +408,7 @@ def orient_page( pageno = int(os.path.basename(page_pdf)[0:6]) - 1 pdfinfo = context.get_pdfinfo() - pdfinfo[pageno]['rotated'] = orient_conf.angle + pdfinfo[pageno].rotation = orient_conf.angle context.set_pdfinfo(pdfinfo) @@ -396,26 +421,28 @@ def rasterize_with_ghostscript( pageinfo = get_pageinfo(input_file, context) device = 'png16m' # 24-bit - if pageinfo['images']: - if all(image['comp'] == 1 for image in pageinfo['images']): - if all(image['bpc'] == 1 for image in pageinfo['images']): + if pageinfo.images: + if all(image.comp == 1 for image in pageinfo.images): + if all(image.bpc == 1 for image in pageinfo.images): device = 'pngmono' - elif all(image['bpc'] > 1 and image['color'] == 'index' - for image in pageinfo['images']): + elif all(image.bpc > 1 and image.color == Colorspace.index + for image in pageinfo.images): device = 'png256' - elif all(image['bpc'] > 1 and image['color'] == 'gray' - for image in pageinfo['images']): + elif all(image.bpc > 1 and image.color == Colorspace.gray + for image in pageinfo.images): device = 'pnggray' log.debug("Rasterize {0} with {1}".format( os.path.basename(input_file), device)) # Produce the page image with square resolution or else deskew and OCR - # will not work properly - dpi = get_page_square_dpi(pageinfo, options) + # will not work properly. + canvas_dpi = get_canvas_square_dpi(pageinfo, options) + page_dpi = get_page_square_dpi(pageinfo, options) + ghostscript.rasterize_pdf( - input_file, output_file, xres=dpi, yres=dpi, raster_device=device, - log=log) + input_file, output_file, xres=canvas_dpi, yres=canvas_dpi, + raster_device=device, log=log, page_dpi=(page_dpi, page_dpi)) def preprocess_remove_background( @@ -430,11 +457,11 @@ def preprocess_remove_background( pageinfo = get_pageinfo(input_file, context) - if any(image['bpc'] > 1 for image in pageinfo['images']): + if any(image.bpc > 1 for image in pageinfo.images): leptonica.remove_background(input_file, output_file) else: log.info("{0:4d}: background removal skipped on mono page".format( - pageinfo['pageno'])) + pageinfo.pageno)) re_symlink(input_file, output_file, log) @@ -475,7 +502,7 @@ def select_ocr_image( infiles, output_file, log, - contenxt): + context): """Select the image we send for OCR. May not be the same as the display image depending on preprocessing.""" @@ -521,8 +548,8 @@ def select_visible_page_image( image = next(ii for ii in infiles if ii.endswith(image_suffix)) pageinfo = get_pageinfo(image, context) - if pageinfo['images'] and \ - all(im['enc'] == 'jpeg' for im in pageinfo['images']): + if pageinfo.images and \ + all(im['enc'] == 'jpeg' for im in pageinfo.images): log.debug('{:4d}: JPEG input -> JPEG output'.format( page_number(image))) # If all images were JPEGs originally, produce a JPEG as output @@ -689,6 +716,12 @@ def combine_layers( pdf_output = pypdf.PdfFileWriter() pdf_output.addPage(page_text) + # If the input was scaled, re-apply the scaling + pageinfo = get_pageinfo(text, context) + if pageinfo.userunit != 1: + page_text[pypdf.generic.NameObject('/UserUnit')] = pageinfo.userunit + pdf_output._header = b'%PDF-1.6' # Hack header to correct version + with open(output_file, "wb") as out: pdf_output.write(out) @@ -832,9 +865,14 @@ def merge_pages_ghostscript( if not f.endswith('.txt')) pdf_pages = sorted(input_files, key=input_file_order) log.debug("Final pages: " + "\n".join(pdf_pages)) + input_pdfinfo = context.get_pdfinfo() ghostscript.generate_pdfa( - pdf_pages, output_file, options.pdfa_image_compression, - log, options.jobs or 1) + pdf_version=input_pdfinfo.min_version, + pdf_pages=pdf_pages, + output_file=output_file, + compression=options.pdfa_image_compression, + log=log, + threads=options.jobs or 1) def merge_pages_qpdf( @@ -876,7 +914,8 @@ def merge_pages_qpdf( pdf_pages[0] = writer_file - qpdf.merge(pdf_pages, output_file) + qpdf.merge(input_files=pdf_pages, output_file=output_file, + min_version=context.get_pdfinfo().min_version) def merge_sidecars( diff --git a/tests/conftest.py b/tests/conftest.py index be0a6d82..319fcbd0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -66,6 +66,18 @@ def spoof(**kwargs): return env +@pytest.fixture +def spoof_tesseract_noop(): + return spoof(tesseract='tesseract_noop.py') + + +@pytest.fixture +def spoof_tesseract_cache(): + if running_in_docker(): + return os.environ.copy() + return spoof(tesseract="tesseract_cache.py") + + @pytest.fixture def resources(): return Path(TESTS_ROOT) / 'resources' @@ -129,7 +141,7 @@ def run_ocrmypdf(input_file, output_file, *args, env=None): @pytest.helpers.register def first_page_dimensions(pdf): - from ocrmypdf import pageinfo - info = pageinfo.pdf_get_all_pageinfo(str(pdf)) + from ocrmypdf import pdfinfo + info = pdfinfo.PdfInfo(pdf) page0 = info[0] - return (page0['width_inches'], page0['height_inches']) + return (page0.width_inches, page0.height_inches) diff --git a/tests/resources/poster.pdf b/tests/resources/poster.pdf new file mode 100644 index 00000000..c62765a8 Binary files /dev/null and b/tests/resources/poster.pdf differ diff --git a/tests/test_main.py b/tests/test_main.py index c463c27a..a5b57f0f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -5,30 +5,20 @@ from subprocess import Popen, PIPE, check_output, check_call, DEVNULL import os import shutil import pytest -from ocrmypdf.pageinfo import pdf_get_all_pageinfo +from ocrmypdf.pdfinfo import PdfInfo, Colorspace, Encoding import PyPDF2 as pypdf from ocrmypdf.exceptions import ExitCode from ocrmypdf import leptonica from ocrmypdf.pdfa import file_claims_pdfa from ocrmypdf.exec import ghostscript import logging +from math import isclose check_ocrmypdf = pytest.helpers.check_ocrmypdf run_ocrmypdf = pytest.helpers.run_ocrmypdf spoof = pytest.helpers.spoof -@pytest.fixture -def spoof_tesseract_noop(): - return spoof(tesseract='tesseract_noop.py') - - -@pytest.fixture -def spoof_tesseract_cache(): - if pytest.helpers.running_in_docker(): - return os.environ.copy() - return spoof(tesseract="tesseract_cache.py") - @pytest.fixture def spoof_tesseract_crash(): @@ -230,10 +220,10 @@ def test_oversample(spoof_tesseract_cache, renderer, resources, outpdf): '-f', '--pdf-renderer', renderer, env=spoof_tesseract_cache) - pdfinfo = pdf_get_all_pageinfo(str(oversampled_pdf)) + pdfinfo = PdfInfo(oversampled_pdf) - print(pdfinfo[0]['xres']) - assert abs(pdfinfo[0]['xres'] - 350) < 1 + print(pdfinfo[0].xres) + assert abs(pdfinfo[0].xres - 350) < 1 def test_repeat_ocr(resources, no_outpdf): @@ -244,8 +234,8 @@ def test_repeat_ocr(resources, no_outpdf): def test_force_ocr(spoof_tesseract_cache, resources, outpdf): out = check_ocrmypdf(resources / 'graph_ocred.pdf', outpdf, '-f', env=spoof_tesseract_cache) - pdfinfo = pdf_get_all_pageinfo(out) - assert pdfinfo[0]['has_text'] + pdfinfo = PdfInfo(out) + assert pdfinfo[0].has_text def test_skip_ocr(spoof_tesseract_cache, resources, outpdf): @@ -358,15 +348,15 @@ def test_autorotate_threshold( def test_ocr_timeout(renderer, resources, outpdf): out = check_ocrmypdf(resources / 'skew.pdf', outpdf, '--tesseract-timeout', '1.0') - pdfinfo = pdf_get_all_pageinfo(str(out)) - assert not pdfinfo[0]['has_text'] + pdfinfo = PdfInfo(out) + assert not pdfinfo[0].has_text def test_skip_big(spoof_tesseract_cache, resources, outpdf): out = check_ocrmypdf(resources / 'enormous.pdf', outpdf, '--skip-big', '10', env=spoof_tesseract_cache) - pdfinfo = pdf_get_all_pageinfo(str(out)) - assert not pdfinfo[0]['has_text'] + pdfinfo = PdfInfo(out) + assert not pdfinfo[0].has_text @pytest.mark.parametrize('renderer', ['hocr', 'tesseract']) @@ -562,18 +552,18 @@ def test_algo4(resources, no_outpdf): def test_non_square_resolution(renderer, spoof_tesseract_cache, resources, outpdf): # Confirm input image is non-square resolution - in_pageinfo = pdf_get_all_pageinfo(str(resources / 'aspect.pdf')) - assert in_pageinfo[0]['xres'] != in_pageinfo[0]['yres'] + in_pageinfo = PdfInfo(resources / 'aspect.pdf') + assert in_pageinfo[0].xres != in_pageinfo[0].yres check_ocrmypdf( resources / 'aspect.pdf', outpdf, '--pdf-renderer', renderer, env=spoof_tesseract_cache) - out_pageinfo = pdf_get_all_pageinfo(str(outpdf)) + out_pageinfo = PdfInfo(outpdf) # Confirm resolution was kept the same - assert in_pageinfo[0]['xres'] == out_pageinfo[0]['xres'] - assert in_pageinfo[0]['yres'] == out_pageinfo[0]['yres'] + assert in_pageinfo[0].xres == out_pageinfo[0].xres + assert in_pageinfo[0].yres == out_pageinfo[0].yres @pytest.mark.parametrize('renderer', [ @@ -585,8 +575,8 @@ def test_convert_to_square_resolution(renderer, spoof_tesseract_cache, from math import isclose # Confirm input image is non-square resolution - in_pageinfo = pdf_get_all_pageinfo(str(resources / 'aspect.pdf')) - assert in_pageinfo[0]['xres'] != in_pageinfo[0]['yres'] + in_pageinfo = PdfInfo(resources / 'aspect.pdf') + assert in_pageinfo[0].xres != in_pageinfo[0].yres # --force-ocr requires means forced conversion to square resolution check_ocrmypdf( @@ -594,25 +584,25 @@ def test_convert_to_square_resolution(renderer, spoof_tesseract_cache, '--force-ocr', '--pdf-renderer', renderer, env=spoof_tesseract_cache) - out_pageinfo = pdf_get_all_pageinfo(str(outpdf)) + out_pageinfo = PdfInfo(outpdf) in_p0, out_p0 = in_pageinfo[0], out_pageinfo[0] # Resolution show now be equal - assert out_p0['xres'] == out_p0['yres'] + assert out_p0.xres == out_p0.yres # Page size should match input page size - assert isclose(in_p0['width_inches'], - out_p0['width_inches']) - assert isclose(in_p0['height_inches'], - out_p0['height_inches']) + assert isclose(in_p0.width_inches, + out_p0.width_inches) + assert isclose(in_p0.height_inches, + out_p0.height_inches) # Because we rasterized the page to produce a new image, it should occupy # the entire page - out_im_w = out_p0['images'][0]['width'] / out_p0['images'][0]['dpi_w'] - out_im_h = out_p0['images'][0]['height'] / out_p0['images'][0]['dpi_h'] - assert isclose(out_p0['width_inches'], out_im_w) - assert isclose(out_p0['height_inches'], out_im_h) + out_im_w = out_p0.images[0]['width'] / out_p0.images[0]['dpi_w'] + out_im_h = out_p0.images[0]['height'] / out_p0.images[0]['dpi_h'] + assert isclose(out_p0.width_inches, out_im_w) + assert isclose(out_p0.height_inches, out_im_h) def test_image_to_pdf(spoof_tesseract_noop, resources, outpdf): @@ -628,8 +618,8 @@ def test_jbig2_passthrough(spoof_tesseract_cache, resources, outpdf): '--pdf-renderer', 'hocr', env=spoof_tesseract_cache) - out_pageinfo = pdf_get_all_pageinfo(str(out)) - assert out_pageinfo[0]['images'][0]['enc'] == 'jbig2' + out_pageinfo = PdfInfo(out) + assert out_pageinfo[0].images[0].enc == Encoding.jbig2 def test_stdin(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf): @@ -702,27 +692,27 @@ def test_rotated_skew_timeout(resources, outpdf): """ input_file = str(resources / 'rotated_skew.pdf') - in_pageinfo = pdf_get_all_pageinfo(input_file)[0] + in_pageinfo = PdfInfo(input_file)[0] - assert in_pageinfo['height_pixels'] < in_pageinfo['width_pixels'], \ + assert in_pageinfo.height_pixels < in_pageinfo.width_pixels, \ "Expected the input page to be landscape" - assert in_pageinfo['rotate'] == 90, "Expected a rotated page" + assert in_pageinfo.rotation == 90, "Expected a rotated page" out = check_ocrmypdf( input_file, outpdf, '--pdf-renderer', 'hocr', '--deskew', '--tesseract-timeout', '0') - out_pageinfo = pdf_get_all_pageinfo(str(out))[0] + out_pageinfo = PdfInfo(out)[0] - assert out_pageinfo['height_pixels'] > out_pageinfo['width_pixels'], \ + assert out_pageinfo.height_pixels > out_pageinfo.width_pixels, \ "Expected the output page to be portrait" - assert out_pageinfo['rotate'] == 0, \ + assert out_pageinfo.rotation == 0, \ "Expected no page rotation for output" - assert in_pageinfo['width_pixels'] == out_pageinfo['height_pixels'] and \ - in_pageinfo['height_pixels'] == out_pageinfo['width_pixels'], \ + assert in_pageinfo.width_pixels == out_pageinfo.height_pixels and \ + in_pageinfo.height_pixels == out_pageinfo.width_pixels, \ "Expected page rotation to be baked in" @@ -743,11 +733,11 @@ def test_very_high_dpi(spoof_tesseract_cache, resources, outpdf): "Checks for a Decimal quantize error with high DPI, etc" check_ocrmypdf(resources / '2400dpi.pdf', outpdf, env=spoof_tesseract_cache) - pdfinfo = pdf_get_all_pageinfo(outpdf) + pdfinfo = PdfInfo(outpdf) - image = pdfinfo[0]['images'][0] - assert image['dpi_w'] == image['dpi_h'] - assert image['dpi_w'] == 2400 + image = pdfinfo[0].images[0] + assert isclose(image.xres, image.yres) + assert isclose(image.xres, 2400) def test_overlay(spoof_tesseract_noop, resources, outpdf): @@ -900,21 +890,21 @@ def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec, assert p.returncode == ExitCode.ok - pdfinfo = pdf_get_all_pageinfo(output_file) + pdfinfo = PdfInfo(output_file) - pdfimage = pdfinfo[0]['images'][0] + pdfimage = pdfinfo[0].images[0] if input_file.endswith('.png'): - assert pdfimage['enc'] != 'jpeg', \ + assert pdfimage.enc != Encoding.jpeg, \ "Lossless compression changed to lossy!" elif input_file.endswith('.jpg'): - assert pdfimage['enc'] == 'jpeg', \ + assert pdfimage.enc == Encoding.jpeg, \ "Lossy compression changed to lossless!" if im.mode.startswith('RGB') or im.mode.startswith('BGR'): - assert pdfimage['color'] == 'rgb', \ + assert pdfimage.color == Colorspace.rgb, \ "Colorspace changed" elif im.mode.startswith('L'): - assert pdfimage['color'] == 'gray', \ + assert pdfimage.color == Colorspace.gray, \ "Colorspace changed" @@ -945,20 +935,20 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec, assert p.returncode == ExitCode.ok - pdfinfo = pdf_get_all_pageinfo(output_file) + pdfinfo = PdfInfo(output_file) - pdfimage = pdfinfo[0]['images'][0] + pdfimage = pdfinfo[0].images[0] - if compression == 'jpeg': - assert pdfimage['enc'] == 'jpeg' + if compression == "jpeg": + assert pdfimage.enc == Encoding.jpeg elif compression == 'lossless': - assert pdfimage['enc'] == 'image' + assert pdfimage.enc not in (Encoding.jpeg, Encoding.jpeg2000) if im.mode.startswith('RGB') or im.mode.startswith('BGR'): - assert pdfimage['color'] == 'rgb', \ + assert pdfimage.color == Colorspace.rgb, \ "Colorspace changed" elif im.mode.startswith('L'): - assert pdfimage['color'] == 'gray', \ + assert pdfimage.color == Colorspace.gray, \ "Colorspace changed" @@ -970,7 +960,7 @@ def test_sidecar_pagecount(spoof_tesseract_cache, resources, outpdf): '--sidecar', sidecar, env=spoof_tesseract_cache) - pdfinfo = pdf_get_all_pageinfo(str(resources / 'multipage.pdf')) + pdfinfo = PdfInfo(resources / 'multipage.pdf') num_pages = len(pdfinfo) with open(sidecar, 'r') as f: diff --git a/tests/test_multiprocessing.py b/tests/test_multiprocessing.py new file mode 100644 index 00000000..b9ff72f3 --- /dev/null +++ b/tests/test_multiprocessing.py @@ -0,0 +1,47 @@ + +from ocrmypdf.pdfinfo import PdfInfo, PageInfo +from ocrmypdf.pipeline import JobContext, JobContextManager +from multiprocessing import Process +from multiprocessing.managers import BaseProxy + + +def test_jobcontext_proxy(resources): + # Prove that managers are set up correctly to share state among processes + manager = JobContextManager() + manager.register('JobContext', JobContext) + + # Start the manager in a child process (or maybe thread) + manager.start() + + # Tell the manager process to retrieve pdf info + context = manager.JobContext() + context.generate_pdfinfo(resources / 'graph.pdf') + + # Get a copy of that information for this process + pdfinfo = context.get_pdfinfo() + assert len(pdfinfo) == 1 + assert pdfinfo[0].rotation == 0 + + # Update information and send back to manager + pdfinfo[0].rotation = 90 + context.set_pdfinfo(pdfinfo) + + # Retrieve again, ensure it stayed changed + pdfinfo2 = context.get_pdfinfo() + assert pdfinfo2[0].rotation == 90 + + # Start a new process which gets its own proxy object + def client(context): + assert isinstance(context, BaseProxy) + pdfinfo = context.get_pdfinfo() + page = pdfinfo[0] + assert page.rotation == 90 + page.rotation += 90 + context.set_pdfinfo(pdfinfo) + + p = Process(target=client, args=(context,)) + p.start() + p.join() + assert p.exitcode == 0, "Child process failed" + + assert context.get_pdfinfo()[0].rotation == 180 \ No newline at end of file diff --git a/tests/test_pageinfo.py b/tests/test_pageinfo.py index 2a1fe05a..1b6459f0 100644 --- a/tests/test_pageinfo.py +++ b/tests/test_pageinfo.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 # © 2015 James R. Barlow: github.com/jbarlow83 -from ocrmypdf import pageinfo +from ocrmypdf import pdfinfo from reportlab.pdfgen.canvas import Canvas from PIL import Image from tempfile import NamedTemporaryFile +from math import isclose +from ocrmypdf.pdfinfo import Colorspace, Encoding from contextlib import suppress import os import shutil @@ -26,13 +28,13 @@ def test_single_page_text(outdir): pdf.showPage() pdf.save() - pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename)) + info = pdfinfo.PdfInfo(filename) - assert len(pdfinfo) == 1 - page = pdfinfo[0] + assert len(info) == 1 + page = info[0] - assert page['has_text'] - assert len(page['images']) == 0 + assert page.has_text + assert len(page.images) == 0 def test_single_page_image(outdir): @@ -53,21 +55,21 @@ def test_single_page_image(outdir): layout_fun=layout_fun) filename.write_bytes(pdf_bytes) - pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename)) + info = pdfinfo.PdfInfo(filename) - assert len(pdfinfo) == 1 - page = pdfinfo[0] + assert len(info) == 1 + page = info[0] - assert not page['has_text'] - assert len(page['images']) == 1 + assert not page.has_text + assert len(page.images) == 1 - pdfimage = page['images'][0] - assert pdfimage['width'] == 8 - assert pdfimage['color'] == 'gray' + pdfimage = page.images[0] + assert pdfimage.width == 8 + assert pdfimage.color == Colorspace.gray # DPI in a 1"x1" is the image width - assert abs(pdfimage['dpi_w'] - 8) < 1e-5 - assert abs(pdfimage['dpi_h'] - 8) < 1e-5 + assert isclose(pdfimage.xres, 8) + assert isclose(pdfimage.yres, 8) def test_single_page_inline_image(outdir): @@ -83,35 +85,41 @@ def test_single_page_inline_image(outdir): pdf.showPage() pdf.save() - pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename)) - print(pdfinfo) - pdfimage = pdfinfo[0]['images'][0] - assert (pdfimage['dpi_w'] - 8) < 1e-5 - assert pdfimage['color'] != '-' - assert pdfimage['width'] == 8 + pdf = pdfinfo.PdfInfo(filename) + print(pdf) + pdfimage = pdf[0].images[0] + assert isclose(pdfimage.xres, 8) + assert pdfimage.color == Colorspace.rgb # reportlab produces color image + assert pdfimage.width == 8 def test_jpeg(resources, outdir): filename = resources / 'c02-22.pdf' - pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename)) + pdf = pdfinfo.PdfInfo(filename) - pdfimage = pdfinfo[0]['images'][0] - assert pdfimage['enc'] == 'jpeg' - assert (pdfimage['dpi_w'] - 150) < 1e-5 + pdfimage = pdf[0].images[0] + assert pdfimage.enc == Encoding.jpeg + assert isclose(pdfimage.xres, 150) def test_form_xobject(resources): filename = resources / 'formxobject.pdf' - pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename)) - pdfimage = pdfinfo[0]['images'][0] - assert pdfimage['width'] == 50 + pdf = pdfinfo.PdfInfo(filename) + pdfimage = pdf[0].images[0] + assert pdfimage.width == 50 def test_no_contents(resources): filename = resources / 'no_contents.pdf' - pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename)) - assert len(pdfinfo[0]['images']) == 0 - assert pdfinfo[0]['has_text'] == False \ No newline at end of file + pdf = pdfinfo.PdfInfo(filename) + assert len(pdf[0].images) == 0 + assert pdf[0].has_text == False + + +def test_oversized_page(resources): + pdf = pdfinfo.PdfInfo(resources / 'poster.pdf') + image = pdf[0].images[0] + assert image.width * image.xres > 200, "this is supposed to be oversized" \ No newline at end of file diff --git a/tests/test_tess4.py b/tests/test_tess4.py index b6d60860..118fa5e0 100644 --- a/tests/test_tess4.py +++ b/tests/test_tess4.py @@ -4,7 +4,7 @@ import pytest from ocrmypdf.exceptions import ExitCode from ocrmypdf.exec import tesseract -from ocrmypdf import pageinfo +from ocrmypdf import pdfinfo import sys import os import PyPDF2 as pypdf @@ -96,9 +96,9 @@ def test_skip_pages_does_not_replicate( env=ensure_tess4 ) - info_in = pageinfo.pdf_get_all_pageinfo(str(infile)) + info_in = pdfinfo.PdfInfo(infile) - info = pageinfo.pdf_get_all_pageinfo(str(outpdf)) + info = pdfinfo.PdfInfo(outpdf) for page in info: assert len(page['images']) == 1, "skipped page was replicated" @@ -115,6 +115,6 @@ def test_content_preservation(ensure_tess4, resources, outpdf): env=ensure_tess4 ) - info = pageinfo.pdf_get_all_pageinfo(str(outpdf)) + info = pdfinfo.PdfInfo(outpdf) page = info[0] - assert len(page['images']) > 1, "masked were rasterized" \ No newline at end of file + assert len(page.images) > 1, "masked were rasterized" \ No newline at end of file diff --git a/tests/test_userunit.py b/tests/test_userunit.py new file mode 100644 index 00000000..f60d4f2a --- /dev/null +++ b/tests/test_userunit.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# © 2017 James R. Barlow: github.com/jbarlow83 + +from subprocess import Popen, PIPE, check_output, check_call, DEVNULL +import os +import shutil +import pytest +from ocrmypdf.pdfinfo import PdfInfo, Colorspace, Encoding +import PyPDF2 as pypdf +from ocrmypdf.exceptions import ExitCode +from ocrmypdf import leptonica +from ocrmypdf.pdfa import file_claims_pdfa +from ocrmypdf.exec import ghostscript +import logging +from math import isclose + + +check_ocrmypdf = pytest.helpers.check_ocrmypdf +run_ocrmypdf = pytest.helpers.run_ocrmypdf +spoof = pytest.helpers.spoof + + +@pytest.fixture +def poster(resources): + return resources / 'poster.pdf' + + +def test_userunit_ghostscript_fails(poster, no_outpdf): + p, out, err = run_ocrmypdf(poster, no_outpdf, '--output-type=pdfa') + assert p.returncode == ExitCode.input_file + + +def test_userunit_qpdf_passes(spoof_tesseract_cache, poster, outpdf): + before = PdfInfo(poster) + check_ocrmypdf(poster, outpdf, '--output-type=pdf', + env=spoof_tesseract_cache) + + after = PdfInfo(outpdf) + assert isclose(before[0].width_inches, after[0].width_inches) + + +def test_rotate_interaction(spoof_tesseract_cache, poster, outpdf): + check_ocrmypdf(poster, outpdf, '--output-type=pdf', + '--rotate-pages', + env=spoof_tesseract_cache)