Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1ebf3144af | ||
|
|
7a1cccbc4e | ||
|
|
ebacff1b39 | ||
|
|
c7c447be66 | ||
|
|
91aa175602 | ||
|
|
b267494e4a | ||
|
|
f687180ecc | ||
|
|
6f4b38b103 | ||
|
|
d32324859c | ||
|
|
48222b87b5 | ||
|
|
62e5edc72b | ||
|
|
2846d46bb8 | ||
|
|
47ef1914d4 | ||
|
|
df157552f3 | ||
|
|
0b3a526049 | ||
|
|
1e80d412fa | ||
|
|
df6e106203 |
-24
@@ -1,24 +0,0 @@
|
|||||||
[paths]
|
|
||||||
source =
|
|
||||||
src
|
|
||||||
*/site-packages
|
|
||||||
|
|
||||||
[run]
|
|
||||||
branch = true
|
|
||||||
parallel = true
|
|
||||||
concurrency =
|
|
||||||
thread
|
|
||||||
multiprocessing
|
|
||||||
source =
|
|
||||||
src/ocrmypdf
|
|
||||||
|
|
||||||
[report]
|
|
||||||
exclude_lines =
|
|
||||||
pragma: no cover
|
|
||||||
def __repr__
|
|
||||||
raise AssertionError
|
|
||||||
raise NotImplementedError
|
|
||||||
if 0:
|
|
||||||
if False:
|
|
||||||
if __name__ == .__main__.:
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
@@ -56,6 +56,11 @@ Programs that call ``ocrmypdf.ocr()`` should also install a SIGBUS signal
|
|||||||
handler (except on Windows), to raise an exception if access to a memory
|
handler (except on Windows), to raise an exception if access to a memory
|
||||||
mapped file fails. OCRmyPDF may use memory mapping.
|
mapped file fails. OCRmyPDF may use memory mapping.
|
||||||
|
|
||||||
|
``ocrmypdf.ocr()`` will take a threading lock to prevent multiple runs of itself
|
||||||
|
in the same Python interpreter process. This is not thread-safe, because of how
|
||||||
|
OCRmyPDF's plugins and Python's library import system work. If you need to parallelize
|
||||||
|
OCRmyPDF, use processes.
|
||||||
|
|
||||||
.. warning::
|
.. warning::
|
||||||
|
|
||||||
On Windows and macOS, the script that calls ``ocrmypdf.ocr()`` must be
|
On Windows and macOS, the script that calls ``ocrmypdf.ocr()`` must be
|
||||||
|
|||||||
@@ -12,6 +12,36 @@ may be unreliable. Use the API to depend on precise behavior.
|
|||||||
The public API may be useful in scripts that launch OCRmyPDF processes or that
|
The public API may be useful in scripts that launch OCRmyPDF processes or that
|
||||||
wish to use some of its features for working with PDFs.
|
wish to use some of its features for working with PDFs.
|
||||||
|
|
||||||
|
v11.5.0
|
||||||
|
=======
|
||||||
|
|
||||||
|
- Fixed an issue where the output page size might differ by a fractional amount
|
||||||
|
due to rounding, when ``--force-ocr`` was used and the page contained objects
|
||||||
|
with multiple resolutions.
|
||||||
|
- When determining the resolution at which to rasterize a page, we now consider
|
||||||
|
printed text on the page as requiring a higher resolution. This fixes issues
|
||||||
|
with certain pages being rendered with unacceptably low resolution text, but
|
||||||
|
may increase output file sizes in some workflows where low resolution text
|
||||||
|
is acceptable.
|
||||||
|
- Added a workaround to fix an exception that occurs when trying to
|
||||||
|
``import ocrmypdf.leptonica`` on Apple ARM silicon (or potentially, other
|
||||||
|
platforms that do not permit write+executable memory).
|
||||||
|
|
||||||
|
v11.4.5
|
||||||
|
=======
|
||||||
|
|
||||||
|
- Fixed an issue where files may not be closed when the API is used.
|
||||||
|
- Improved ``setup.cfg`` with better settings for test coverage.
|
||||||
|
|
||||||
|
v11.4.4
|
||||||
|
=======
|
||||||
|
|
||||||
|
- Fixed ``AttributeError: 'NoneType' object has no attribute 'userunit'``, issue #700,
|
||||||
|
related to OCRmyPDF not properly forwarded an error message from pdfminer.six.
|
||||||
|
- Adjusted typing of some arguments.
|
||||||
|
- ``ocrmypdf.ocr`` now takes a ``threading.Lock`` for reasons outlined in the
|
||||||
|
documentation.
|
||||||
|
|
||||||
v11.4.3
|
v11.4.3
|
||||||
=======
|
=======
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ filterwarnings =
|
|||||||
ignore:.*XMLParser.*:DeprecationWarning
|
ignore:.*XMLParser.*:DeprecationWarning
|
||||||
markers =
|
markers =
|
||||||
slow
|
slow
|
||||||
|
addopts =
|
||||||
|
-n auto
|
||||||
|
|
||||||
[isort]
|
[isort]
|
||||||
multi_line_output=3
|
multi_line_output=3
|
||||||
@@ -27,3 +29,30 @@ known_third_party = PIL,_cffi_backend,cffi,flask,img2pdf,pdfminer,pikepdf,pkg_re
|
|||||||
|
|
||||||
[metadata]
|
[metadata]
|
||||||
license_file = LICENSE
|
license_file = LICENSE
|
||||||
|
|
||||||
|
[coverage:paths]
|
||||||
|
source =
|
||||||
|
src/
|
||||||
|
|
||||||
|
[coverage:run]
|
||||||
|
branch = true
|
||||||
|
parallel = true
|
||||||
|
concurrency = multiprocessing
|
||||||
|
source =
|
||||||
|
src/ocrmypdf
|
||||||
|
|
||||||
|
[coverage:report]
|
||||||
|
# Regexes for lines to exclude from consideration
|
||||||
|
exclude_lines =
|
||||||
|
# Have to re-enable the standard pragma
|
||||||
|
pragma: no cover
|
||||||
|
|
||||||
|
# Don't complain if tests don't hit defensive assertion code:
|
||||||
|
raise AssertionError
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
# Don't complain if non-runnable code isn't run:
|
||||||
|
if 0:
|
||||||
|
if False:
|
||||||
|
if __name__ == .__main__.:
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
|||||||
@@ -109,15 +109,11 @@ def exec_progress_pool(
|
|||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
results = pool.imap_unordered(task, task_arguments)
|
results = pool.imap_unordered(task, task_arguments)
|
||||||
while True:
|
for result in results:
|
||||||
try:
|
if task_finished:
|
||||||
result = results.next()
|
task_finished(result, pbar)
|
||||||
if task_finished:
|
else:
|
||||||
task_finished(result, pbar)
|
pbar.update()
|
||||||
else:
|
|
||||||
pbar.update()
|
|
||||||
except StopIteration:
|
|
||||||
break
|
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
# Terminate pool so we exit instantly
|
# Terminate pool so we exit instantly
|
||||||
pool.terminate()
|
pool.terminate()
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ from os import fspath
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from shutil import which
|
from shutil import which
|
||||||
from subprocess import PIPE, CalledProcessError
|
from subprocess import PIPE, CalledProcessError
|
||||||
from typing import Optional, cast
|
from typing import Optional
|
||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|
||||||
@@ -56,16 +56,14 @@ def version():
|
|||||||
def jpeg_passthrough_available() -> bool:
|
def jpeg_passthrough_available() -> bool:
|
||||||
"""Returns True if the installed version of Ghostscript supports JPEG passthru
|
"""Returns True if the installed version of Ghostscript supports JPEG passthru
|
||||||
|
|
||||||
Prior to 9.23, Ghostscript decode and re-encoded JPEGs internally. In 9.23
|
Prior to 9.23, Ghostscript decoded and re-encoded JPEGs internally. In 9.23
|
||||||
it gained the ability to keep JPEGs unmodified. However, the 9.23
|
it gained the ability to keep JPEGs unmodified. However, the 9.23
|
||||||
implementation was buggy and would deletes the last two bytes of images in
|
implementation was buggy and would deletes the last two bytes of images in
|
||||||
some cases, as reported here.
|
some cases, as reported here.
|
||||||
https://bugs.ghostscript.com/show_bug.cgi?id=699216
|
https://bugs.ghostscript.com/show_bug.cgi?id=699216
|
||||||
|
|
||||||
The issue was fixed for 9.24, hence that is the first version we consider
|
The issue was fixed for 9.24, hence that is the first version we consider
|
||||||
the feature available. (However, we don't use 9.24 at all, so the first
|
the feature available. (Ghostscript 9.24 has its own problems is blacklisted.)
|
||||||
version that allows JPEG passthrough is 9.25.
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
return version() >= '9.24'
|
return version() >= '9.24'
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from collections import namedtuple
|
|||||||
from os import fspath
|
from os import fspath
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired
|
from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired
|
||||||
from typing import List
|
from typing import List, Optional
|
||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|
||||||
@@ -118,7 +118,7 @@ def get_languages():
|
|||||||
return set(lang.strip() for lang in rest)
|
return set(lang.strip() for lang in rest)
|
||||||
|
|
||||||
|
|
||||||
def tess_base_args(langs: List[str], engine_mode: int) -> List[str]:
|
def tess_base_args(langs: List[str], engine_mode: Optional[int]) -> List[str]:
|
||||||
args = ['tesseract']
|
args = ['tesseract']
|
||||||
if langs:
|
if langs:
|
||||||
args.extend(['-l', '+'.join(langs)])
|
args.extend(['-l', '+'.join(langs)])
|
||||||
@@ -127,7 +127,7 @@ def tess_base_args(langs: List[str], engine_mode: int) -> List[str]:
|
|||||||
return args
|
return args
|
||||||
|
|
||||||
|
|
||||||
def get_orientation(input_file: Path, engine_mode: int, timeout: float):
|
def get_orientation(input_file: Path, engine_mode: Optional[int], timeout: float):
|
||||||
args_tesseract = tess_base_args(['osd'], engine_mode) + [
|
args_tesseract = tess_base_args(['osd'], engine_mode) + [
|
||||||
'--psm',
|
'--psm',
|
||||||
'0',
|
'0',
|
||||||
|
|||||||
@@ -206,17 +206,21 @@ def validate_pdfinfo_options(context: PdfContext):
|
|||||||
context.plugin_manager.hook.validate(pdfinfo=pdfinfo, options=options)
|
context.plugin_manager.hook.validate(pdfinfo=pdfinfo, options=options)
|
||||||
|
|
||||||
|
|
||||||
|
def _vector_page_dpi(pageinfo):
|
||||||
|
return VECTOR_PAGE_DPI if pageinfo.has_vector or pageinfo.has_text else 0.0
|
||||||
|
|
||||||
|
|
||||||
def get_page_dpi(pageinfo, options):
|
def get_page_dpi(pageinfo, options):
|
||||||
"Get the DPI when nonsquare DPI is tolerable"
|
"Get the DPI when nonsquare DPI is tolerable"
|
||||||
xres = max(
|
xres = max(
|
||||||
pageinfo.dpi.x or VECTOR_PAGE_DPI,
|
pageinfo.dpi.x or VECTOR_PAGE_DPI,
|
||||||
options.oversample or 0.0,
|
options.oversample or 0.0,
|
||||||
VECTOR_PAGE_DPI if pageinfo.has_vector else 0.0,
|
_vector_page_dpi(pageinfo),
|
||||||
)
|
)
|
||||||
yres = max(
|
yres = max(
|
||||||
pageinfo.dpi.y or VECTOR_PAGE_DPI,
|
pageinfo.dpi.y or VECTOR_PAGE_DPI,
|
||||||
options.oversample or 0,
|
options.oversample or 0,
|
||||||
VECTOR_PAGE_DPI if pageinfo.has_vector else 0.0,
|
_vector_page_dpi(pageinfo),
|
||||||
)
|
)
|
||||||
return Resolution(float(xres), float(yres))
|
return Resolution(float(xres), float(yres))
|
||||||
|
|
||||||
@@ -230,7 +234,7 @@ def get_page_square_dpi(pageinfo, options) -> Resolution:
|
|||||||
max(
|
max(
|
||||||
(xres * userunit) or VECTOR_PAGE_DPI,
|
(xres * userunit) or VECTOR_PAGE_DPI,
|
||||||
(yres * userunit) or VECTOR_PAGE_DPI,
|
(yres * userunit) or VECTOR_PAGE_DPI,
|
||||||
VECTOR_PAGE_DPI if pageinfo.has_vector else 0.0,
|
_vector_page_dpi(pageinfo),
|
||||||
options.oversample or 0.0,
|
options.oversample or 0.0,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -243,7 +247,7 @@ def get_canvas_square_dpi(pageinfo, options) -> Resolution:
|
|||||||
max(
|
max(
|
||||||
(pageinfo.dpi.x) or VECTOR_PAGE_DPI,
|
(pageinfo.dpi.x) or VECTOR_PAGE_DPI,
|
||||||
(pageinfo.dpi.y) or VECTOR_PAGE_DPI,
|
(pageinfo.dpi.y) or VECTOR_PAGE_DPI,
|
||||||
VECTOR_PAGE_DPI if pageinfo.has_vector else 0.0,
|
_vector_page_dpi(pageinfo),
|
||||||
options.oversample or 0.0,
|
options.oversample or 0.0,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -588,12 +592,17 @@ def create_pdf_page_from_image(image: Path, page_context: PageContext):
|
|||||||
# except that the hocr renderer does not understand non-square DPI. The
|
# except that the hocr renderer does not understand non-square DPI. The
|
||||||
# sandwich renderer would be fine.
|
# sandwich renderer would be fine.
|
||||||
output_file = page_context.get_path('visible.pdf')
|
output_file = page_context.get_path('visible.pdf')
|
||||||
dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
|
|
||||||
layout_fun = img2pdf.get_fixed_dpi_layout_fun(dpi)
|
pageinfo = page_context.pageinfo
|
||||||
|
pagesize = 72.0 * float(pageinfo.width_inches), 72.0 * float(pageinfo.height_inches)
|
||||||
|
if pageinfo.rotation % 180 == 90:
|
||||||
|
pagesize = pagesize[1], pagesize[0]
|
||||||
|
|
||||||
# This create a single page PDF
|
# This create a single page PDF
|
||||||
with open(image, 'rb') as imfile, open(output_file, 'wb') as pdf:
|
with open(image, 'rb') as imfile, open(output_file, 'wb') as pdf:
|
||||||
log.debug('convert')
|
log.debug('convert')
|
||||||
|
|
||||||
|
layout_fun = img2pdf.get_layout_fun(pagesize)
|
||||||
img2pdf.convert(
|
img2pdf.convert(
|
||||||
imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf
|
imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -302,7 +302,7 @@ def exec_concurrent(context: PdfContext):
|
|||||||
copy_final(pdf, options.output_file, context)
|
copy_final(pdf, options.output_file, context)
|
||||||
|
|
||||||
|
|
||||||
def configure_debug_logging(log_filename, prefix: str = ''):
|
def configure_debug_logging(log_filename: Path, prefix: str = ''):
|
||||||
"""
|
"""
|
||||||
Create a debug log file at a specified location.
|
Create a debug log file at a specified location.
|
||||||
|
|
||||||
@@ -338,7 +338,11 @@ def run_pipeline(options, *, plugin_manager, api=False):
|
|||||||
and not api
|
and not api
|
||||||
):
|
):
|
||||||
# Debug log for command line interface only with verbose output
|
# Debug log for command line interface only with verbose output
|
||||||
debug_log_handler = configure_debug_logging(Path(work_folder) / "debug.log")
|
# See https://github.com/pytest-dev/pytest/issues/5502 for why we skip this
|
||||||
|
# when pytest is running
|
||||||
|
debug_log_handler = configure_debug_logging(
|
||||||
|
Path(work_folder) / "debug.log"
|
||||||
|
) # pragma: no cover
|
||||||
|
|
||||||
pikepdf_enable_mmap()
|
pikepdf_enable_mmap()
|
||||||
|
|
||||||
|
|||||||
+16
-7
@@ -8,6 +8,7 @@
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
import threading
|
||||||
from enum import IntEnum
|
from enum import IntEnum
|
||||||
from io import IOBase
|
from io import IOBase
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -30,6 +31,8 @@ except ModuleNotFoundError:
|
|||||||
StrPath = Union[os.PathLike, AnyStr]
|
StrPath = Union[os.PathLike, AnyStr]
|
||||||
PathOrIO = Union[BinaryIO, StrPath]
|
PathOrIO = Union[BinaryIO, StrPath]
|
||||||
|
|
||||||
|
_api_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
class Verbosity(IntEnum):
|
class Verbosity(IntEnum):
|
||||||
"""Verbosity level for configure_logging."""
|
"""Verbosity level for configure_logging."""
|
||||||
@@ -306,12 +309,18 @@ def ocr( # pylint: disable=unused-argument
|
|||||||
|
|
||||||
parser = get_parser()
|
parser = get_parser()
|
||||||
create_options_kwargs['parser'] = parser
|
create_options_kwargs['parser'] = parser
|
||||||
plugin_manager = get_plugin_manager(plugins)
|
|
||||||
plugin_manager.hook.add_options(parser=parser) # pylint: disable=no-member
|
|
||||||
|
|
||||||
if 'verbose' in kwargs:
|
with _api_lock:
|
||||||
warn("ocrmypdf.ocr(verbose=) is ignored. Use ocrmypdf.configure_logging().")
|
# We can't allow multiple ocrmypdf.ocr() threads to run in parallel, because
|
||||||
|
# they might install different plugins, and generally speaking we have areas
|
||||||
|
# of code that use global state.
|
||||||
|
|
||||||
options = create_options(**create_options_kwargs)
|
plugin_manager = get_plugin_manager(plugins)
|
||||||
check_options(options, plugin_manager)
|
plugin_manager.hook.add_options(parser=parser) # pylint: disable=no-member
|
||||||
return run_pipeline(options=options, plugin_manager=plugin_manager, api=True)
|
|
||||||
|
if 'verbose' in kwargs:
|
||||||
|
warn("ocrmypdf.ocr(verbose=) is ignored. Use ocrmypdf.configure_logging().")
|
||||||
|
|
||||||
|
options = create_options(**create_options_kwargs)
|
||||||
|
check_options(options, plugin_manager)
|
||||||
|
return run_pipeline(options=options, plugin_manager=plugin_manager, api=True)
|
||||||
|
|||||||
+18
-15
@@ -13,6 +13,7 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import platform
|
||||||
import sys
|
import sys
|
||||||
import threading
|
import threading
|
||||||
import warnings
|
import warnings
|
||||||
@@ -170,20 +171,6 @@ tls = threading.local()
|
|||||||
tls.trap = None
|
tls.trap = None
|
||||||
|
|
||||||
|
|
||||||
@ffi.callback("void(char *)")
|
|
||||||
def _stderr_handler(cstr):
|
|
||||||
msg = ffi.string(cstr).decode(errors='replace')
|
|
||||||
if msg.startswith("Error"):
|
|
||||||
logger.error(msg)
|
|
||||||
elif msg.startswith("Warning"):
|
|
||||||
logger.warning(msg)
|
|
||||||
else:
|
|
||||||
logger.debug(msg)
|
|
||||||
if tls.trap is not None:
|
|
||||||
tls.trap.append(msg)
|
|
||||||
return
|
|
||||||
|
|
||||||
|
|
||||||
class _LeptonicaErrorTrap_Queue:
|
class _LeptonicaErrorTrap_Queue:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.queue = deque()
|
self.queue = deque()
|
||||||
@@ -213,9 +200,25 @@ class _LeptonicaErrorTrap_Queue:
|
|||||||
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
||||||
|
@ffi.callback("void(char *)")
|
||||||
|
def _stderr_handler(cstr):
|
||||||
|
msg = ffi.string(cstr).decode(errors='replace')
|
||||||
|
if msg.startswith("Error"):
|
||||||
|
logger.error(msg)
|
||||||
|
elif msg.startswith("Warning"):
|
||||||
|
logger.warning(msg)
|
||||||
|
else:
|
||||||
|
logger.debug(msg)
|
||||||
|
if tls.trap is not None:
|
||||||
|
tls.trap.append(msg)
|
||||||
|
return
|
||||||
|
|
||||||
lept.leptSetStderrHandler(_stderr_handler)
|
lept.leptSetStderrHandler(_stderr_handler)
|
||||||
except ffi.error:
|
except (ffi.error, MemoryError):
|
||||||
# Pre-1.79 Leptonica does not have leptSetStderrHandler
|
# Pre-1.79 Leptonica does not have leptSetStderrHandler
|
||||||
|
# And some platforms, notably Apple ARM 64, do not allow the write+execute
|
||||||
|
# memory needed to set up the callback function.
|
||||||
_LeptonicaErrorTrap = _LeptonicaErrorTrap_Redirect
|
_LeptonicaErrorTrap = _LeptonicaErrorTrap_Redirect
|
||||||
else:
|
else:
|
||||||
# 1.79 have this new symbol
|
# 1.79 have this new symbol
|
||||||
|
|||||||
@@ -498,7 +498,7 @@ def transcode_pngs(
|
|||||||
|
|
||||||
|
|
||||||
@deprecated
|
@deprecated
|
||||||
def rewrite_png_as_g4(pike: Pdf, im_obj: Object, compdata) -> None:
|
def rewrite_png_as_g4(pike: Pdf, im_obj: Object, compdata) -> None: # pragma: no cover
|
||||||
im_obj.BitsPerComponent = 1
|
im_obj.BitsPerComponent = 1
|
||||||
im_obj.Width = compdata.w
|
im_obj.Width = compdata.w
|
||||||
im_obj.Height = compdata.h
|
im_obj.Height = compdata.h
|
||||||
@@ -519,7 +519,7 @@ def rewrite_png_as_g4(pike: Pdf, im_obj: Object, compdata) -> None:
|
|||||||
|
|
||||||
|
|
||||||
@deprecated
|
@deprecated
|
||||||
def rewrite_png(pike: Pdf, im_obj: Object, compdata) -> None:
|
def rewrite_png(pike: Pdf, im_obj: Object, compdata) -> None: # pragma: no cover
|
||||||
# When a PNG is inserted into a PDF, we more or less copy the IDAT section from
|
# When a PNG is inserted into a PDF, we more or less copy the IDAT section from
|
||||||
# the PDF and transfer the rest of the PNG headers to PDF image metadata.
|
# the PDF and transfer the rest of the PNG headers to PDF image metadata.
|
||||||
# One thing we have to do is tell the PDF reader whether a predictor was used
|
# One thing we have to do is tell the PDF reader whether a predictor was used
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ import pikepdf
|
|||||||
from pikepdf import Object, Pdf, PdfMatrix
|
from pikepdf import Object, Pdf, PdfMatrix
|
||||||
|
|
||||||
from ocrmypdf._concurrent import exec_progress_pool
|
from ocrmypdf._concurrent import exec_progress_pool
|
||||||
from ocrmypdf.exceptions import EncryptedPdfError
|
from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
|
||||||
from ocrmypdf.helpers import Resolution, available_cpu_count, pikepdf_enable_mmap
|
from ocrmypdf.helpers import Resolution, available_cpu_count, pikepdf_enable_mmap
|
||||||
from ocrmypdf.pdfinfo.layout import get_page_analysis, get_text_boxes
|
from ocrmypdf.pdfinfo.layout import get_page_analysis, get_text_boxes
|
||||||
|
|
||||||
@@ -598,6 +598,8 @@ def _pdf_pageinfo_concurrent(
|
|||||||
|
|
||||||
def update_pageinfo(result, pbar):
|
def update_pageinfo(result, pbar):
|
||||||
page = result
|
page = result
|
||||||
|
if not page:
|
||||||
|
raise InputFileError("Could read a page in the PDF")
|
||||||
pages[page.pageno] = page
|
pages[page.pageno] = page
|
||||||
pbar.update()
|
pbar.update()
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from pdfminer.pdffont import PDFSimpleFont, PDFUnicodeNotDefined
|
|||||||
from pdfminer.pdfpage import PDFPage
|
from pdfminer.pdfpage import PDFPage
|
||||||
from pdfminer.utils import bbox2str, matrix2str
|
from pdfminer.utils import bbox2str, matrix2str
|
||||||
|
|
||||||
from ocrmypdf.exceptions import EncryptedPdfError
|
from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
|
||||||
|
|
||||||
STRIP_NAME = re.compile(r'[0-9]+')
|
STRIP_NAME = re.compile(r'[0-9]+')
|
||||||
|
|
||||||
@@ -236,8 +236,13 @@ def get_page_analysis(infile, pageno, pscript5_mode):
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
with Path(infile).open('rb') as f:
|
with Path(infile).open('rb') as f:
|
||||||
page = PDFPage.get_pages(f, pagenos=[pageno], maxpages=0)
|
page_iter = PDFPage.get_pages(f, pagenos=[pageno], maxpages=0)
|
||||||
interp.process_page(next(page))
|
page = next(page_iter, None)
|
||||||
|
if page is None:
|
||||||
|
raise InputFileError(
|
||||||
|
f"pdfminer could not process page {pageno} (counting from 0)."
|
||||||
|
)
|
||||||
|
interp.process_page(page)
|
||||||
except PDFTextExtractionNotAllowed as e:
|
except PDFTextExtractionNotAllowed as e:
|
||||||
raise EncryptedPdfError() from e
|
raise EncryptedPdfError() from e
|
||||||
finally:
|
finally:
|
||||||
|
|||||||
@@ -77,20 +77,19 @@ def run_polling_stderr(args, *, callback, check=False, env=None, **kwargs):
|
|||||||
args, env, process_log, text = _fix_process_args(args, env, kwargs)
|
args, env, process_log, text = _fix_process_args(args, env, kwargs)
|
||||||
assert text, "Must use text=True"
|
assert text, "Must use text=True"
|
||||||
|
|
||||||
proc = Popen(args, env=env, **kwargs)
|
with Popen(args, env=env, **kwargs) as proc:
|
||||||
|
lines = []
|
||||||
|
while proc.poll() is None:
|
||||||
|
for msg in iter(proc.stderr.readline, ''):
|
||||||
|
if process_log.isEnabledFor(logging.DEBUG):
|
||||||
|
process_log.debug(msg.strip())
|
||||||
|
callback(msg)
|
||||||
|
lines.append(msg)
|
||||||
|
stderr = ''.join(lines)
|
||||||
|
|
||||||
lines = []
|
if check and proc.returncode != 0:
|
||||||
while proc.poll() is None:
|
raise CalledProcessError(proc.returncode, args, output=None, stderr=stderr)
|
||||||
for msg in iter(proc.stderr.readline, ''):
|
return CompletedProcess(args, proc.returncode, None, stderr=stderr)
|
||||||
if process_log.isEnabledFor(logging.DEBUG):
|
|
||||||
process_log.debug(msg.strip())
|
|
||||||
callback(msg)
|
|
||||||
lines.append(msg)
|
|
||||||
stderr = ''.join(lines)
|
|
||||||
|
|
||||||
if check and proc.returncode != 0:
|
|
||||||
raise CalledProcessError(proc.returncode, args, output=None, stderr=stderr)
|
|
||||||
return CompletedProcess(args, proc.returncode, None, stderr=stderr)
|
|
||||||
|
|
||||||
|
|
||||||
def _fix_process_args(args, env, kwargs):
|
def _fix_process_args(args, env, kwargs):
|
||||||
|
|||||||
@@ -137,16 +137,7 @@ def run_ocrmypdf(input_file, output_file, *args, text=True):
|
|||||||
+ [str(input_file), str(output_file)]
|
+ [str(input_file), str(output_file)]
|
||||||
)
|
)
|
||||||
|
|
||||||
# Tell subprocess where to find coverage.py configuration
|
|
||||||
# This has no unless except when coverage is running
|
|
||||||
# Details: https://coverage.readthedocs.io/en/coverage-5.0/subprocess.html
|
|
||||||
coverage_rc = Path(__file__).parent.parent / '.coveragerc'
|
|
||||||
env = os.environ.copy()
|
env = os.environ.copy()
|
||||||
if coverage_rc.exists():
|
|
||||||
env['COVERAGE_PROCESS_START'] = os.fspath(coverage_rc)
|
|
||||||
elif not running_in_docker():
|
|
||||||
assert False, "could not find .coveragerc"
|
|
||||||
|
|
||||||
p = run(
|
p = run(
|
||||||
p_args,
|
p_args,
|
||||||
stdout=PIPE,
|
stdout=PIPE,
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# © 2021 James R. Barlow: github.com/jbarlow83
|
||||||
|
#
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from ocrmypdf._sync import configure_debug_logging
|
||||||
|
|
||||||
|
|
||||||
|
def test_debug_logging(tmp_path):
|
||||||
|
# Just exercise the debug logger but don't validate it
|
||||||
|
# See https://github.com/pytest-dev/pytest/issues/5502 for pytest logging quirks
|
||||||
|
prefix = 'test_debug_logging'
|
||||||
|
log = logging.getLogger(prefix)
|
||||||
|
handler = configure_debug_logging(tmp_path / 'test.log', prefix)
|
||||||
|
log.info("test message")
|
||||||
|
log.removeHandler(handler)
|
||||||
+42
-21
@@ -4,28 +4,31 @@
|
|||||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
|
||||||
import pickle
|
import pickle
|
||||||
|
from io import BytesIO
|
||||||
from math import isclose
|
from math import isclose
|
||||||
|
|
||||||
import img2pdf
|
import img2pdf
|
||||||
import pikepdf
|
import pikepdf
|
||||||
import pytest
|
import pytest
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
from reportlab.lib.units import inch
|
||||||
from reportlab.pdfgen.canvas import Canvas
|
from reportlab.pdfgen.canvas import Canvas
|
||||||
|
|
||||||
from ocrmypdf import pdfinfo
|
from ocrmypdf import pdfinfo
|
||||||
|
from ocrmypdf.exceptions import InputFileError
|
||||||
from ocrmypdf.pdfinfo import Colorspace, Encoding
|
from ocrmypdf.pdfinfo import Colorspace, Encoding
|
||||||
|
from ocrmypdf.pdfinfo.layout import PDFPage
|
||||||
|
|
||||||
# pylint: disable=protected-access
|
# pylint: disable=protected-access
|
||||||
|
|
||||||
|
|
||||||
def test_single_page_text(outdir):
|
def test_single_page_text(outdir):
|
||||||
filename = outdir / 'text.pdf'
|
filename = outdir / 'text.pdf'
|
||||||
pdf = Canvas(str(filename), pagesize=(8 * 72, 6 * 72))
|
pdf = Canvas(str(filename), pagesize=(8 * inch, 6 * inch))
|
||||||
text = pdf.beginText()
|
text = pdf.beginText()
|
||||||
text.setFont('Helvetica', 12)
|
text.setFont('Helvetica', 12)
|
||||||
text.setTextOrigin(1 * 72, 3 * 72)
|
text.setTextOrigin(1 * inch, 3 * inch)
|
||||||
text.textLine(
|
text.textLine(
|
||||||
"Methink'st thou art a general offence and every" " man should beat thee."
|
"Methink'st thou art a general offence and every" " man should beat thee."
|
||||||
)
|
)
|
||||||
@@ -42,25 +45,32 @@ def test_single_page_text(outdir):
|
|||||||
assert len(page.images) == 0
|
assert len(page.images) == 0
|
||||||
|
|
||||||
|
|
||||||
def test_single_page_image(outdir):
|
@pytest.fixture(scope='session')
|
||||||
filename = outdir / 'image-mono.pdf'
|
def eight_by_eight():
|
||||||
|
|
||||||
im_tmp = outdir / 'tmp.png'
|
|
||||||
im = Image.new('1', (8, 8), 0)
|
im = Image.new('1', (8, 8), 0)
|
||||||
for n in range(8):
|
for n in range(8):
|
||||||
im.putpixel((n, n), 1)
|
im.putpixel((n, n), 1)
|
||||||
im.save(str(im_tmp), format='PNG')
|
return im
|
||||||
|
|
||||||
|
|
||||||
|
def test_single_page_image(eight_by_eight, outpdf):
|
||||||
|
im = eight_by_eight
|
||||||
|
bio = BytesIO()
|
||||||
|
im.save(bio, format='PNG')
|
||||||
|
bio.seek(0)
|
||||||
|
|
||||||
imgsize = ((img2pdf.ImgSize.dpi, 8), (img2pdf.ImgSize.dpi, 8))
|
imgsize = ((img2pdf.ImgSize.dpi, 8), (img2pdf.ImgSize.dpi, 8))
|
||||||
layout_fun = img2pdf.get_layout_fun(None, imgsize, None, None, None)
|
layout_fun = img2pdf.get_layout_fun(None, imgsize, None, None, None)
|
||||||
|
|
||||||
im_bytes = im_tmp.read_bytes()
|
with outpdf.open('wb') as f:
|
||||||
pdf_bytes = img2pdf.convert(
|
img2pdf.convert(
|
||||||
im_bytes, producer="img2pdf", with_pdfrw=False, layout_fun=layout_fun
|
bio,
|
||||||
)
|
producer="img2pdf",
|
||||||
filename.write_bytes(pdf_bytes)
|
with_pdfrw=False,
|
||||||
|
layout_fun=layout_fun,
|
||||||
info = pdfinfo.PdfInfo(filename)
|
outputstream=f,
|
||||||
|
)
|
||||||
|
info = pdfinfo.PdfInfo(outpdf)
|
||||||
|
|
||||||
assert len(info) == 1
|
assert len(info) == 1
|
||||||
page = info[0]
|
page = info[0]
|
||||||
@@ -77,16 +87,12 @@ def test_single_page_image(outdir):
|
|||||||
assert isclose(pdfimage.dpi.y, 8)
|
assert isclose(pdfimage.dpi.y, 8)
|
||||||
|
|
||||||
|
|
||||||
def test_single_page_inline_image(outdir):
|
def test_single_page_inline_image(eight_by_eight, outdir):
|
||||||
filename = outdir / 'image-mono-inline.pdf'
|
filename = outdir / 'image-mono-inline.pdf'
|
||||||
pdf = Canvas(str(filename), pagesize=(8 * 72, 6 * 72))
|
pdf = Canvas(str(filename), pagesize=(8 * 72, 6 * 72))
|
||||||
|
|
||||||
im = Image.new('1', (8, 8), 0)
|
|
||||||
for n in range(8):
|
|
||||||
im.putpixel((n, n), 1)
|
|
||||||
|
|
||||||
# Draw image in a 72x72 pt or 1"x1" area
|
# Draw image in a 72x72 pt or 1"x1" area
|
||||||
pdf.drawInlineImage(im, 0, 0, width=72, height=72)
|
pdf.drawInlineImage(eight_by_eight, 0, 0, width=72, height=72)
|
||||||
pdf.showPage()
|
pdf.showPage()
|
||||||
pdf.save()
|
pdf.save()
|
||||||
|
|
||||||
@@ -179,3 +185,18 @@ def test_stack_abuse():
|
|||||||
with pytest.warns(None):
|
with pytest.warns(None):
|
||||||
with pytest.raises(RuntimeError):
|
with pytest.raises(RuntimeError):
|
||||||
pdfinfo.info._interpret_contents(stream)
|
pdfinfo.info._interpret_contents(stream)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pages_issue700(monkeypatch, resources):
|
||||||
|
def get_no_pages(*args, **kwargs):
|
||||||
|
return iter([])
|
||||||
|
|
||||||
|
monkeypatch.setattr(PDFPage, 'get_pages', get_no_pages)
|
||||||
|
|
||||||
|
with pytest.raises(InputFileError, match="pdfminer"):
|
||||||
|
pdfinfo.PdfInfo(
|
||||||
|
resources / 'cardinal.pdf',
|
||||||
|
detailed_analysis=True,
|
||||||
|
progbar=False,
|
||||||
|
max_workers=1,
|
||||||
|
)
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
# © 2021 James R. Barlow: github.com/jbarlow83
|
||||||
|
#
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
from unittest.mock import Mock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from PIL import Image
|
||||||
|
from reportlab.lib.units import inch
|
||||||
|
from reportlab.lib.utils import ImageReader
|
||||||
|
from reportlab.pdfgen.canvas import Canvas
|
||||||
|
|
||||||
|
from ocrmypdf import _pipeline, pdfinfo
|
||||||
|
from ocrmypdf.helpers import Resolution
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope='session')
|
||||||
|
def rgb_image():
|
||||||
|
im = Image.new('RGB', (8, 8))
|
||||||
|
im.putpixel((4, 4), (255, 0, 0))
|
||||||
|
im.putpixel((5, 5), (0, 255, 0))
|
||||||
|
im.putpixel((6, 6), (0, 0, 255))
|
||||||
|
return ImageReader(im)
|
||||||
|
|
||||||
|
|
||||||
|
DUMMY_OVERSAMPLE_RESOLUTION = Resolution(42.0, 42.0)
|
||||||
|
VECTOR_RESOLUTION = Resolution(_pipeline.VECTOR_PAGE_DPI, _pipeline.VECTOR_PAGE_DPI)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'image, text, vector, result',
|
||||||
|
[
|
||||||
|
(False, False, False, VECTOR_RESOLUTION),
|
||||||
|
(False, True, False, VECTOR_RESOLUTION),
|
||||||
|
(True, False, False, DUMMY_OVERSAMPLE_RESOLUTION),
|
||||||
|
(True, True, False, VECTOR_RESOLUTION),
|
||||||
|
(False, False, True, VECTOR_RESOLUTION),
|
||||||
|
(False, True, True, VECTOR_RESOLUTION),
|
||||||
|
(True, False, True, VECTOR_RESOLUTION),
|
||||||
|
(True, True, True, VECTOR_RESOLUTION),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_dpi_needed(image, text, vector, result, rgb_image, outdir):
|
||||||
|
|
||||||
|
c = Canvas(str(outdir / 'dpi.pdf'), pagesize=(5 * inch, 5 * inch))
|
||||||
|
if image:
|
||||||
|
c.drawImage(rgb_image, 1 * inch, 1 * inch, width=1 * inch, height=1 * inch)
|
||||||
|
if text:
|
||||||
|
c.drawString(1 * inch, 4 * inch, "Actual text")
|
||||||
|
if vector:
|
||||||
|
c.ellipse(3 * inch, 3 * inch, 4 * inch, 4 * inch)
|
||||||
|
c.showPage()
|
||||||
|
c.save()
|
||||||
|
|
||||||
|
mock = Mock()
|
||||||
|
mock.oversample = DUMMY_OVERSAMPLE_RESOLUTION[0]
|
||||||
|
|
||||||
|
pi = pdfinfo.PdfInfo(outdir / 'dpi.pdf')
|
||||||
|
|
||||||
|
assert _pipeline.get_canvas_square_dpi(pi[0], mock) == result
|
||||||
|
assert _pipeline.get_page_square_dpi(pi[0], mock) == result
|
||||||
@@ -45,7 +45,8 @@ def test_skip_pages_does_not_replicate(resources, basename, outdir):
|
|||||||
assert len(page.images) == 1, "skipped page was replicated"
|
assert len(page.images) == 1, "skipped page was replicated"
|
||||||
|
|
||||||
for n, info_out_n in enumerate(info):
|
for n, info_out_n in enumerate(info):
|
||||||
assert info_out_n.width_inches == info_in[n].width_inches
|
assert info_out_n.width_inches == info_in[n].width_inches, "output resized"
|
||||||
|
assert info_out_n.height_inches == info_in[n].height_inches, "output resized"
|
||||||
|
|
||||||
|
|
||||||
def test_content_preservation(resources, outpdf):
|
def test_content_preservation(resources, outpdf):
|
||||||
|
|||||||
Reference in New Issue
Block a user