Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
709c65b41a | ||
|
|
67f99c5bb7 | ||
|
|
d55e673d9c | ||
|
|
21b90d2d14 | ||
|
|
2def7e3392 | ||
|
|
b0dcaa7512 | ||
|
|
e8285b1d10 | ||
|
|
5ba56adb53 | ||
|
|
ca735278e0 | ||
|
|
b5ccbfdf25 | ||
|
|
8c35d6e6e4 | ||
|
|
d1e0c81eda | ||
|
|
10c8e4f8b4 | ||
|
|
6be2242c21 | ||
|
|
204c9d6ae1 | ||
|
|
6eb393590b | ||
|
|
07c6654057 | ||
|
|
4e15eb8d14 | ||
|
|
8b01ab8ad2 | ||
|
|
e0a522ad50 |
+41
-1
@@ -12,10 +12,50 @@ may be unreliable. Use the API to depend on precise behavior.
|
|||||||
The public API may be useful in scripts that launch OCRmyPDF processes or that
|
The public API may be useful in scripts that launch OCRmyPDF processes or that
|
||||||
wish to use some of its features for working with PDFs.
|
wish to use some of its features for working with PDFs.
|
||||||
|
|
||||||
|
v11.3.1
|
||||||
|
=======
|
||||||
|
|
||||||
|
- Declare support for new versions: pdfminer.six 20201018 and pikepdf 2.x
|
||||||
|
- Fix warning related to ``--pdfa-image-compression`` that appears at the wrong
|
||||||
|
time.
|
||||||
|
|
||||||
|
v11.3.0
|
||||||
|
=======
|
||||||
|
|
||||||
|
- The "OCR" step is describing as "Image processing" in the output messages when
|
||||||
|
OCR is disabled, to better explain the application's behavior.
|
||||||
|
- Debug logs are now only created when run as a command line, and not when OCR
|
||||||
|
is performed for an API call. It is the calling application's responsibility
|
||||||
|
to set up logging.
|
||||||
|
- For PDFs with a low number of pages, we gathered information about the input PDF
|
||||||
|
in a thread rather than process (when there are more pages). When run as a
|
||||||
|
thread, we did not close the file handle to the working PDF, leaking one file
|
||||||
|
handle per call of ``ocrmypdf.ocr``.
|
||||||
|
- Fixed an issue where debug messages send by child worker processes did not match
|
||||||
|
the log settings of parent process, causing messages to be dropped. This affected
|
||||||
|
macOS and Windows only where the parent process is not forked.
|
||||||
|
- Fixed the hookspec of rasterize_pdf_page to remove default parameters that
|
||||||
|
were not handled in an expected way by pluggy.
|
||||||
|
- Fixed another issue with automatic page rotation (#658) due to the issue above.
|
||||||
|
|
||||||
|
v11.2.1
|
||||||
|
=======
|
||||||
|
|
||||||
|
- Fixed an issue where optimization of a 1-bit image with a color palette or
|
||||||
|
associated ICC that was optimized to JBIG2 could have its colors inverted.
|
||||||
|
|
||||||
|
v11.2.0
|
||||||
|
=======
|
||||||
|
|
||||||
|
- Fixed an issue with optimizing PNG-type images that had soft masks or image masks.
|
||||||
|
This is a regression introduced in (or about) v11.1.0.
|
||||||
|
- Improved type checking of the ``plugins`` parameter for the ``ocrmypdf.ocr``
|
||||||
|
API call.
|
||||||
|
|
||||||
v11.1.2
|
v11.1.2
|
||||||
=======
|
=======
|
||||||
|
|
||||||
- Fix hOCR renderer writing the text in roughly reverse order. This should not
|
- Fixed hOCR renderer writing the text in roughly reverse order. This should not
|
||||||
affect reasonably smart PDF readers that properly locate the position of all
|
affect reasonably smart PDF readers that properly locate the position of all
|
||||||
text, but may confuse those that rely on the order of objects in the content
|
text, but may confuse those that rely on the order of objects in the content
|
||||||
stream. (#642)
|
stream. (#642)
|
||||||
|
|||||||
@@ -18,6 +18,25 @@
|
|||||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
# SOFTWARE.
|
# SOFTWARE.
|
||||||
|
|
||||||
|
"""
|
||||||
|
An example of an OCRmyPDF plugin.
|
||||||
|
|
||||||
|
This plugin adds two new command line arguments
|
||||||
|
--grayscale-ocr: converts the image to grayscale before performing OCR on it
|
||||||
|
(This is occasionally useful for images whose color confounds OCR. It only
|
||||||
|
affects the image shown to OCR. The image is not saved.)
|
||||||
|
--mono-page: converts pages all pages in the output file to black and white
|
||||||
|
|
||||||
|
To use this from the command line:
|
||||||
|
ocrmypdf --plugin path/to/example_plugin.py --mono-page input.pdf output.pdf
|
||||||
|
|
||||||
|
To use this as an API:
|
||||||
|
import ocrmypdf
|
||||||
|
ocrmypdf.ocr('input.pdf', 'output.pdf',
|
||||||
|
plugins=['path/to/example_plugin.py'], mono_page=True
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|||||||
@@ -63,7 +63,6 @@ setup(
|
|||||||
python_requires=' >= 3.6',
|
python_requires=' >= 3.6',
|
||||||
setup_requires=[ # can be removed whenever we can drop pip 9 support
|
setup_requires=[ # can be removed whenever we can drop pip 9 support
|
||||||
'cffi >= 1.9.1', # to build the leptonica module
|
'cffi >= 1.9.1', # to build the leptonica module
|
||||||
'pytest-runner', # to enable python setup.py test
|
|
||||||
'setuptools_scm', # so that version will work
|
'setuptools_scm', # so that version will work
|
||||||
'setuptools_scm_git_archive', # enable version from github tarballs
|
'setuptools_scm_git_archive', # enable version from github tarballs
|
||||||
],
|
],
|
||||||
@@ -73,10 +72,10 @@ setup(
|
|||||||
'cffi >= 1.9.1', # must be a setup and install requirement
|
'cffi >= 1.9.1', # must be a setup and install requirement
|
||||||
'coloredlogs >= 14.0', # strictly optional
|
'coloredlogs >= 14.0', # strictly optional
|
||||||
'img2pdf >= 0.3.0, < 0.5', # pure Python, so track HEAD closely
|
'img2pdf >= 0.3.0, < 0.5', # pure Python, so track HEAD closely
|
||||||
'pdfminer.six >= 20191110, != 20200720, <= 20200726',
|
'pdfminer.six >= 20191110, != 20200720, <= 20201018',
|
||||||
'pikepdf >= 1.14.0, < 2',
|
'pikepdf >= 1.14.0, < 3',
|
||||||
'Pillow >= 7.0.0',
|
'Pillow >= 7.0.0',
|
||||||
'pluggy >= 0.13.0',
|
'pluggy >= 0.13.0, < 1.0',
|
||||||
'reportlab >= 3.3.0', # oldest released version with sane image handling
|
'reportlab >= 3.3.0', # oldest released version with sane image handling
|
||||||
'tqdm >= 4',
|
'tqdm >= 4',
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ def process_sigbus(*args):
|
|||||||
raise InputFileError("A worker process lost access to an input file")
|
raise InputFileError("A worker process lost access to an input file")
|
||||||
|
|
||||||
|
|
||||||
def process_init(queue, user_init):
|
def process_init(queue, user_init, loglevel):
|
||||||
"""Initialize a process pool worker"""
|
"""Initialize a process pool worker"""
|
||||||
|
|
||||||
# Ignore SIGINT (our parent process will kill us gracefully)
|
# Ignore SIGINT (our parent process will kill us gracefully)
|
||||||
@@ -62,6 +62,7 @@ def process_init(queue, user_init):
|
|||||||
# Reconfigure the root logger for this process to send all messages to a queue
|
# Reconfigure the root logger for this process to send all messages to a queue
|
||||||
h = logging.handlers.QueueHandler(queue)
|
h = logging.handlers.QueueHandler(queue)
|
||||||
root = logging.getLogger()
|
root = logging.getLogger()
|
||||||
|
root.setLevel(loglevel)
|
||||||
root.handlers = []
|
root.handlers = []
|
||||||
root.addHandler(h)
|
root.addHandler(h)
|
||||||
|
|
||||||
@@ -69,7 +70,7 @@ def process_init(queue, user_init):
|
|||||||
user_init()
|
user_init()
|
||||||
|
|
||||||
|
|
||||||
def thread_init(_queue, user_init):
|
def thread_init(_queue, user_init, _loglevel):
|
||||||
# As a thread, block SIGBUS so the main thread deals with it...
|
# As a thread, block SIGBUS so the main thread deals with it...
|
||||||
if hasattr(signal, 'SIGBUS'):
|
if hasattr(signal, 'SIGBUS'):
|
||||||
signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGBUS})
|
signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGBUS})
|
||||||
@@ -102,7 +103,7 @@ def exec_progress_pool(
|
|||||||
pool = pool_class(
|
pool = pool_class(
|
||||||
processes=max_workers,
|
processes=max_workers,
|
||||||
initializer=initializer,
|
initializer=initializer,
|
||||||
initargs=(log_queue, task_initializer),
|
initargs=(log_queue, task_initializer, logging.getLogger("").level),
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
results = pool.imap_unordered(task, task_arguments)
|
results = pool.imap_unordered(task, task_arguments)
|
||||||
|
|||||||
@@ -81,8 +81,8 @@ def rasterize_pdf(
|
|||||||
raster_device: str,
|
raster_device: str,
|
||||||
raster_dpi: Resolution,
|
raster_dpi: Resolution,
|
||||||
pageno: int = 1,
|
pageno: int = 1,
|
||||||
page_dpi: Resolution = None,
|
page_dpi: Optional[Resolution] = None,
|
||||||
rotation: int = None,
|
rotation: Optional[int] = None,
|
||||||
filter_vector: bool = False,
|
filter_vector: bool = False,
|
||||||
):
|
):
|
||||||
"""Rasterize one page of a PDF at resolution raster_dpi in canvas units."""
|
"""Rasterize one page of a PDF at resolution raster_dpi in canvas units."""
|
||||||
|
|||||||
+15
-17
@@ -109,6 +109,7 @@ class OcrGrafter:
|
|||||||
if textpdf and not self.font:
|
if textpdf and not self.font:
|
||||||
self.font, self.font_key = self._find_font(textpdf)
|
self.font, self.font_key = self._find_font(textpdf)
|
||||||
|
|
||||||
|
emplaced_page = False
|
||||||
content_rotation = self.pdfinfo[pageno].rotation
|
content_rotation = self.pdfinfo[pageno].rotation
|
||||||
path_image = Path(image).resolve() if image else None
|
path_image = Path(image).resolve() if image else None
|
||||||
if path_image is not None and path_image != self.path_base:
|
if path_image is not None and path_image != self.path_base:
|
||||||
@@ -122,24 +123,21 @@ class OcrGrafter:
|
|||||||
local_image_page = self.pdf_base.pages[-1]
|
local_image_page = self.pdf_base.pages[-1]
|
||||||
self.pdf_base.pages[pageno].emplace(local_image_page)
|
self.pdf_base.pages[pageno].emplace(local_image_page)
|
||||||
del self.pdf_base.pages[-1]
|
del self.pdf_base.pages[-1]
|
||||||
# The pdf_image_page will always be created with any /Rotate applied
|
emplaced_page = True
|
||||||
# applied already
|
|
||||||
content_rotation = 0
|
|
||||||
|
|
||||||
if content_rotation != 0:
|
# Calculate if the text is misaligned compared to the content
|
||||||
# Text can be misaligned on a /Rotate'd page.
|
if emplaced_page:
|
||||||
# That is because we rasterize pages with /Rotate applied,
|
content_rotation = autorotate_correction
|
||||||
# so that the OCR image text is upright and comes back upright.
|
text_rotation = autorotate_correction
|
||||||
text_misaligned = (autorotate_correction - content_rotation) % 360
|
text_misaligned = (text_rotation - content_rotation) % 360
|
||||||
log.debug(
|
log.debug(
|
||||||
f"Text rotation: (autorotate, content) -> text misalignment = "
|
f"Text rotation: (text, autorotate, content) -> text misalignment = "
|
||||||
f"({autorotate_correction}, {content_rotation}) -> {text_misaligned}"
|
f"({text_rotation}, {autorotate_correction}, {content_rotation}) -> {text_misaligned}"
|
||||||
)
|
)
|
||||||
else:
|
|
||||||
text_misaligned = 0
|
|
||||||
|
|
||||||
if textpdf and self.font:
|
if textpdf and self.font:
|
||||||
# Graft the text layer onto this page, whether new or old
|
# Graft the text layer onto this page, whether new or old, possibly
|
||||||
|
# rotating the text layer by the amount is misaligned.
|
||||||
strip_old = self.context.options.redo_ocr
|
strip_old = self.context.options.redo_ocr
|
||||||
self._graft_text_layer(
|
self._graft_text_layer(
|
||||||
page_num=pageno + 1,
|
page_num=pageno + 1,
|
||||||
@@ -151,14 +149,14 @@ class OcrGrafter:
|
|||||||
strip_old_text=strip_old,
|
strip_old_text=strip_old,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Correct the page rotation
|
# Correct the overall page rotation if needed, now that the text and content
|
||||||
|
# are aligned
|
||||||
page_rotation = (content_rotation - autorotate_correction) % 360
|
page_rotation = (content_rotation - autorotate_correction) % 360
|
||||||
self.pdf_base.pages[pageno].Rotate = page_rotation
|
self.pdf_base.pages[pageno].Rotate = page_rotation
|
||||||
log.debug(
|
log.debug(
|
||||||
f"Page rotation: (content, auto) -> page = "
|
f"Page rotation: (content, auto) -> page = "
|
||||||
f"({content_rotation}, {autorotate_correction}) -> {page_rotation}"
|
f"({content_rotation}, {autorotate_correction}) -> {page_rotation}"
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.emplacements % MAX_REPLACE_PAGES == 0:
|
if self.emplacements % MAX_REPLACE_PAGES == 0:
|
||||||
self.save_and_reload()
|
self.save_and_reload()
|
||||||
|
|
||||||
|
|||||||
@@ -332,8 +332,10 @@ def rasterize_preview(input_file: Path, page_context: PageContext):
|
|||||||
output_file=output_file,
|
output_file=output_file,
|
||||||
raster_device='jpeggray',
|
raster_device='jpeggray',
|
||||||
raster_dpi=canvas_dpi,
|
raster_dpi=canvas_dpi,
|
||||||
page_dpi=page_dpi,
|
|
||||||
pageno=page_context.pageinfo.pageno + 1,
|
pageno=page_context.pageinfo.pageno + 1,
|
||||||
|
page_dpi=page_dpi,
|
||||||
|
rotation=0,
|
||||||
|
filter_vector=False,
|
||||||
)
|
)
|
||||||
return output_file
|
return output_file
|
||||||
|
|
||||||
@@ -433,7 +435,7 @@ def rasterize(
|
|||||||
|
|
||||||
device = colorspaces[device_idx]
|
device = colorspaces[device_idx]
|
||||||
|
|
||||||
log.debug(f"Rasterize with {device}")
|
log.debug(f"Rasterize with {device}, rotation {correction}")
|
||||||
|
|
||||||
# Produce the page image with square resolution or else deskew and OCR
|
# Produce the page image with square resolution or else deskew and OCR
|
||||||
# will not work properly.
|
# will not work properly.
|
||||||
@@ -534,6 +536,9 @@ def create_ocr_image(image: Path, page_context: PageContext):
|
|||||||
|
|
||||||
# Pillow requires integer DPI
|
# Pillow requires integer DPI
|
||||||
dpi = tuple(round(coord) for coord in im.info['dpi'])
|
dpi = tuple(round(coord) for coord in im.info['dpi'])
|
||||||
|
if page_context.pageinfo.rotation != 0:
|
||||||
|
log.info(f"Rotating {page_context.pageinfo.rotation}")
|
||||||
|
im = im.rotate(page_context.pageinfo.rotation)
|
||||||
im.save(output_file, dpi=dpi)
|
im.save(output_file, dpi=dpi)
|
||||||
return output_file
|
return output_file
|
||||||
|
|
||||||
|
|||||||
+16
-11
@@ -209,9 +209,10 @@ def exec_page_sync(page_context: PageContext):
|
|||||||
if options.pdf_renderer == 'hocr':
|
if options.pdf_renderer == 'hocr':
|
||||||
(hocr_out, text_out) = ocr_engine_hocr(ocr_image_out, page_context)
|
(hocr_out, text_out) = ocr_engine_hocr(ocr_image_out, page_context)
|
||||||
ocr_out = render_hocr_page(hocr_out, page_context)
|
ocr_out = render_hocr_page(hocr_out, page_context)
|
||||||
|
elif options.pdf_renderer == 'sandwich':
|
||||||
if options.pdf_renderer == 'sandwich':
|
|
||||||
(ocr_out, text_out) = ocr_engine_textonly_pdf(ocr_image_out, page_context)
|
(ocr_out, text_out) = ocr_engine_textonly_pdf(ocr_image_out, page_context)
|
||||||
|
else:
|
||||||
|
raise NotImplementedError(f"pdf_renderer {options.pdf_renderer}")
|
||||||
|
|
||||||
return PageResult(
|
return PageResult(
|
||||||
pageno=page_context.pageno,
|
pageno=page_context.pageno,
|
||||||
@@ -244,7 +245,8 @@ def exec_concurrent(context: PdfContext):
|
|||||||
"""Execute the pipeline concurrently"""
|
"""Execute the pipeline concurrently"""
|
||||||
|
|
||||||
# Run exec_page_sync on every page context
|
# Run exec_page_sync on every page context
|
||||||
max_workers = min(len(context.pdfinfo), context.options.jobs)
|
options = context.options
|
||||||
|
max_workers = min(len(context.pdfinfo), options.jobs)
|
||||||
if max_workers > 1:
|
if max_workers > 1:
|
||||||
log.info("Start processing %d pages concurrently", max_workers)
|
log.info("Start processing %d pages concurrently", max_workers)
|
||||||
|
|
||||||
@@ -267,14 +269,14 @@ def exec_concurrent(context: PdfContext):
|
|||||||
tls.pageno = None
|
tls.pageno = None
|
||||||
|
|
||||||
exec_progress_pool(
|
exec_progress_pool(
|
||||||
use_threads=context.options.use_threads,
|
use_threads=options.use_threads,
|
||||||
max_workers=max_workers,
|
max_workers=max_workers,
|
||||||
tqdm_kwargs=dict(
|
tqdm_kwargs=dict(
|
||||||
total=(2 * len(context.pdfinfo)),
|
total=(2 * len(context.pdfinfo)),
|
||||||
desc='OCR',
|
desc='OCR' if options.tesseract_timeout > 0 else 'Image processing',
|
||||||
unit='page',
|
unit='page',
|
||||||
unit_scale=0.5,
|
unit_scale=0.5,
|
||||||
disable=not context.options.progress_bar,
|
disable=not options.progress_bar,
|
||||||
),
|
),
|
||||||
task_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
|
task_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
|
||||||
task=exec_page_sync,
|
task=exec_page_sync,
|
||||||
@@ -283,10 +285,10 @@ def exec_concurrent(context: PdfContext):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Output sidecar text
|
# Output sidecar text
|
||||||
if context.options.sidecar:
|
if options.sidecar:
|
||||||
text = merge_sidecars(sidecars, context)
|
text = merge_sidecars(sidecars, context)
|
||||||
# Copy text file to destination
|
# Copy text file to destination
|
||||||
copy_final(text, context.options.sidecar, context)
|
copy_final(text, options.sidecar, context)
|
||||||
|
|
||||||
# Merge layers to one single pdf
|
# Merge layers to one single pdf
|
||||||
pdf = ocrgraft.finalize()
|
pdf = ocrgraft.finalize()
|
||||||
@@ -296,7 +298,7 @@ def exec_concurrent(context: PdfContext):
|
|||||||
pdf = post_process(pdf, context)
|
pdf = post_process(pdf, context)
|
||||||
|
|
||||||
# Copy PDF file to destination
|
# Copy PDF file to destination
|
||||||
copy_final(pdf, context.options.output_file, context)
|
copy_final(pdf, options.output_file, context)
|
||||||
|
|
||||||
|
|
||||||
class NeverRaise(Exception):
|
class NeverRaise(Exception):
|
||||||
@@ -328,9 +330,12 @@ def run_pipeline(options, *, plugin_manager, api=False):
|
|||||||
|
|
||||||
work_folder = Path(mkdtemp(prefix="com.github.ocrmypdf."))
|
work_folder = Path(mkdtemp(prefix="com.github.ocrmypdf."))
|
||||||
debug_log_handler = None
|
debug_log_handler = None
|
||||||
if (options.keep_temporary_files or options.verbose >= 1) and not os.environ.get(
|
if (
|
||||||
'PYTEST_CURRENT_TEST', ''
|
(options.keep_temporary_files or options.verbose >= 1)
|
||||||
|
and not os.environ.get('PYTEST_CURRENT_TEST', '')
|
||||||
|
and not api
|
||||||
):
|
):
|
||||||
|
# Debug log for command line interface only with verbose output
|
||||||
debug_log_handler = configure_debug_logging(Path(work_folder) / "debug.log")
|
debug_log_handler = configure_debug_logging(Path(work_folder) / "debug.log")
|
||||||
|
|
||||||
pikepdf_enable_mmap()
|
pikepdf_enable_mmap()
|
||||||
|
|||||||
@@ -213,12 +213,12 @@ def check_options_optimizing(options):
|
|||||||
|
|
||||||
|
|
||||||
def check_options_advanced(options):
|
def check_options_advanced(options):
|
||||||
if options.pdfa_image_compression != 'auto' and options.output_type.startswith(
|
if options.pdfa_image_compression != 'auto' and not options.output_type.startswith(
|
||||||
'pdfa'
|
'pdfa'
|
||||||
):
|
):
|
||||||
log.warning(
|
log.warning(
|
||||||
"--pdfa-image-compression argument has no effect when "
|
"--pdfa-image-compression argument only applies when "
|
||||||
"--output-type is not 'pdfa', 'pdfa-1', or 'pdfa-2'"
|
"--output-type is one of 'pdfa', 'pdfa-1', or 'pdfa-2'"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+27
-9
@@ -11,6 +11,7 @@ import sys
|
|||||||
from enum import IntEnum
|
from enum import IntEnum
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import BinaryIO, Iterable, Union
|
from typing import BinaryIO, Iterable, Union
|
||||||
|
from warnings import warn
|
||||||
|
|
||||||
from ocrmypdf._logging import PageNumberFilter, TqdmConsole
|
from ocrmypdf._logging import PageNumberFilter, TqdmConsole
|
||||||
from ocrmypdf._plugin_manager import get_plugin_manager
|
from ocrmypdf._plugin_manager import get_plugin_manager
|
||||||
@@ -44,16 +45,28 @@ def configure_logging(
|
|||||||
):
|
):
|
||||||
"""Set up logging.
|
"""Set up logging.
|
||||||
|
|
||||||
Library users may wish to use this function if they want their log output to be
|
Before calling :func:`ocrmypdf.ocr()`, you can use this function to
|
||||||
similar to ocrmypdf command line interface. If not used, the external application
|
configure logging, if you want ocrmypdf's output to look like the ocrmypdf
|
||||||
should configure logging on its own.
|
command line interface. It will register log handlers, log filters, and
|
||||||
|
formatters, configure color logging to standard error, and adjust the log
|
||||||
|
levels of third party libraries. Details of this are fine-tuned and subject
|
||||||
|
to change. The ``verbosity`` argument is equivalent to the argument
|
||||||
|
``--verbose`` and applies those settings.
|
||||||
|
|
||||||
ocrmypdf will perform all of its logging under the ``"ocrmypdf"`` logging namespace.
|
If this function is not called, ocrmypdf will not configure logging, and it
|
||||||
In addition, ocrmypdf imports pdfminer, which logs under ``"pdfminer"``. A library
|
is up to the caller of ``ocrmypdf.ocr()`` to set up logging as it wishes using
|
||||||
user may wish to configure both; note that pdfminer is extremely chatty at the log
|
the Python standard library's logging module. If this function is called,
|
||||||
level ``logging.INFO``.
|
the caller may of course make further adjustments to logging.
|
||||||
|
|
||||||
Library users may perform additional configuration afterwards.
|
Regardless of whether this function is called, ocrmypdf will perform all of
|
||||||
|
its logging under the ``"ocrmypdf"`` logging namespace. In addition,
|
||||||
|
ocrmypdf imports pdfminer, which logs under ``"pdfminer"``. A library user
|
||||||
|
may wish to configure both; note that pdfminer is extremely chatty at the
|
||||||
|
log level ``logging.INFO``.
|
||||||
|
|
||||||
|
This function does not set up the ``debug.log`` log file that the command
|
||||||
|
line interface does at certain verbosity levels. Applications should configure
|
||||||
|
their own debug logging.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
verbosity (Verbosity): Verbosity level.
|
verbosity (Verbosity): Verbosity level.
|
||||||
@@ -226,7 +239,7 @@ def ocr( # pylint: disable=unused-argument
|
|||||||
user_words: os.PathLike = None,
|
user_words: os.PathLike = None,
|
||||||
user_patterns: os.PathLike = None,
|
user_patterns: os.PathLike = None,
|
||||||
fast_web_view: float = None,
|
fast_web_view: float = None,
|
||||||
plugins: Iterable[str] = None,
|
plugins: Iterable[Union[str, Path]] = None,
|
||||||
keep_temporary_files: bool = None,
|
keep_temporary_files: bool = None,
|
||||||
progress_bar: bool = None,
|
progress_bar: bool = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
@@ -280,6 +293,8 @@ def ocr( # pylint: disable=unused-argument
|
|||||||
"""
|
"""
|
||||||
if not plugins:
|
if not plugins:
|
||||||
plugins = []
|
plugins = []
|
||||||
|
elif isinstance(plugins, (str, Path)):
|
||||||
|
plugins = [plugins]
|
||||||
else:
|
else:
|
||||||
plugins = list(plugins)
|
plugins = list(plugins)
|
||||||
|
|
||||||
@@ -292,6 +307,9 @@ def ocr( # pylint: disable=unused-argument
|
|||||||
}
|
}
|
||||||
create_options_kwargs.update(kwargs)
|
create_options_kwargs.update(kwargs)
|
||||||
|
|
||||||
|
if 'verbose' in kwargs:
|
||||||
|
warn("ocrmypdf.ocr(verbose=) is ignored. Use ocrmypdf.configure_logging().")
|
||||||
|
|
||||||
options = create_options(**create_options_kwargs)
|
options = create_options(**create_options_kwargs)
|
||||||
check_options(options, _plugin_manager)
|
check_options(options, _plugin_manager)
|
||||||
return run_pipeline(options=options, plugin_manager=_plugin_manager, api=True)
|
return run_pipeline(options=options, plugin_manager=_plugin_manager, api=True)
|
||||||
|
|||||||
@@ -61,9 +61,9 @@ def rasterize_pdf_page(
|
|||||||
raster_device,
|
raster_device,
|
||||||
raster_dpi,
|
raster_dpi,
|
||||||
pageno,
|
pageno,
|
||||||
page_dpi=None,
|
page_dpi,
|
||||||
rotation=None,
|
rotation,
|
||||||
filter_vector=False,
|
filter_vector,
|
||||||
):
|
):
|
||||||
ghostscript.rasterize_pdf(
|
ghostscript.rasterize_pdf(
|
||||||
input_file,
|
input_file,
|
||||||
|
|||||||
@@ -121,9 +121,7 @@ def validate(pdfinfo, options):
|
|||||||
os.environ['OMP_THREAD_LIMIT'] = str(tess_threads)
|
os.environ['OMP_THREAD_LIMIT'] = str(tess_threads)
|
||||||
else:
|
else:
|
||||||
tess_threads = int(os.environ['OMP_THREAD_LIMIT'])
|
tess_threads = int(os.environ['OMP_THREAD_LIMIT'])
|
||||||
|
log.debug("Using Tesseract OpenMP thread limit %d", tess_threads)
|
||||||
if tess_threads > 1:
|
|
||||||
log.info("Using Tesseract OpenMP thread limit %d", tess_threads)
|
|
||||||
|
|
||||||
|
|
||||||
class TesseractOcrEngine(OcrEngine):
|
class TesseractOcrEngine(OcrEngine):
|
||||||
|
|||||||
@@ -77,23 +77,26 @@ def extract_image_filter(
|
|||||||
if image.Subtype != Name.Image:
|
if image.Subtype != Name.Image:
|
||||||
return None
|
return None
|
||||||
if image.Length < 100:
|
if image.Length < 100:
|
||||||
log.debug("Skipping small image, xref %s", xref)
|
log.debug(f"Skipping small image, xref {xref}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
pim = PdfImage(image)
|
pim = PdfImage(image)
|
||||||
|
|
||||||
if len(pim.filter_decodeparms) > 1:
|
if len(pim.filter_decodeparms) > 1:
|
||||||
log.debug("Skipping multiply filtered, xref %s", xref)
|
log.debug(f"Skipping multiply filtered image, xref {xref}")
|
||||||
return None
|
return None
|
||||||
filtdp = pim.filter_decodeparms[0]
|
filtdp = pim.filter_decodeparms[0]
|
||||||
|
|
||||||
if pim.bits_per_component > 8:
|
if pim.bits_per_component > 8:
|
||||||
|
log.debug(f"Skipping wide gamut image, xref {xref}")
|
||||||
return None # Don't mess with wide gamut images
|
return None # Don't mess with wide gamut images
|
||||||
|
|
||||||
if filtdp[0] == Name.JPXDecode:
|
if filtdp[0] == Name.JPXDecode:
|
||||||
|
log.debug(f"Skipping JPEG2000 iamge, xref {xref}")
|
||||||
return None # Don't do JPEG2000
|
return None # Don't do JPEG2000
|
||||||
|
|
||||||
if Name.Decode in image:
|
if Name.Decode in image:
|
||||||
|
log.debug(f"Skipping image with Decode table, xref {xref}")
|
||||||
return None # Don't mess with custom Decode tables
|
return None # Don't mess with custom Decode tables
|
||||||
|
|
||||||
return pim, filtdp
|
return pim, filtdp
|
||||||
@@ -113,12 +116,23 @@ def extract_image_jbig2(
|
|||||||
and jbig2enc.available()
|
and jbig2enc.available()
|
||||||
):
|
):
|
||||||
try:
|
try:
|
||||||
|
# Save any colorspace associated with the image, so that we
|
||||||
|
# will export a pure 1-bit PNG with no palette or ICC profile.
|
||||||
|
# Showing the palette or ICC to jbig2enc will cause it to perform
|
||||||
|
# colorspace transform to 1bpp, which will conflict the palette or
|
||||||
|
# ICC if it exists.
|
||||||
|
colorspace = pim.obj.ColorSpace
|
||||||
|
# Set to DeviceGray temporarily; we already in 1 bpc.
|
||||||
|
pim.obj.ColorSpace = pikepdf.Name.DeviceGray
|
||||||
imgname = root / f'{xref:08d}'
|
imgname = root / f'{xref:08d}'
|
||||||
with imgname.open('wb') as f:
|
with imgname.open('wb') as f:
|
||||||
ext = pim.extract_to(stream=f)
|
ext = pim.extract_to(stream=f)
|
||||||
imgname.rename(imgname.with_suffix(ext))
|
imgname.rename(imgname.with_suffix(ext))
|
||||||
except pikepdf.UnsupportedImageTypeError:
|
except pikepdf.UnsupportedImageTypeError:
|
||||||
return None
|
return None
|
||||||
|
finally:
|
||||||
|
# Restore image colorspace after temporarily setting it to DeviceGray
|
||||||
|
pim.obj.ColorSpace = colorspace
|
||||||
return XrefExt(xref, ext)
|
return XrefExt(xref, ext)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -229,7 +243,9 @@ def extract_images(
|
|||||||
# Ignore soft masks
|
# Ignore soft masks
|
||||||
smask_xref = Xref(image.SMask.objgen[0])
|
smask_xref = Xref(image.SMask.objgen[0])
|
||||||
exclude_xrefs.add(smask_xref)
|
exclude_xrefs.add(smask_xref)
|
||||||
|
log.debug(f"Skipping image {smask_xref} because it is an SMask")
|
||||||
include_xrefs.add(xref)
|
include_xrefs.add(xref)
|
||||||
|
log.debug(f"Treating {xref} as an optimization candidate")
|
||||||
if xref not in pageno_for_xref:
|
if xref not in pageno_for_xref:
|
||||||
pageno_for_xref[xref] = pageno
|
pageno_for_xref[xref] = pageno
|
||||||
|
|
||||||
@@ -411,9 +427,25 @@ def _transcode_png(pike: Pdf, filename: Path, xref: Xref) -> bool:
|
|||||||
decode_parms=local_image.DecodeParms,
|
decode_parms=local_image.DecodeParms,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Don't copy keys from the new image...
|
||||||
del_keys = set(im_obj.keys()) - set(local_image.keys())
|
del_keys = set(im_obj.keys()) - set(local_image.keys())
|
||||||
|
# ...except for the keep_fields, which are essential to displaying
|
||||||
|
# the image correctly and preserving its metadata. (/Decode arrays
|
||||||
|
# and /SMaskInData are implicitly discarded prior to this point.)
|
||||||
|
keep_fields = {
|
||||||
|
'/ID',
|
||||||
|
'/Intent',
|
||||||
|
'/Interpolate',
|
||||||
|
'/Mask',
|
||||||
|
'/Metadata',
|
||||||
|
'/OC',
|
||||||
|
'/OPI',
|
||||||
|
'/SMask',
|
||||||
|
'/StructParent',
|
||||||
|
}
|
||||||
|
del_keys -= keep_fields
|
||||||
for key in local_image.keys():
|
for key in local_image.keys():
|
||||||
if key != Name.Length:
|
if key != Name.Length and str(key) not in keep_fields:
|
||||||
im_obj[key] = local_image[key]
|
im_obj[key] = local_image[key]
|
||||||
for key in del_keys:
|
for key in del_keys:
|
||||||
del im_obj[key]
|
del im_obj[key]
|
||||||
@@ -581,7 +613,7 @@ def optimize(input_file: Path, output_file: Path, context, save_settings) -> Non
|
|||||||
)
|
)
|
||||||
ratio = input_size / output_size
|
ratio = input_size / output_size
|
||||||
savings = 1 - output_size / input_size
|
savings = 1 - output_size / input_size
|
||||||
log.info(f"Optimize ratio: {ratio:.2f} savings: {(100 * savings):.1f}%")
|
log.info(f"Optimize ratio: {ratio:.2f} savings: {(savings):.1%}")
|
||||||
|
|
||||||
if savings < 0:
|
if savings < 0:
|
||||||
log.info("Image optimization did not improve the file - discarded")
|
log.info("Image optimization did not improve the file - discarded")
|
||||||
|
|||||||
@@ -630,6 +630,9 @@ worker_pdf = None
|
|||||||
def _pdf_pageinfo_sync_init(infile):
|
def _pdf_pageinfo_sync_init(infile):
|
||||||
global worker_pdf # pylint: disable=global-statement
|
global worker_pdf # pylint: disable=global-statement
|
||||||
pikepdf_enable_mmap()
|
pikepdf_enable_mmap()
|
||||||
|
# If this function is called as a thread initializer, we need a messy hack
|
||||||
|
# to close worker_pdf. If called as a process, it will be released when the
|
||||||
|
# process is terminated.
|
||||||
worker_pdf = pikepdf.open(infile)
|
worker_pdf = pikepdf.open(infile)
|
||||||
|
|
||||||
|
|
||||||
@@ -643,6 +646,7 @@ def _pdf_pageinfo_sync(args):
|
|||||||
def _pdf_pageinfo_concurrent(
|
def _pdf_pageinfo_concurrent(
|
||||||
pdf, infile, progbar, max_workers, check_pages, detailed_analysis=False
|
pdf, infile, progbar, max_workers, check_pages, detailed_analysis=False
|
||||||
):
|
):
|
||||||
|
global worker_pdf # pylint: disable=global-statement
|
||||||
pages = [None] * len(pdf.pages)
|
pages = [None] * len(pdf.pages)
|
||||||
|
|
||||||
def update_pageinfo(result, pbar):
|
def update_pageinfo(result, pbar):
|
||||||
@@ -663,17 +667,23 @@ def _pdf_pageinfo_concurrent(
|
|||||||
# a separate process.
|
# a separate process.
|
||||||
use_threads = True
|
use_threads = True
|
||||||
|
|
||||||
exec_progress_pool(
|
try:
|
||||||
use_threads=use_threads,
|
exec_progress_pool(
|
||||||
max_workers=n_workers,
|
use_threads=use_threads,
|
||||||
tqdm_kwargs=dict(
|
max_workers=n_workers,
|
||||||
total=total, desc="Scanning contents", unit='page', disable=not progbar
|
tqdm_kwargs=dict(
|
||||||
),
|
total=total, desc="Scanning contents", unit='page', disable=not progbar
|
||||||
task_initializer=partial(_pdf_pageinfo_sync_init, infile),
|
),
|
||||||
task=_pdf_pageinfo_sync,
|
task_initializer=partial(_pdf_pageinfo_sync_init, infile),
|
||||||
task_arguments=contexts,
|
task=_pdf_pageinfo_sync,
|
||||||
task_finished=update_pageinfo,
|
task_arguments=contexts,
|
||||||
)
|
task_finished=update_pageinfo,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
if worker_pdf and use_threads:
|
||||||
|
assert n_workers == 1, "Should have only one worker when threaded"
|
||||||
|
# This is messy, but if we ran in thread, close worker_pdf
|
||||||
|
worker_pdf.close()
|
||||||
return pages
|
return pages
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -89,9 +89,9 @@ def rasterize_pdf_page(
|
|||||||
raster_device: str,
|
raster_device: str,
|
||||||
raster_dpi: Resolution,
|
raster_dpi: Resolution,
|
||||||
pageno: int,
|
pageno: int,
|
||||||
page_dpi: Optional[Resolution] = None,
|
page_dpi: Optional[Resolution],
|
||||||
rotation: Optional[int] = None,
|
rotation: Optional[int],
|
||||||
filter_vector: bool = False,
|
filter_vector: bool,
|
||||||
) -> Path:
|
) -> Path:
|
||||||
"""Rasterize one page of a PDF at resolution raster_dpi in canvas units.
|
"""Rasterize one page of a PDF at resolution raster_dpi in canvas units.
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ from PIL import Image
|
|||||||
|
|
||||||
from ocrmypdf import leptonica
|
from ocrmypdf import leptonica
|
||||||
from ocrmypdf._exec import ghostscript, tesseract
|
from ocrmypdf._exec import ghostscript, tesseract
|
||||||
|
from ocrmypdf._plugin_manager import get_plugin_manager
|
||||||
from ocrmypdf.helpers import Resolution
|
from ocrmypdf.helpers import Resolution
|
||||||
from ocrmypdf.pdfinfo import PdfInfo
|
from ocrmypdf.pdfinfo import PdfInfo
|
||||||
|
|
||||||
@@ -256,3 +257,33 @@ def test_tesseract_orientation(resources, tmp_path):
|
|||||||
tesseract.get_orientation( # Test results of this are unreliable
|
tesseract.get_orientation( # Test results of this are unreliable
|
||||||
tmp_path / '000001.png', engine_mode='3', timeout=10
|
tmp_path / '000001.png', engine_mode='3', timeout=10
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_rasterize_rotates(resources, tmp_path):
|
||||||
|
pm = get_plugin_manager([])
|
||||||
|
|
||||||
|
img = tmp_path / 'img90.png'
|
||||||
|
pm.hook.rasterize_pdf_page(
|
||||||
|
input_file=resources / 'graph.pdf',
|
||||||
|
output_file=img,
|
||||||
|
raster_device='pngmono',
|
||||||
|
raster_dpi=Resolution(20, 20),
|
||||||
|
page_dpi=Resolution(20, 20),
|
||||||
|
pageno=1,
|
||||||
|
rotation=90,
|
||||||
|
filter_vector=False,
|
||||||
|
)
|
||||||
|
assert Image.open(img).size == (123, 151), "Image not rotated"
|
||||||
|
|
||||||
|
img = tmp_path / 'img180.png'
|
||||||
|
pm.hook.rasterize_pdf_page(
|
||||||
|
input_file=resources / 'graph.pdf',
|
||||||
|
output_file=img,
|
||||||
|
raster_device='pngmono',
|
||||||
|
raster_dpi=Resolution(20, 20),
|
||||||
|
page_dpi=Resolution(20, 20),
|
||||||
|
pageno=1,
|
||||||
|
rotation=180,
|
||||||
|
filter_vector=False,
|
||||||
|
)
|
||||||
|
assert Image.open(img).size == (151, 123), "Image not rotated"
|
||||||
|
|||||||
Reference in New Issue
Block a user