menekis/OCRmyPDF
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/gscan2pdf'

Browse Source 
Reconcile release notes and copy_final() with new pipeline.
This commit is contained in:
James R. Barlow
2023-10-30 00:01:28 -07:00
parent ae123fd209 c0b60dae6a
commit 580252a1a0
67 changed files with 2903 additions and 1105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ocrmypdf/__init__.py
+8 -1
View File
@@ -10,8 +10,11 @@ from pluggy import HookimplMarker as _HookimplMarker
from ocrmypdf import helpers, hocrtransform, pdfa, pdfinfo
from ocrmypdf._concurrent import Executor
from ocrmypdf._jobcontext import PageContext, PdfContext
from ocrmypdf._pipelines._common import (
configure_debug_logging,
)
from ocrmypdf._version import PROGRAM_NAME, __version__
from ocrmypdf.api import Verbosity, configure_logging, ocr
from ocrmypdf.api import Verbosity, configure_logging, hocr_to_ocr_pdf, ocr, pdf_to_hocr
from ocrmypdf.exceptions import (
BadArgsError,
DpiError,
@@ -30,9 +33,11 @@ from ocrmypdf.pluginspec import OcrEngine, OrientationConfidence
hookimpl = _HookimplMarker('ocrmypdf')
__all__ = [
'__version__',
'BadArgsError',
'configure_debug_logging',
'configure_logging',
'DpiError',
'EncryptedPdfError',
@@ -40,6 +45,7 @@ __all__ = [
'ExitCode',
'ExitCodeException',
'helpers',
'hocr_to_ocr_pdf',
'hocrtransform',
'hookimpl',
'InputFileError',
@@ -49,6 +55,7 @@ __all__ = [
'OrientationConfidence',
'OutputFileAccessError',
'PageContext',
'pdf_to_hocr',
'pdfa',
'PdfContext',
'pdfinfo',
src/ocrmypdf/__main__.py
+6 -2
View File
@@ -7,14 +7,15 @@
from __future__ import annotations
import logging
import multiprocessing
import os
import signal
import sys
from contextlib import suppress
from ocrmypdf import __version__
from ocrmypdf._pipelines.ocr import run_pipeline_cli
from ocrmypdf._plugin_manager import get_parser_options_plugins
from ocrmypdf._sync import run_pipeline
from ocrmypdf._validation import check_options
from ocrmypdf.api import Verbosity, configure_logging
from ocrmypdf.exceptions import (
@@ -71,9 +72,12 @@ def run(args=None):
with suppress(AttributeError, OSError):
signal.signal(signal.SIGBUS, sigbus)
result = run_pipeline(options=options, plugin_manager=plugin_manager)
result = run_pipeline_cli(options=options, plugin_manager=plugin_manager)
return result
if __name__ == '__main__':
multiprocessing.freeze_support()
if os.name == 'posix':
multiprocessing.set_start_method('forkserver')
sys.exit(run())
src/ocrmypdf/_concurrent.py
+14 -26
View File
@@ -8,29 +8,17 @@ from __future__ import annotations
import threading
from abc import ABC, abstractmethod
from collections.abc import Iterable
from typing import Callable
from typing import Callable, TypeVar
from ocrmypdf._progressbar import NullProgressBar, ProgressBar
T = TypeVar('T')
def _task_noop(*_args, **_kwargs):
return
class NullProgressBar:
"""Progress bar API that takes no actions."""
def __init__(self, **kwargs):
pass
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
return False
def update(self, _arg=None):
return
class Executor(ABC):
"""Abstract concurrent executor."""
@@ -46,11 +34,11 @@ class Executor(ABC):
*,
use_threads: bool,
max_workers: int,
tqdm_kwargs: dict,
progress_kwargs: dict,
worker_initializer: Callable | None = None,
task: Callable | None = None,
task: Callable[..., T] | None = None,
task_arguments: Iterable | None = None,
task_finished: Callable | None = None,
task_finished: Callable[[T, ProgressBar], None] | None = None,
) -> None:
"""Set up parallel execution and progress reporting.
@@ -60,7 +48,7 @@ class Executor(ABC):
heavily, and parallelizing it with threads is not expected to be
performant).
max_workers: The maximum number of workers that should be run.
tqdm_kwargs: Arguments to set up the progress bar.
progress_kwargs: Arguments to set up the progress bar.
worker_initializer: Called when a worker is initialized, in the worker's
execution context. If the child workers are processes, it must be
possible to marshall/pickle the worker initializer.
@@ -86,7 +74,7 @@ class Executor(ABC):
self._execute(
use_threads=use_threads,
max_workers=max_workers,
tqdm_kwargs=tqdm_kwargs,
progress_kwargs=progress_kwargs,
worker_initializer=worker_initializer,
task=task,
task_arguments=task_arguments,
@@ -99,7 +87,7 @@ class Executor(ABC):
*,
use_threads: bool,
max_workers: int,
tqdm_kwargs: dict,
progress_kwargs: dict,
worker_initializer: Callable,
task: Callable,
task_arguments: Iterable,
@@ -125,13 +113,13 @@ class SerialExecutor(Executor):
*,
use_threads: bool,
max_workers: int,
tqdm_kwargs: dict,
progress_kwargs: dict,
worker_initializer: Callable,
task: Callable,
task_arguments: Iterable,
task_finished: Callable,
): # pylint: disable=unused-argument
with self.pbar_class(**tqdm_kwargs) as pbar:
with self.pbar_class(**progress_kwargs) as pbar:
for args in task_arguments:
result = task(args)
result = task(*args)
task_finished(result, pbar)
src/ocrmypdf/_exec/jbig2enc.py
+2 -14
View File
@@ -25,7 +25,7 @@ def available():
return True
def convert_group(*, cwd, infiles, out_prefix, threshold):
def convert_group(cwd, infiles, out_prefix, threshold):
args = [
'jbig2',
'-b',
@@ -43,21 +43,9 @@ def convert_group(*, cwd, infiles, out_prefix, threshold):
return proc
def convert_group_mp(args):
return convert_group(
cwd=args[0], infiles=args[1], out_prefix=args[2], threshold=args[3]
)
def convert_single(*, cwd, infile, outfile, threshold):
def convert_single(cwd, infile, outfile, threshold):
args = ['jbig2', '--pdf', '-t', str(threshold), infile]
with open(outfile, 'wb') as fstdout:
proc = run(args, cwd=cwd, stdout=fstdout, stderr=PIPE)
proc.check_returncode()
return proc
def convert_single_mp(args):
return convert_single(
cwd=args[0], infile=args[1], outfile=args[2], threshold=args[3]
)
src/ocrmypdf/_exec/pngquant.py
+9 -21
View File
@@ -5,13 +5,10 @@
from __future__ import annotations
from contextlib import contextmanager
from io import BytesIO
from pathlib import Path
from subprocess import PIPE
from packaging.version import Version
from PIL import Image
from ocrmypdf.exceptions import MissingDependencyError
from ocrmypdf.subprocess import get_version, run
@@ -29,21 +26,16 @@ def available():
return True
@contextmanager
def input_as_png(input_file: Path):
if not input_file.name.endswith('.png'):
with Image.open(input_file) as im:
bio = BytesIO()
im.save(bio, format='png')
bio.seek(0)
yield bio
else:
with open(input_file, 'rb') as f:
yield f
def quantize(input_file: Path, output_file: Path, quality_min: int, quality_max: int):
with input_as_png(input_file) as input_stream:
"""Quantize a PNG image using pngquant.
Args:
input_file: Input PNG image
output_file: Output PNG image
quality_min: Minimum quality to use
quality_max: Maximum quality to use
"""
with open(input_file, 'rb') as input_stream:
args = [
'pngquant',
'--force',
@@ -58,7 +50,3 @@ def quantize(input_file: Path, output_file: Path, quality_min: int, quality_max:
if result.returncode == 0:
# input_file could be the same as output_file, so we defer the write
output_file.write_bytes(result.stdout)
def quantize_mp(args):
return quantize(*args)
src/ocrmypdf/_exec/tesseract.py
+2 -25
View File
@@ -14,7 +14,6 @@ from pathlib import Path
from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired
from packaging.version import Version
from PIL import Image
from ocrmypdf.exceptions import (
MissingDependencyError,
@@ -27,25 +26,6 @@ from ocrmypdf.subprocess import get_version, run
log = logging.getLogger(__name__)
HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<title></title>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
<meta name='ocr-system' content='tesseract 4.1.1' />
<meta name='ocr-capabilities'
content='ocr_page ocr_carea ocr_par ocr_line ocrx_word ocrp_wconf'/>
</head>
<body>
<div class='ocr_page' id='page_1'
title='image "_blank.png"; bbox 0 0 {0} {1}; ppageno 0'>
</div>
</body>
</html>
"""
TESSERACT_THRESHOLDING_METHODS: dict[str, int] = {
'auto': 0,
'otsu': 0,
@@ -285,14 +265,11 @@ def page_timedout(timeout: float) -> None:
def _generate_null_hocr(output_hocr: Path, output_text: Path, image: Path) -> None:
"""Produce a .hocr file that reports no text detected.
"""Produce an empty .hocr file.
Ensures page is the same size as the input image.
"""
with Image.open(image) as im:
w, h = im.size
output_hocr.write_text(HOCR_TEMPLATE.format(w, h), encoding='utf-8')
output_hocr.write_text('', encoding='utf-8')
output_text.write_text('[skipped page]', encoding='utf-8')
src/ocrmypdf/_exec/unpaper.py
+8 -40
View File
@@ -73,52 +73,20 @@ def version() -> Version:
return Version(get_version('unpaper'))
SUPPORTED_MODES = {'1', 'L', 'RGB'}
def _convert_image(im: Image.Image) -> tuple[Image.Image, bool]:
im_modified = False
if im.mode not in SUPPORTED_MODES:
log.info("Converting image to other colorspace")
try:
if im.mode == 'P' and len(im.getcolors()) == 2:
im = im.convert(mode='1')
else:
im = im.convert(mode='RGB')
except OSError as e:
raise MissingDependencyError(
"Could not convert image with type " + im.mode
) from e
else:
im_modified = True
if im.mode not in SUPPORTED_MODES:
raise MissingDependencyError(
"Failed to convert image to a supported format."
) from None
return im, im_modified
@contextmanager
def _setup_unpaper_io(input_file: Path) -> Iterator[tuple[Path, Path, Path]]:
with Image.open(input_file) as im:
if im.width * im.height >= UNPAPER_IMAGE_PIXEL_LIMIT:
raise UnpaperImageTooLargeError(w=im.width, h=im.height)
im, im_modified = _convert_image(im)
with TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
tmppath = Path(tmpdir)
if im_modified or input_file.suffix != '.png':
input_png = tmppath / 'input.png'
im.save(input_png, format='PNG')
else:
# No changes, PNG input, just use the file we already have
input_png = input_file
# unpaper can write .png too, but it seems to write them slowly
# adds a few seconds to test suite - so just use pnm
output_pnm = tmppath / 'output.pnm'
yield input_png, output_pnm, tmppath
with TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
tmppath = Path(tmpdir)
# No changes, PNG input, just use the file we already have
input_png = input_file
# unpaper can write .png too, but it seems to write them slowly
# adds a few seconds to test suite - so just use pnm
output_pnm = tmppath / 'output.pnm'
yield input_png, output_pnm, tmppath
def run_unpaper(
src/ocrmypdf/_graft.py
+2 -2
View File
@@ -248,10 +248,10 @@ class OcrGrafter:
# content may have a rotation applied. Wrap the text stream with a rotation
# so it will be oriented the same way as the rest of the page content.
# (Previous versions OCRmyPDF rotated the content layer to match the text.)
mediabox = [float(pdf_text.pages[0].MediaBox[v]) for v in range(4)]
mediabox = pdf_text.pages[0].mediabox
wt, ht = mediabox[2] - mediabox[0], mediabox[3] - mediabox[1]
mediabox = [float(base_page.MediaBox[v]) for v in range(4)]
mediabox = base_page.mediabox
wp, hp = mediabox[2] - mediabox[0], mediabox[3] - mediabox[1]
translate = PdfMatrix().translated(-wt / 2, -ht / 2)
src/ocrmypdf/_jobcontext.py
+6 -9
View File
@@ -6,8 +6,6 @@
from __future__ import annotations
import os
import shutil
import sys
from argparse import Namespace
from collections.abc import Iterator
from copy import copy
@@ -55,6 +53,12 @@ class PdfContext:
for n in range(npages):
yield PageContext(self, n)
def get_page_context_args(self) -> Iterator[tuple[PageContext]]:
"""Get all ``PageContext`` for this PDF packaged in tuple for args-splatting."""
npages = len(self.pdfinfo)
for n in range(npages):
yield (PageContext(self, n),)
class PageContext:
"""Holds our context for a page.
@@ -94,10 +98,3 @@ class PageContext:
if not isinstance(state['options'].output_file, (str, bytes, os.PathLike)):
state['options'].output_file = 'stream'
return state
def cleanup_working_files(work_folder: Path, options: Namespace):
if options.keep_temporary_files:
print(f"Temporary working files retained at:\n{work_folder}", file=sys.stderr)
else:
shutil.rmtree(work_folder, ignore_errors=True)
src/ocrmypdf/_logging.py
-62
View File
@@ -9,15 +9,6 @@ import logging
from rich.console import Console
from rich.logging import RichHandler
from rich.progress import (
BarColumn,
MofNCompleteColumn,
Progress,
TaskProgressColumn,
TextColumn,
TimeRemainingColumn,
)
from rich.table import Column
class PageNumberFilter(logging.Filter):
@@ -37,56 +28,3 @@ class RichLoggingHandler(RichHandler):
super().__init__(
console=console, show_level=False, show_time=False, markup=True, **kwargs
)
class RichTqdmProgressAdapter:
"""Adapt tqdm API to rich progress bar."""
def __init__(
self,
*,
console: Console,
desc: str,
total: float | None = None,
unit: str | None = None,
unit_scale: float | None = 1.0,
disable: bool = False,
**kwargs,
):
self.progress = Progress(
TextColumn(
"[progress.description]{task.description}",
table_column=Column(min_width=20),
),
BarColumn(),
TaskProgressColumn(),
MofNCompleteColumn(),
TimeRemainingColumn(),
console=console,
auto_refresh=True,
redirect_stderr=True,
redirect_stdout=False,
disable=disable,
**kwargs,
)
self.unit_scale = unit_scale
self.progress_bar = self.progress.add_task(
desc,
total=total * self.unit_scale
if total is not None and self.unit_scale is not None
else None,
unit=unit,
)
def __enter__(self):
self.progress.start()
return self
def __exit__(self, exc_type, exc_value, traceback):
self.progress.refresh()
self.progress.stop()
return False
def update(self, value=None):
advance = self.unit_scale if value is None else value
self.progress.update(self.progress_bar, advance=advance)
src/ocrmypdf/_metadata.py
+184
View File
@@ -0,0 +1,184 @@
# SPDX-FileCopyrightText: 2023 James R. Barlow
# SPDX-License-Identifier: MPL-2.0
"""OCRmyPDF page processing pipeline functions."""
from __future__ import annotations
import logging
import os
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from pikepdf import Dictionary, Name, Pdf
from pikepdf import __version__ as PIKEPDF_VERSION
from pikepdf.models.metadata import PdfMetadata, encode_pdf_date
from ocrmypdf._jobcontext import PdfContext
from ocrmypdf._version import PROGRAM_NAME
from ocrmypdf._version import __version__ as OCRMYPF_VERSION
from ocrmypdf.languages import iso_639_2_from_3
log = logging.getLogger(__name__)
def get_docinfo(base_pdf: Pdf, context: PdfContext) -> dict[str, str]:
"""Read the document info and store it in a dictionary."""
options = context.options
def from_document_info(key):
try:
s = base_pdf.docinfo[key]
return str(s)
except (KeyError, TypeError):
return ''
pdfmark = {
k: from_document_info(k)
for k in ('/Title', '/Author', '/Keywords', '/Subject', '/CreationDate')
}
if options.title:
pdfmark['/Title'] = options.title
if options.author:
pdfmark['/Author'] = options.author
if options.keywords:
pdfmark['/Keywords'] = options.keywords
if options.subject:
pdfmark['/Subject'] = options.subject
creator_tag = context.plugin_manager.hook.get_ocr_engine().creator_tag(options)
pdfmark['/Creator'] = f'{PROGRAM_NAME} {OCRMYPF_VERSION} / {creator_tag}'
pdfmark['/Producer'] = f'pikepdf {PIKEPDF_VERSION}'
pdfmark['/ModDate'] = encode_pdf_date(datetime.now(timezone.utc))
return pdfmark
def report_on_metadata(options, missing):
if not missing:
return
if options.output_type.startswith('pdfa'):
log.warning(
"Some input metadata could not be copied because it is not "
"permitted in PDF/A. You may wish to examine the output "
"PDF's XMP metadata."
)
log.debug("The following metadata fields were not copied: %r", missing)
else:
log.error(
"Some input metadata could not be copied."
"You may wish to examine the output PDF's XMP metadata."
)
log.info("The following metadata fields were not copied: %r", missing)
def repair_docinfo_nuls(pdf):
"""If the DocumentInfo block contains NUL characters, remove them.
If the DocumentInfo block is malformed, log an error and continue.
"""
modified = False
try:
if not isinstance(pdf.docinfo, Dictionary):
raise TypeError("DocumentInfo is not a dictionary")
for k, v in pdf.docinfo.items():
if isinstance(v, str) and b'\x00' in bytes(v):
pdf.docinfo[k] = bytes(v).replace(b'\x00', b'')
modified = True
except TypeError:
# TypeError can also be raised if dictionary items are unexpected types
log.error("File contains a malformed DocumentInfo block - continuing anyway.")
return modified
def should_linearize(working_file: Path, context: PdfContext) -> bool:
"""Determine whether the PDF should be linearized.
For smaller files, linearization is not worth the effort.
"""
filesize = os.stat(working_file).st_size
if filesize > (context.options.fast_web_view * 1_000_000):
return True
return False
def _fix_metadata(meta_original: PdfMetadata, meta_pdf: PdfMetadata):
# If xmp:CreateDate is missing, set it to the modify date to
# ensure consistency with Ghostscript.
if 'xmp:CreateDate' not in meta_pdf:
meta_pdf['xmp:CreateDate'] = meta_pdf.get('xmp:ModifyDate', '')
if meta_pdf.get('dc:title') == 'Untitled':
# Ghostscript likes to set title to Untitled if omitted from input.
# Reverse this, because PDF/A TechNote 0003:Metadata in PDF/A-1
# and the XMP Spec do not make this recommendation.
if 'dc:title' not in meta_original:
del meta_pdf['dc:title']
def _unset_empty_metadata(meta: PdfMetadata, options):
"""Unset metadata fields that were explicitly set to empty strings.
If the user explicitly specified an empty string for any of the
following, they should be unset and not reported as missing in
the output pdf. Note that some metadata fields use differing names
between PDF/A and PDF.
"""
if options.title == '' and 'dc:title' in meta:
del meta['dc:title'] # PDF/A and PDF
if options.author == '':
if 'dc:creator' in meta:
del meta['dc:creator'] # PDF/A (Not xmp:CreatorTool)
if 'pdf:Author' in meta:
del meta['pdf:Author'] # PDF
if options.subject == '':
if 'dc:description' in meta:
del meta['dc:description'] # PDF/A
if 'dc:subject' in meta:
del meta['dc:subject'] # PDF
if options.keywords == '' and 'pdf:Keywords' in meta:
del meta['pdf:Keywords'] # PDF/A and PDF
def _set_language(pdf: Pdf, languages: list[str]):
"""Set the language of the PDF."""
if Name.Lang in pdf.Root or not languages:
return # Already set or can't change
primary_language_iso639_3 = languages[0]
if not primary_language_iso639_3:
return
iso639_2 = iso_639_2_from_3(primary_language_iso639_3)
if not iso639_2:
return
pdf.Root.Lang = iso639_2
def metadata_fixup(
working_file: Path, context: PdfContext, pdf_save_settings: dict[str, Any]
) -> Path:
"""Fix certain metadata fields after Ghostscript PDF/A conversion.
Also report on metadata in the input file that was not retained during
PDF/A conversion.
"""
output_file = context.get_path('metafix.pdf')
options = context.options
with Pdf.open(context.origin) as original, Pdf.open(working_file) as pdf:
docinfo = get_docinfo(original, context)
with original.open_metadata(
set_pikepdf_as_editor=False, update_docinfo=False, strict=False
) as meta_original, pdf.open_metadata() as meta_pdf:
meta_pdf.load_from_docinfo(
docinfo, delete_missing=False, raise_failure=False
)
_fix_metadata(meta_original, meta_pdf)
_unset_empty_metadata(meta_original, options)
_unset_empty_metadata(meta_pdf, options)
meta_missing = set(meta_original.keys()) - set(meta_pdf.keys())
report_on_metadata(options, meta_missing)
_set_language(pdf, options.languages)
pdf.save(output_file, **pdf_save_settings)
return output_file
src/ocrmypdf/_pipeline.py
+21 -146
View File
@@ -12,21 +12,18 @@ import re
import sys
from collections.abc import Iterable, Iterator, Sequence
from contextlib import suppress
from datetime import datetime, timezone
from pathlib import Path
from shutil import copyfileobj, copystat
from typing import Any, BinaryIO, TypeVar, cast
import img2pdf
import pikepdf
from pikepdf.models.metadata import encode_pdf_date
from PIL import Image, ImageColor, ImageDraw
from ocrmypdf._concurrent import Executor
from ocrmypdf._exec import unpaper
from ocrmypdf._jobcontext import PageContext, PdfContext
from ocrmypdf._version import PROGRAM_NAME
from ocrmypdf._version import __version__ as VERSION
from ocrmypdf._metadata import repair_docinfo_nuls
from ocrmypdf.exceptions import (
DigitalSignatureError,
DpiError,
@@ -168,6 +165,7 @@ def get_pdfinfo(
detailed_analysis: bool = False,
progbar: bool = False,
max_workers: int | None = None,
use_threads: bool = True,
check_pages=None,
) -> PdfInfo:
"""Get the PDF info."""
@@ -177,6 +175,7 @@ def get_pdfinfo(
detailed_analysis=detailed_analysis,
progbar=progbar,
max_workers=max_workers,
use_threads=use_threads,
check_pages=check_pages,
executor=executor,
)
@@ -519,6 +518,7 @@ def rasterize(
device_idx = at_least('png16m')
if pageinfo.has_vector:
log.debug("Page has vector content, using png16m")
device_idx = at_least('png16m')
device = colorspaces[device_idx]
@@ -655,7 +655,7 @@ def ocr_engine_hocr(input_file: Path, page_context: PageContext) -> tuple[Path,
output_text=hocr_text_out,
options=options,
)
return (hocr_out, hocr_text_out)
return hocr_out, hocr_text_out
def should_visible_page_image_use_jpg(pageinfo: PageInfo) -> bool:
@@ -735,10 +735,15 @@ def render_hocr_page(hocr: Path, page_context: PageContext) -> Path:
"""Render the hOCR page to a PDF."""
options = page_context.options
output_file = page_context.get_path('ocr_hocr.pdf')
if hocr.stat().st_size == 0:
# If hOCR file is empty (skipped page marker), create an empty PDF file
output_file.touch()
return output_file
dpi = get_page_square_dpi(page_context, calculate_image_dpi(page_context))
debug_mode = options.pdf_renderer == 'hocrdebug'
hocrtransform = HocrTransform(hocr_filename=hocr, dpi=dpi.x) # square
hocrtransform = HocrTransform(hocr_filename=hocr, dpi=dpi.to_scalar()) # square
hocrtransform.to_pdf(
out_filename=output_file,
image_filename=None,
@@ -767,38 +772,6 @@ def ocr_engine_textonly_pdf(
return (output_pdf, output_text)
def get_docinfo(base_pdf: pikepdf.Pdf, context: PdfContext) -> dict[str, str]:
"""Read the document info and store it in a dictionary."""
options = context.options
def from_document_info(key):
try:
s = base_pdf.docinfo[key]
return str(s)
except (KeyError, TypeError):
return ''
pdfmark = {
k: from_document_info(k)
for k in ('/Title', '/Author', '/Keywords', '/Subject', '/CreationDate')
}
if options.title:
pdfmark['/Title'] = options.title
if options.author:
pdfmark['/Author'] = options.author
if options.keywords:
pdfmark['/Keywords'] = options.keywords
if options.subject:
pdfmark['/Subject'] = options.subject
creator_tag = context.plugin_manager.hook.get_ocr_engine().creator_tag(options)
pdfmark['/Creator'] = f'{PROGRAM_NAME} {VERSION} / {creator_tag}'
pdfmark['/Producer'] = f'pikepdf {pikepdf.__version__}'
pdfmark['/ModDate'] = encode_pdf_date(datetime.now(timezone.utc))
return pdfmark
def generate_postscript_stub(context: PdfContext) -> Path:
"""Generates a PostScript file stub for the given PDF context.
@@ -833,7 +806,7 @@ def convert_to_pdfa(input_pdf: Path, input_ps_stub: Path, context: PdfContext) -
# pikepdf can deal with this, but we make the world a better place by
# stamping them out as soon as possible.
with pikepdf.open(input_pdf) as pdf_file:
if _repair_docinfo_nuls(pdf_file):
if repair_docinfo_nuls(pdf_file):
pdf_file.save(fix_docinfo_file)
else:
safe_symlink(input_pdf, fix_docinfo_file)
@@ -856,25 +829,6 @@ def convert_to_pdfa(input_pdf: Path, input_ps_stub: Path, context: PdfContext) -
return output_file
def _repair_docinfo_nuls(pdf):
"""If the DocumentInfo block contains NUL characters, remove them.
If the DocumentInfo block is malformed, log an error and continue.
"""
modified = False
try:
if not isinstance(pdf.docinfo, pikepdf.Dictionary):
raise TypeError("DocumentInfo is not a dictionary")
for k, v in pdf.docinfo.items():
if isinstance(v, str) and b'\x00' in bytes(v):
pdf.docinfo[k] = bytes(v).replace(b'\x00', b'')
modified = True
except TypeError:
# TypeError can also be raised if dictionary items are unexpected types
log.error("File contains a malformed DocumentInfo block - continuing anyway.")
return modified
def should_linearize(working_file: Path, context: PdfContext) -> bool:
"""Determine whether the PDF should be linearized.
@@ -909,86 +863,6 @@ def get_pdf_save_settings(output_type: str) -> dict[str, Any]:
)
def metadata_fixup(working_file: Path, context: PdfContext) -> Path:
"""Fix certain metadata fields after Ghostscript PDF/A conversion.
Also report on metadata in the input file that was not retained during
PDF/A conversion.
"""
output_file = context.get_path('metafix.pdf')
options = context.options
def report_on_metadata(missing):
if not missing:
return
if options.output_type.startswith('pdfa'):
log.warning(
"Some input metadata could not be copied because it is not "
"permitted in PDF/A. You may wish to examine the output "
"PDF's XMP metadata."
)
log.debug("The following metadata fields were not copied: %r", missing)
else:
log.error(
"Some input metadata could not be copied."
"You may wish to examine the output PDF's XMP metadata."
)
log.info("The following metadata fields were not copied: %r", missing)
with pikepdf.open(context.origin) as original, pikepdf.open(working_file) as pdf:
docinfo = get_docinfo(original, context)
with original.open_metadata(
set_pikepdf_as_editor=False, update_docinfo=False, strict=False
) as meta_original, pdf.open_metadata() as meta_pdf:
meta_pdf.load_from_docinfo(
docinfo, delete_missing=False, raise_failure=False
)
# If xmp:CreateDate is missing, set it to the modify date to
# ensure consistency with Ghostscript.
if 'xmp:CreateDate' not in meta_pdf:
meta_pdf['xmp:CreateDate'] = meta_pdf.get('xmp:ModifyDate', '')
if meta_pdf.get('dc:title') == 'Untitled':
# Ghostscript likes to set title to Untitled if omitted from input.
# Reverse this, because PDF/A TechNote 0003:Metadata in PDF/A-1
# and the XMP Spec do not make this recommendation.
if 'dc:title' not in meta_original:
del meta_pdf['dc:title']
# If the user explicitly specified an empty string for any of the
# following, they should be unset and not reported as missing in
# the output pdf. Note that some metadata fields use differing names
# between PDF-A and PDF.
for meta in [meta_pdf, meta_original]:
if options.title == '' and 'dc:title' in meta:
del meta['dc:title'] # PDF-A and PDF
if options.author == '':
if 'dc:creator' in meta:
del meta['dc:creator'] # PDF-A (Not xmp:CreatorTool)
if 'pdf:Author' in meta:
del meta['pdf:Author'] # PDF
if options.subject == '':
if 'dc:description' in meta:
del meta['dc:description'] # PDF-A
if 'dc:subject' in meta:
del meta['dc:subject'] # PDF
if options.keywords == '' and 'pdf:Keywords' in meta:
del meta['pdf:Keywords'] # PDF-A and PDF
meta_missing = set(meta_original.keys()) - set(meta_pdf.keys())
report_on_metadata(meta_missing)
optimizing = context.plugin_manager.hook.is_optimization_enabled(
context=context
)
pdf.save(
output_file,
**get_pdf_save_settings(options.output_type),
linearize=( # Don't linearize if optimize() will be linearizing too
not optimizing and should_linearize(working_file, context)
),
)
return output_file
def _file_size_ratio(
input_file: Path, output_file: Path
) -> tuple[float | None, float | None]:
@@ -1036,7 +910,7 @@ def optimize_pdf(
def enumerate_compress_ranges(
iterable: Iterable[T],
) -> Iterator[tuple[tuple[int, int], T]]:
) -> Iterator[tuple[tuple[int, int], T | None]]:
"""Enumerate the ranges of non-empty elements in an iterable.
Compresses consecutive ranges of length 1 into single elements.
@@ -1090,14 +964,14 @@ def merge_sidecars(txt_files: Iterable[Path | None], context: PdfContext) -> Pat
def copy_final(
input_file: Path, output_file: str | Path | BinaryIO, context: PdfContext
input_file: Path, output_file: str | Path | BinaryIO, original_file: Path | None
) -> None:
"""Copy the final temporary file to the output destination.
Args:
input_file (Path): The intermediate input file to copy.
output_file (str | Path | BinaryIO): The output file to copy to.
context (PdfContext): The PDF context.
original_file: The original file to copy attributes from.
Returns:
None
@@ -1119,8 +993,9 @@ def copy_final(
with open(output_file, 'w+b') as output_stream:
copyfileobj(input_stream, output_stream)
# Attempt to copy file attributes from input to output
with suppress(OSError):
# Copy original file's permissions, ownership, etc. if possible
copystat(context.options.input_file, output_file)
# Set output file's modification time to now
Path(output_file).touch(exist_ok=True)
if original_file:
with suppress(OSError):
# Copy original file's permissions, ownership, etc. if possible
copystat(original_file, output_file)
# Set output file's modification time to now
Path(output_file).touch(exist_ok=True)
src/ocrmypdf/_pipelines/__init__.py
+5
View File
@@ -0,0 +1,5 @@
# SPDX-FileCopyrightText: 2023 James R. Barlow
# SPDX-License-Identifier: MPL-2.0
from __future__ import annotations
src/ocrmypdf/_sync.py → src/ocrmypdf/_pipelines/_common.py
+255 -301
View File
@@ -1,66 +1,53 @@
# SPDX-FileCopyrightText: 2019-2022 James R. Barlow
# SPDX-FileCopyrightText: 2019 Martin Wind
# SPDX-FileCopyrightText: 2023 James R. Barlow
# SPDX-License-Identifier: MPL-2.0
"""Implements the concurrent and page synchronous parts of the pipeline."""
from __future__ import annotations
import argparse
import json
import logging
import logging.handlers
import os
import shutil
import sys
import threading
from collections.abc import Sequence
from concurrent.futures.process import BrokenProcessPool
from concurrent.futures.thread import BrokenThreadPool
from functools import partial
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from tempfile import mkdtemp
from typing import NamedTuple, cast
from typing import Callable, NamedTuple, cast
import PIL
from ocrmypdf._concurrent import Executor, setup_executor
from ocrmypdf._graft import OcrGrafter
from ocrmypdf._jobcontext import PageContext, PdfContext, cleanup_working_files
from ocrmypdf._jobcontext import PageContext, PdfContext
from ocrmypdf._logging import PageNumberFilter
from ocrmypdf._metadata import metadata_fixup
from ocrmypdf._pipeline import (
convert_to_pdfa,
copy_final,
create_ocr_image,
create_pdf_page_from_image,
create_visible_page_jpg,
generate_postscript_stub,
get_orientation_correction,
get_pdfinfo,
is_ocr_required,
merge_sidecars,
metadata_fixup,
ocr_engine_hocr,
ocr_engine_textonly_pdf,
get_pdf_save_settings,
optimize_pdf,
preprocess_clean,
preprocess_deskew,
preprocess_remove_background,
rasterize,
rasterize_preview,
render_hocr_page,
should_linearize,
should_visible_page_image_use_jpg,
triage,
validate_pdfinfo_options,
)
from ocrmypdf._plugin_manager import OcrmypdfPluginManager, get_plugin_manager
from ocrmypdf._plugin_manager import OcrmypdfPluginManager
from ocrmypdf._validation import (
check_requested_output_file,
create_input_file,
report_output_file_size,
)
from ocrmypdf.exceptions import ExitCode, ExitCodeException
from ocrmypdf.helpers import (
NeverRaise,
available_cpu_count,
check_pdf,
pikepdf_enable_mmap,
@@ -69,33 +56,232 @@ from ocrmypdf.helpers import (
from ocrmypdf.pdfa import file_claims_pdfa
log = logging.getLogger(__name__)
tls = threading.local()
tls.pageno = None
def _set_logging_tls(tls):
"""Inject current page number (when available) into log records."""
old_factory = logging.getLogRecordFactory()
def wrapper(*args, **kwargs):
record = old_factory(*args, **kwargs)
if hasattr(tls, 'pageno'):
record.pageno = tls.pageno
return record
logging.setLogRecordFactory(wrapper)
_set_logging_tls(tls)
def set_thread_pageno(pageno: int | None):
"""Set page number (1-based) that the current thread is processing."""
tls.pageno = pageno
class PageResult(NamedTuple):
"""Result when a page is finished processing."""
pageno: int
pdf_page_from_image: Path | None
ocr: Path | None
text: Path | None
orientation_correction: int
"""Page number, 0-based."""
pdf_page_from_image: Path | None = None
"""Single page PDF from image."""
ocr: Path | None = None
"""Single page OCR PDF."""
text: Path | None = None
"""Single page text file."""
orientation_correction: int = 0
"""Orientation correction in degrees."""
tls = threading.local()
tls.pageno = None
@dataclass
class HOCRResult:
"""Result when hOCR is finished processing."""
pageno: int
"""Page number, 0-based."""
pdf_page_from_image: Path | None = None
"""Single page PDF from image."""
hocr: Path | None = None
"""Single page hOCR file."""
textpdf: Path | None = None
"""hOCR file after conversion to PDF."""
orientation_correction: int = 0
"""Orientation correction in degrees."""
def __getstate__(self):
"""Return state values to be pickled."""
return {
k: (
('Path://' + str(v))
if k in ('pdf_page_from_image', 'hocr', 'textpdf') and v is not None
else v
)
for k, v in self.__dict__.items()
}
def __setstate__(self, state):
"""Restore state from the unpickled state values."""
self.__dict__.update(
{
k: (
Path(v.removeprefix('Path://'))
if k in ('pdf_page_from_image', 'hocr', 'textpdf') and v is not None
else v
)
for k, v in state.items()
}
)
@classmethod
def from_json(cls, json_str: str) -> HOCRResult:
"""Create an instance from a dict."""
return cls(**json.loads(json_str))
def to_json(self) -> str:
"""Serialize to a JSON string."""
return json.dumps(self.__getstate__())
old_factory = logging.getLogRecordFactory()
def configure_debug_logging(
log_filename: Path, prefix: str = ''
) -> logging.FileHandler:
"""Create a debug log file at a specified location.
Args:
log_filename: Where to the put the log file.
prefix: The logging domain prefix that should be sent to the log.
"""
log_file_handler = logging.FileHandler(log_filename, delay=True)
log_file_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'[%(asctime)s] - %(name)s - %(levelname)7s -%(pageno)s %(message)s'
)
log_file_handler.setFormatter(formatter)
log_file_handler.addFilter(PageNumberFilter())
logging.getLogger(prefix).addHandler(log_file_handler)
return log_file_handler
def record_factory(*args, **kwargs):
record = old_factory(*args, **kwargs)
if hasattr(tls, 'pageno'):
record.pageno = tls.pageno
return record
def worker_init(max_pixels: int) -> None:
"""Initialize a worker thread or process."""
# In Windows, child process will not inherit our change to this value in
# the parent process, so ensure workers get it set. Not needed when running
# threaded, but harmless to set again.
PIL.Image.MAX_IMAGE_PIXELS = max_pixels
pikepdf_enable_mmap()
logging.setLogRecordFactory(record_factory)
@contextmanager
def manage_debug_log_handler(
*,
options: argparse.Namespace,
work_folder: Path,
):
debug_log_handler = None
if (options.keep_temporary_files or options.verbose >= 1) and not os.environ.get(
'PYTEST_CURRENT_TEST', ''
):
# Debug log for command line interface only with verbose output
# See https://github.com/pytest-dev/pytest/issues/5502 for why we skip this
# when pytest is running
debug_log_handler = configure_debug_logging(
work_folder / "debug.log"
) # pragma: no cover
try:
yield
finally:
if debug_log_handler:
try:
debug_log_handler.close()
log.removeHandler(debug_log_handler)
except OSError as e:
print(e, file=sys.stderr)
@contextmanager
def manage_work_folder(*, work_folder: Path, retain: bool, print_location: bool):
try:
yield work_folder
finally:
if retain:
if print_location:
print(
f"Temporary working files retained at:\n{work_folder}",
file=sys.stderr,
)
else:
shutil.rmtree(work_folder, ignore_errors=True)
def cli_exception_handler(
fn: Callable[[argparse.Namespace, OcrmypdfPluginManager], ExitCode],
options: argparse.Namespace,
plugin_manager: OcrmypdfPluginManager,
) -> ExitCode:
try:
return fn(options, plugin_manager)
except KeyboardInterrupt:
if options.verbose >= 1:
log.exception("KeyboardInterrupt")
else:
log.error("KeyboardInterrupt")
return ExitCode.ctrl_c
except ExitCodeException as e:
e = cast(ExitCodeException, e)
if options.verbose >= 1:
log.exception("ExitCodeException")
elif str(e):
log.error("%s: %s", type(e).__name__, str(e))
else:
log.error(type(e).__name__)
return e.exit_code
except PIL.Image.DecompressionBombError:
log.exception(
"A decompression bomb error was encountered while executing the "
"pipeline. Use the argument --max-image-mpixels to raise the maximum "
"image pixel limit."
)
return ExitCode.other_error
except (
BrokenProcessPool,
BrokenThreadPool,
):
log.exception(
"A worker process was terminated unexpectedly. This is known to occur if "
"processing your file takes all available swap space and RAM. It may "
"help to try again with a smaller number of jobs, using the --jobs "
"argument."
)
return ExitCode.child_process_error
except Exception: # pylint: disable=broad-except
log.exception("An exception occurred while executing the pipeline")
return ExitCode.other_error
def setup_pipeline(
options: argparse.Namespace,
plugin_manager: OcrmypdfPluginManager,
) -> Executor:
# Any changes to options will not take effect for options that are already
# bound to function parameters in the pipeline. (For example
# options.input_file, options.pdf_renderer are already bound.)
if not options.jobs:
options.jobs = available_cpu_count()
pikepdf_enable_mmap()
executor = setup_executor(plugin_manager)
return executor
def preprocess(
@@ -175,20 +361,9 @@ def make_intermediate_images(
return ocr_image, preprocess_out
def exec_page_sync(page_context: PageContext) -> PageResult:
"""Execute a pipeline for a single page synchronously."""
def process_page(page_context: PageContext) -> tuple[Path, Path | None, int]:
"""Process page to create OCR image, visible page image and orientation."""
options = page_context.options
tls.pageno = page_context.pageno + 1
if not is_ocr_required(page_context):
return PageResult(
pageno=page_context.pageno,
pdf_page_from_image=None,
ocr=None,
text=None,
orientation_correction=0,
)
orientation_correction = 0
if options.rotate_pages:
# Rasterize
@@ -216,25 +391,10 @@ def exec_page_sync(page_context: PageContext) -> PageResult:
pdf_page_from_image_out = create_pdf_page_from_image(
visible_image_out, page_context, orientation_correction
)
if options.pdf_renderer.startswith('hocr'):
(hocr_out, text_out) = ocr_engine_hocr(ocr_image_out, page_context)
ocr_out = render_hocr_page(hocr_out, page_context)
elif options.pdf_renderer == 'sandwich':
(ocr_out, text_out) = ocr_engine_textonly_pdf(ocr_image_out, page_context)
else:
raise NotImplementedError(f"pdf_renderer {options.pdf_renderer}")
return PageResult(
pageno=page_context.pageno,
pdf_page_from_image=pdf_page_from_image_out,
ocr=ocr_out,
text=text_out,
orientation_correction=orientation_correction,
)
return ocr_image_out, pdf_page_from_image_out, orientation_correction
def post_process(
def postprocess(
pdf_file: Path, context: PdfContext, executor: Executor
) -> tuple[Path, Sequence[str]]:
"""Postprocess the PDF file."""
@@ -243,242 +403,36 @@ def post_process(
ps_stub_out = generate_postscript_stub(context)
pdf_out = convert_to_pdfa(pdf_out, ps_stub_out, context)
pdf_out = metadata_fixup(pdf_out, context)
optimizing = context.plugin_manager.hook.is_optimization_enabled(context=context)
save_settings = get_pdf_save_settings(context.options.output_type)
save_settings['linearize'] = not optimizing and should_linearize(pdf_out, context)
pdf_out = metadata_fixup(pdf_out, context, pdf_save_settings=save_settings)
return optimize_pdf(pdf_out, context, executor)
def worker_init(max_pixels: int) -> None:
"""Initialize a worker thread or process."""
# In Windows, child process will not inherit our change to this value in
# the parent process, so ensure workers get it set. Not needed when running
# threaded, but harmless to set again.
PIL.Image.MAX_IMAGE_PIXELS = max_pixels
pikepdf_enable_mmap()
def exec_concurrent(context: PdfContext, executor: Executor) -> Sequence[str]:
"""Execute the OCR pipeline concurrently."""
# Run exec_page_sync on every page
options = context.options
max_workers = min(len(context.pdfinfo), options.jobs)
if max_workers > 1:
log.info("Start processing %d pages concurrently", max_workers)
sidecars: list[Path | None] = [None] * len(context.pdfinfo)
ocrgraft = OcrGrafter(context)
def update_page(result: PageResult, pbar):
"""After OCR is complete for a page, update the PDF."""
try:
tls.pageno = result.pageno + 1
sidecars[result.pageno] = result.text
pbar.update()
ocrgraft.graft_page(
pageno=result.pageno,
image=result.pdf_page_from_image,
textpdf=result.ocr,
autorotate_correction=result.orientation_correction,
)
pbar.update()
finally:
tls.pageno = None
executor(
use_threads=options.use_threads,
max_workers=max_workers,
tqdm_kwargs=dict(
total=(2 * len(context.pdfinfo)),
desc='OCR' if options.tesseract_timeout > 0 else 'Image processing',
unit='page',
unit_scale=0.5,
disable=not options.progress_bar,
),
worker_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
task=exec_page_sync,
task_arguments=context.get_page_contexts(),
task_finished=update_page,
)
# Output sidecar text
if options.sidecar:
text = merge_sidecars(sidecars, context)
# Copy text file to destination
copy_final(text, options.sidecar, context)
# Merge layers to one single pdf
pdf = ocrgraft.finalize()
messages: Sequence[str] = []
if options.output_type != 'none':
# PDF/A and metadata
log.info("Postprocessing...")
pdf, messages = post_process(pdf, context, executor)
# Copy PDF file to destination
copy_final(pdf, options.output_file, context)
return messages
def configure_debug_logging(
log_filename: Path, prefix: str = ''
) -> logging.FileHandler:
"""Create a debug log file at a specified location.
Args:
log_filename: Where to the put the log file.
prefix: The logging domain prefix that should be sent to the log.
"""
log_file_handler = logging.FileHandler(log_filename, delay=True)
log_file_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'[%(asctime)s] - %(name)s - %(levelname)7s -%(pageno)s %(message)s'
)
log_file_handler.setFormatter(formatter)
log_file_handler.addFilter(PageNumberFilter())
logging.getLogger(prefix).addHandler(log_file_handler)
return log_file_handler
def run_pipeline(
options: argparse.Namespace,
*,
plugin_manager: OcrmypdfPluginManager | None,
api: bool = False,
) -> ExitCode:
"""Run the OCR pipeline.
Args:
options: The parsed command line options.
plugin_manager: The plugin manager to use. If not provided, one will be
created.
api: If ``True``, the pipeline is being run from the API. This is used
to manage exceptions in a way appropriate for API or CLI usage.
For CLI (``api=False``), exceptions are printed and described;
for API use, they are propagated to the caller.
"""
# Any changes to options will not take effect for options that are already
# bound to function parameters in the pipeline. (For example
# options.input_file, options.pdf_renderer are already bound.)
if not options.jobs:
options.jobs = available_cpu_count()
if not plugin_manager:
plugin_manager = get_plugin_manager(options.plugins)
work_folder = Path(mkdtemp(prefix="ocrmypdf.io."))
debug_log_handler = None
if (
(options.keep_temporary_files or options.verbose >= 1)
and not os.environ.get('PYTEST_CURRENT_TEST', '')
and not api
):
# Debug log for command line interface only with verbose output
# See https://github.com/pytest-dev/pytest/issues/5502 for why we skip this
# when pytest is running
debug_log_handler = configure_debug_logging(
Path(work_folder) / "debug.log"
) # pragma: no cover
pikepdf_enable_mmap()
executor = setup_executor(plugin_manager)
try:
check_requested_output_file(options)
start_input_file, original_filename = create_input_file(options, work_folder)
# Triage image or pdf
origin_pdf = triage(
original_filename, start_input_file, work_folder / 'origin.pdf', options
def report_output_pdf(options, start_input_file, optimize_messages) -> ExitCode:
if options.output_file == '-':
log.info("Output sent to stdout")
elif hasattr(options.output_file, 'writable') and options.output_file.writable():
log.info("Output written to stream")
elif samefile(options.output_file, Path(os.devnull)):
pass # Say nothing when sending to dev null
else:
if options.output_type.startswith('pdfa'):
pdfa_info = file_claims_pdfa(options.output_file)
if pdfa_info['pass']:
log.info("Output file is a %s (as expected)", pdfa_info['conformance'])
else:
log.warning(
"Output file is okay but is not PDF/A (seems to be %s)",
pdfa_info['conformance'],
)
return ExitCode.pdfa_conversion_failed
if not check_pdf(options.output_file):
log.warning('Output file: The generated PDF is INVALID')
return ExitCode.invalid_output_pdf
report_output_file_size(
options, start_input_file, options.output_file, optimize_messages
)
# Gather pdfinfo and create context
pdfinfo = get_pdfinfo(
origin_pdf,
executor=executor,
detailed_analysis=options.redo_ocr,
progbar=options.progress_bar,
max_workers=options.jobs if not options.use_threads else 1, # To help debug
check_pages=options.pages,
)
context = PdfContext(options, work_folder, origin_pdf, pdfinfo, plugin_manager)
# Validate options are okay for this pdf
validate_pdfinfo_options(context)
# Execute the pipeline
optimize_messages = exec_concurrent(context, executor)
if options.output_file == '-':
log.info("Output sent to stdout")
elif (
hasattr(options.output_file, 'writable') and options.output_file.writable()
):
log.info("Output written to stream")
elif samefile(options.output_file, Path(os.devnull)):
pass # Say nothing when sending to dev null
else:
if options.output_type.startswith('pdfa'):
pdfa_info = file_claims_pdfa(options.output_file)
if pdfa_info['pass']:
log.info(
"Output file is a %s (as expected)", pdfa_info['conformance']
)
else:
log.warning(
"Output file is okay but is not PDF/A (seems to be %s)",
pdfa_info['conformance'],
)
return ExitCode.pdfa_conversion_failed
if not check_pdf(options.output_file):
log.warning('Output file: The generated PDF is INVALID')
return ExitCode.invalid_output_pdf
report_output_file_size(
options, start_input_file, options.output_file, optimize_messages
)
except KeyboardInterrupt if not api else NeverRaise:
if options.verbose >= 1:
log.exception("KeyboardInterrupt")
else:
log.error("KeyboardInterrupt")
return ExitCode.ctrl_c
except ExitCodeException if not api else NeverRaise as e:
e = cast(ExitCodeException, e)
if options.verbose >= 1:
log.exception("ExitCodeException")
elif str(e):
log.error("%s: %s", type(e).__name__, str(e))
else:
log.error(type(e).__name__)
return e.exit_code
except PIL.Image.DecompressionBombError if not api else NeverRaise:
log.exception(
"A decompression bomb error was encountered while executing the "
"pipeline. Use the argument --max-image-mpixels to raise the maximum "
"image pixel limit."
)
return ExitCode.other_error
except (
BrokenProcessPool if not api else NeverRaise,
BrokenThreadPool if not api else NeverRaise,
):
log.exception(
"A worker process was terminated unexpectedly. This is known to occur if "
"processing your file takes all available swap space and RAM. It may "
"help to try again with a smaller number of jobs, using the --jobs "
"argument."
)
return ExitCode.child_process_error
except Exception if not api else NeverRaise: # pylint: disable=broad-except
log.exception("An exception occurred while executing the pipeline")
return ExitCode.other_error
finally:
if debug_log_handler:
try:
debug_log_handler.close()
log.removeHandler(debug_log_handler)
except OSError as e:
print(e, file=sys.stderr)
cleanup_working_files(work_folder, options)
return ExitCode.ok
src/ocrmypdf/_pipelines/hocr_to_ocr_pdf.py
+134
View File
@@ -0,0 +1,134 @@
# SPDX-FileCopyrightText: 2019-2023 James R. Barlow
# SPDX-FileCopyrightText: 2019 Martin Wind
# SPDX-License-Identifier: MPL-2.0
"""Implements the concurrent and page synchronous parts of the pipeline."""
from __future__ import annotations
import argparse
import logging
import logging.handlers
from collections.abc import Sequence
from functools import partial
import PIL
from ocrmypdf._concurrent import Executor
from ocrmypdf._graft import OcrGrafter
from ocrmypdf._jobcontext import PageContext, PdfContext
from ocrmypdf._pipeline import (
copy_final,
get_pdfinfo,
render_hocr_page,
)
from ocrmypdf._pipelines._common import (
HOCRResult,
manage_work_folder,
postprocess,
report_output_pdf,
set_thread_pageno,
setup_pipeline,
worker_init,
)
from ocrmypdf._plugin_manager import OcrmypdfPluginManager
from ocrmypdf._progressbar import ProgressBar
from ocrmypdf.exceptions import ExitCode
log = logging.getLogger(__name__)
def _exec_hocrtransform_sync(page_context: PageContext) -> HOCRResult:
"""Process each page."""
hocr_json = page_context.get_path('hocr.json')
if not hocr_json.exists():
# No hOCR file, so no OCR was performed on this page.
return HOCRResult(pageno=page_context.pageno)
hocr_result = HOCRResult.from_json(hocr_json.read_text())
hocr_result.textpdf = render_hocr_page(
page_context.get_path('ocr_hocr.hocr'), page_context
)
return hocr_result
def exec_hocr_to_ocr_pdf(context: PdfContext, executor: Executor) -> Sequence[str]:
"""Convert hOCR files to OCR PDF."""
# Run exec_page_sync on every page
options = context.options
max_workers = min(len(context.pdfinfo), options.jobs)
if max_workers > 1:
log.info("Continue processing %d pages concurrently", max_workers)
ocrgraft = OcrGrafter(context)
def graft_page(result: HOCRResult, pbar: ProgressBar):
"""Graft text only PDF on to main PDF's page."""
try:
set_thread_pageno(result.pageno + 1)
pbar.update()
ocrgraft.graft_page(
pageno=result.pageno,
image=result.pdf_page_from_image,
textpdf=result.textpdf,
autorotate_correction=result.orientation_correction,
)
pbar.update()
finally:
set_thread_pageno(None)
executor(
use_threads=options.use_threads,
max_workers=max_workers,
progress_kwargs=dict(
total=(2 * len(context.pdfinfo)),
desc='Grafting hOCR to PDF',
unit='page',
unit_scale=0.5,
disable=not options.progress_bar,
),
worker_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
task=_exec_hocrtransform_sync,
task_arguments=context.get_page_context_args(),
task_finished=graft_page,
)
pdf = ocrgraft.finalize()
messages: Sequence[str] = []
if options.output_type != 'none':
# PDF/A and metadata
log.info("Postprocessing...")
pdf, messages = postprocess(pdf, context, executor)
# Copy PDF file to destination (we don't know the input PDF file name)
copy_final(pdf, options.output_file, None)
return messages
def run_hocr_to_ocr_pdf_pipeline(
options: argparse.Namespace,
*,
plugin_manager: OcrmypdfPluginManager,
) -> ExitCode:
"""Run pipeline to convert hOCR to final output PDF."""
with manage_work_folder(
work_folder=options.work_folder, retain=True, print_location=False
) as work_folder:
executor = setup_pipeline(options, plugin_manager)
origin_pdf = work_folder / 'origin.pdf'
# Gather pdfinfo and create context
pdfinfo = get_pdfinfo(
origin_pdf,
executor=executor,
detailed_analysis=options.redo_ocr,
progbar=options.progress_bar,
max_workers=options.jobs,
use_threads=options.use_threads,
check_pages=options.pages,
)
context = PdfContext(options, work_folder, origin_pdf, pdfinfo, plugin_manager)
plugin_manager.hook.check_options(options=options)
optimize_messages = exec_hocr_to_ocr_pdf(context, executor)
return report_output_pdf(options, origin_pdf, optimize_messages)
src/ocrmypdf/_pipelines/ocr.py
+225
View File
@@ -0,0 +1,225 @@
# SPDX-FileCopyrightText: 2019-2023 James R. Barlow
# SPDX-FileCopyrightText: 2019 Martin Wind
# SPDX-License-Identifier: MPL-2.0
"""Implements the concurrent and page synchronous parts of the pipeline."""
from __future__ import annotations
import argparse
import logging
import logging.handlers
from collections.abc import Sequence
from functools import partial
from pathlib import Path
from tempfile import mkdtemp
import PIL
from ocrmypdf._concurrent import Executor
from ocrmypdf._graft import OcrGrafter
from ocrmypdf._jobcontext import PageContext, PdfContext
from ocrmypdf._pipeline import (
copy_final,
get_pdfinfo,
is_ocr_required,
merge_sidecars,
ocr_engine_hocr,
ocr_engine_textonly_pdf,
render_hocr_page,
triage,
validate_pdfinfo_options,
)
from ocrmypdf._pipelines._common import (
PageResult,
cli_exception_handler,
manage_debug_log_handler,
manage_work_folder,
postprocess,
process_page,
report_output_pdf,
set_thread_pageno,
setup_pipeline,
worker_init,
)
from ocrmypdf._plugin_manager import OcrmypdfPluginManager
from ocrmypdf._progressbar import ProgressBar
from ocrmypdf._validation import (
check_requested_output_file,
create_input_file,
)
from ocrmypdf.exceptions import ExitCode
log = logging.getLogger(__name__)
def _image_to_ocr_text(
page_context: PageContext, ocr_image_out: Path
) -> tuple[Path, Path]:
"""Run OCR engine on image to create OCR PDF and text file."""
options = page_context.options
if options.pdf_renderer.startswith('hocr'):
hocr_out, text_out = ocr_engine_hocr(ocr_image_out, page_context)
ocr_out = render_hocr_page(hocr_out, page_context)
elif options.pdf_renderer == 'sandwich':
ocr_out, text_out = ocr_engine_textonly_pdf(ocr_image_out, page_context)
else:
raise NotImplementedError(f"pdf_renderer {options.pdf_renderer}")
return ocr_out, text_out
def _exec_page_sync(page_context: PageContext) -> PageResult:
"""Execute a pipeline for a single page synchronously."""
set_thread_pageno(page_context.pageno + 1)
if not is_ocr_required(page_context):
return PageResult(pageno=page_context.pageno)
ocr_image_out, pdf_page_from_image_out, orientation_correction = process_page(
page_context
)
ocr_out, text_out = _image_to_ocr_text(page_context, ocr_image_out)
return PageResult(
pageno=page_context.pageno,
pdf_page_from_image=pdf_page_from_image_out,
ocr=ocr_out,
text=text_out,
orientation_correction=orientation_correction,
)
def exec_concurrent(context: PdfContext, executor: Executor) -> Sequence[str]:
"""Execute the OCR pipeline concurrently."""
options = context.options
max_workers = min(len(context.pdfinfo), options.jobs)
if max_workers > 1:
log.info("Start processing %d pages concurrently", max_workers)
sidecars: list[Path | None] = [None] * len(context.pdfinfo)
ocrgraft = OcrGrafter(context)
def update_page(result: PageResult, pbar: ProgressBar):
"""After OCR is complete for a page, update the PDF."""
try:
set_thread_pageno(result.pageno + 1)
sidecars[result.pageno] = result.text
pbar.update()
ocrgraft.graft_page(
pageno=result.pageno,
image=result.pdf_page_from_image,
textpdf=result.ocr,
autorotate_correction=result.orientation_correction,
)
pbar.update()
finally:
set_thread_pageno(None)
executor(
use_threads=options.use_threads,
max_workers=max_workers,
progress_kwargs=dict(
total=(2 * len(context.pdfinfo)),
desc='OCR' if options.tesseract_timeout > 0 else 'Image processing',
unit='page',
unit_scale=0.5,
disable=not options.progress_bar,
),
worker_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
task=_exec_page_sync,
task_arguments=context.get_page_context_args(),
task_finished=update_page,
)
# Output sidecar text
if options.sidecar:
text = merge_sidecars(sidecars, context)
# Copy text file to destination
copy_final(text, options.sidecar, options.input_file)
# Merge layers to one single pdf
pdf = ocrgraft.finalize()
messages: Sequence[str] = []
if options.output_type != 'none':
# PDF/A and metadata
log.info("Postprocessing...")
pdf, messages = postprocess(pdf, context, executor)
# Copy PDF file to destination
copy_final(pdf, options.output_file, options.input_file)
return messages
def _run_pipeline(
options: argparse.Namespace,
plugin_manager: OcrmypdfPluginManager,
) -> ExitCode:
with manage_work_folder(
work_folder=Path(mkdtemp(prefix="ocrmypdf.io.")),
retain=options.keep_temporary_files,
print_location=options.keep_temporary_files,
) as work_folder, manage_debug_log_handler(
options=options, work_folder=work_folder
):
executor = setup_pipeline(options, plugin_manager)
check_requested_output_file(options)
start_input_file, original_filename = create_input_file(options, work_folder)
# Triage image or pdf
origin_pdf = triage(
original_filename, start_input_file, work_folder / 'origin.pdf', options
)
# Gather pdfinfo and create context
pdfinfo = get_pdfinfo(
origin_pdf,
executor=executor,
detailed_analysis=options.redo_ocr,
progbar=options.progress_bar,
max_workers=options.jobs,
use_threads=options.use_threads,
check_pages=options.pages,
)
context = PdfContext(options, work_folder, origin_pdf, pdfinfo, plugin_manager)
# Validate options are okay for this pdf
validate_pdfinfo_options(context)
# Execute the pipeline
optimize_messages = exec_concurrent(context, executor)
exitcode = report_output_pdf(options, start_input_file, optimize_messages)
return exitcode
def run_pipeline_cli(
options: argparse.Namespace,
*,
plugin_manager: OcrmypdfPluginManager,
) -> ExitCode:
"""Run the OCR pipeline with command line exception handling.
Args:
options: The parsed command line options.
plugin_manager: The plugin manager to use. If not provided, one will be
created.
"""
return cli_exception_handler(_run_pipeline, options, plugin_manager)
def run_pipeline(
options: argparse.Namespace,
*,
plugin_manager: OcrmypdfPluginManager,
) -> ExitCode:
"""Run the OCR pipeline without command line exception handling.
Args:
options: The parsed command line options.
plugin_manager: The plugin manager to use. If not provided, one will be
created.
"""
return _run_pipeline(options, plugin_manager)
src/ocrmypdf/_pipelines/pdf_to_hocr.py
+116
View File
@@ -0,0 +1,116 @@
# SPDX-FileCopyrightText: 2019-2023 James R. Barlow
# SPDX-FileCopyrightText: 2019 Martin Wind
# SPDX-License-Identifier: MPL-2.0
"""Implements the concurrent and page synchronous parts of the pipeline."""
from __future__ import annotations
import argparse
import logging
import logging.handlers
import shutil
from functools import partial
import PIL
from ocrmypdf._concurrent import Executor
from ocrmypdf._jobcontext import PageContext, PdfContext
from ocrmypdf._pipeline import (
get_pdfinfo,
is_ocr_required,
ocr_engine_hocr,
validate_pdfinfo_options,
)
from ocrmypdf._pipelines._common import (
HOCRResult,
manage_work_folder,
process_page,
set_thread_pageno,
setup_pipeline,
worker_init,
)
from ocrmypdf._plugin_manager import OcrmypdfPluginManager
from ocrmypdf._validation import (
set_lossless_reconstruction,
)
log = logging.getLogger(__name__)
def _exec_page_hocr_sync(page_context: PageContext) -> HOCRResult:
"""Execute a pipeline for a single page hOCR."""
set_thread_pageno(page_context.pageno + 1)
if not is_ocr_required(page_context):
return HOCRResult(pageno=page_context.pageno)
ocr_image_out, pdf_page_from_image_out, orientation_correction = process_page(
page_context
)
hocr_out, _ = ocr_engine_hocr(ocr_image_out, page_context)
result = HOCRResult(
pageno=page_context.pageno,
pdf_page_from_image=pdf_page_from_image_out,
hocr=hocr_out,
orientation_correction=orientation_correction,
)
page_context.get_path('hocr.json').write_text(result.to_json())
return result
def exec_pdf_to_hocr(context: PdfContext, executor: Executor) -> None:
"""Execute the OCR pipeline concurrently and output hOCR."""
# Run exec_page_sync on every page
options = context.options
max_workers = min(len(context.pdfinfo), options.jobs)
if max_workers > 1:
log.info("Start processing %d pages concurrently", max_workers)
executor(
use_threads=options.use_threads,
max_workers=max_workers,
progress_kwargs=dict(
total=(2 * len(context.pdfinfo)),
desc='hOCR',
unit='page',
unit_scale=0.5,
disable=not options.progress_bar,
),
worker_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
task=_exec_page_hocr_sync,
task_arguments=context.get_page_context_args(),
)
def run_hocr_pipeline(
options: argparse.Namespace,
*,
plugin_manager: OcrmypdfPluginManager,
) -> None:
"""Run pipeline to output hOCR."""
with manage_work_folder(
work_folder=options.output_folder, retain=True, print_location=False
) as work_folder:
executor = setup_pipeline(options, plugin_manager)
shutil.copy2(options.input_file, work_folder / 'origin.pdf')
# Gather pdfinfo and create context
pdfinfo = get_pdfinfo(
options.input_file,
executor=executor,
detailed_analysis=options.redo_ocr,
progbar=options.progress_bar,
max_workers=options.jobs,
use_threads=options.use_threads,
check_pages=options.pages,
)
context = PdfContext(
options, work_folder, options.input_file, pdfinfo, plugin_manager
)
# Validate options are okay for this pdf
set_lossless_reconstruction(options)
validate_pdfinfo_options(context)
exec_pdf_to_hocr(context, executor)
src/ocrmypdf/_plugin_manager.py
+3 -3
View File
@@ -33,7 +33,7 @@ class OcrmypdfPluginManager(pluggy.PluginManager):
def __init__(
self,
*args,
plugins: list[str | Path],
plugins: Sequence[str | Path],
builtins: bool = True,
**kwargs,
):
@@ -101,11 +101,11 @@ class OcrmypdfPluginManager(pluggy.PluginManager):
def get_plugin_manager(
plugins: list[str | Path], builtins=True
plugins: Sequence[str | Path] | None = None, builtins=True
) -> OcrmypdfPluginManager:
return OcrmypdfPluginManager(
project_name='ocrmypdf',
plugins=plugins,
plugins=plugins if plugins is not None else [],
builtins=builtins,
)
src/ocrmypdf/_progressbar.py
+135
View File
@@ -0,0 +1,135 @@
from typing import Protocol
from rich.console import Console
from rich.progress import (
BarColumn,
MofNCompleteColumn,
Progress,
TaskProgressColumn,
TextColumn,
TimeRemainingColumn,
)
from rich.table import Column
class ProgressBar(Protocol):
"""The protocol that OCRmyPDF expects progress bar classes to be compatible with.
In practice this could be used for any time of monitoring, not just a progress bar.
Calling the class should return a new progress bar object, which is activated
with ``__enter__`` and terminated with ``__exit__``. An update method is called
whenever the progress bar is updated. Progress bar objects will not be reused;
a new one will be created for each group of tasks.
The progress bar is held in the main process/thread and not updated by child
process/threads. When a child notifies the parent of completed work, the
parent updates the progress bar.
Progress bars should never write to ``sys.stdout``, or they will corrupt the
output if OCRmyPDF writes a PDF to standard output.
The type of events that OCRmyPDF reports to a progress bar may change in
minor releases.
"""
def __init__(
self,
*,
total: int | float | None,
desc: str | None,
unit: str | None,
disable: bool = False,
**kwargs,
):
"""Initialize a progress bar.
*total* indicates the total number of work units. If None, the total
number of work units is unknown. If *disable* is True, the progress bar
should be disabled. *unit* is a description of the work unit.
*desc* is a description of the overall task to be performed.
Unrecognized keyword arguments must be ignored, as the list of keyword
arguments may grow with time.
"""
def __enter__(self):
"""Enter a progress bar context."""
def __exit__(self, *args):
"""Exit a progress bar context."""
def update(self, n=1):
"""Update the progress bar by an increment.
For use within a progress bar context.
"""
class NullProgressBar:
"""Progress bar API that takes no actions."""
def __init__(self, **kwargs):
pass
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
return False
def update(self, _arg=None):
return
class RichProgressBar:
"""Display progress bar using rich."""
def __init__(
self,
*,
console: Console,
desc: str,
total: float | None = None,
unit: str | None = None,
unit_scale: float | None = 1.0,
disable: bool = False,
**kwargs,
):
self.progress = Progress(
TextColumn(
"[progress.description]{task.description}",
table_column=Column(min_width=20),
),
BarColumn(),
TaskProgressColumn(),
MofNCompleteColumn(),
TimeRemainingColumn(),
console=console,
auto_refresh=True,
redirect_stderr=True,
redirect_stdout=False,
disable=disable,
**kwargs,
)
self.unit_scale = unit_scale
self.progress_bar = self.progress.add_task(
desc,
total=total * self.unit_scale
if total is not None and self.unit_scale is not None
else None,
unit=unit,
)
def __enter__(self):
self.progress.start()
return self
def __exit__(self, exc_type, exc_value, traceback):
self.progress.refresh()
self.progress.stop()
return False
def update(self, value=None):
advance = self.unit_scale if value is None else value
self.progress.update(self.progress_bar, advance=advance)
src/ocrmypdf/_validation.py
+4 -1
View File
@@ -99,6 +99,8 @@ def check_options_output(options: Namespace) -> None:
f"`-` to suppress this message."
)
def set_lossless_reconstruction(options: Namespace) -> None:
lossless_reconstruction = False
if not any(
(
@@ -228,6 +230,7 @@ def _check_plugin_invariant_options(options: Namespace) -> None:
check_platform()
check_options_metadata(options)
check_options_output(options)
set_lossless_reconstruction(options)
check_options_sidecar(options)
check_options_preprocessing(options)
check_options_ocr_behavior(options)
@@ -282,7 +285,7 @@ def create_input_file(options: Namespace, work_folder: Path) -> tuple[Path, str]
msg = f"File not found - {options.input_file}"
if _in_docker(): # pragma: no cover
msg += (
"\nDocker cannot your working directory unless you "
"\nDocker cannot access your working directory unless you "
"explicitly share it with the Docker container and set up"
"permissions correctly.\n"
"You may find it easier to use stdin/stdout:"
src/ocrmypdf/api.py
+213 -35
View File
@@ -10,7 +10,7 @@ import os
import sys
import threading
from argparse import Namespace
from collections.abc import Iterable
from collections.abc import Iterable, Sequence
from enum import IntEnum
from io import IOBase
from pathlib import Path
@@ -20,8 +20,10 @@ from warnings import warn
import pluggy
from ocrmypdf._logging import PageNumberFilter
from ocrmypdf._pipelines.hocr_to_ocr_pdf import run_hocr_to_ocr_pdf_pipeline
from ocrmypdf._pipelines.ocr import run_pipeline, run_pipeline_cli
from ocrmypdf._pipelines.pdf_to_hocr import run_hocr_pipeline
from ocrmypdf._plugin_manager import get_plugin_manager
from ocrmypdf._sync import run_pipeline
from ocrmypdf._validation import check_options
from ocrmypdf.cli import ArgumentParser, get_parser
from ocrmypdf.helpers import is_iterable_notstr
@@ -29,6 +31,9 @@ from ocrmypdf.helpers import is_iterable_notstr
StrPath = Union[Path, AnyStr]
PathOrIO = Union[BinaryIO, StrPath]
# Installing plugins affects the global state of the Python interpreter,
# so we need to use a lock to prevent multiple threads from installing
# plugins at the same time.
_api_lock = threading.Lock()
@@ -133,34 +138,19 @@ def configure_logging(
return log
def create_options(
*, input_file: PathOrIO, output_file: PathOrIO, parser: ArgumentParser, **kwargs
) -> Namespace:
"""Construct an options object from the input/output files and keyword arguments.
Args:
input_file: Input file path or file object.
output_file: Output file path or file object.
parser: ArgumentParser object.
**kwargs: Keyword arguments.
Returns:
argparse.Namespace: A Namespace object containing the parsed arguments.
Raises:
TypeError: If the type of a keyword argument is not supported.
"""
def _kwargs_to_cmdline(
*, defer_kwargs: set[str], **kwargs
) -> tuple[list[str], dict[str, AnyStr]]:
"""Convert kwargs to command line arguments."""
cmdline = []
deferred = []
deferred = {}
for arg, val in kwargs.items():
if val is None:
continue
# These arguments with special handling for which we bypass
# argparse
if arg in {'progress_bar', 'plugins'}:
deferred.append((arg, val))
# Skip arguments that are handled elsewhere
if arg in defer_kwargs:
deferred[arg] = val
continue
cmd_style_arg = arg.replace('_', '-')
@@ -187,7 +177,30 @@ def create_options(
cmdline.append(str(val))
else:
raise TypeError(f"{arg}: {val} ({type(val)})")
return cmdline, deferred
def create_options(
*, input_file: PathOrIO, output_file: PathOrIO, parser: ArgumentParser, **kwargs
) -> Namespace:
"""Construct an options object from the input/output files and keyword arguments.
Args:
input_file: Input file path or file object.
output_file: Output file path or file object.
parser: ArgumentParser object.
**kwargs: Keyword arguments.
Returns:
argparse.Namespace: A Namespace object containing the parsed arguments.
Raises:
TypeError: If the type of a keyword argument is not supported.
"""
cmdline, deferred = _kwargs_to_cmdline(
defer_kwargs={'progress_bar', 'plugins', 'parser', 'input_file', 'output_file'},
**kwargs,
)
if isinstance(input_file, (BinaryIO, IOBase)):
cmdline.append('stream://input_file')
else:
@@ -199,7 +212,7 @@ def create_options(
parser.enable_api_mode()
options = parser.parse_args(cmdline)
for keyword, val in deferred:
for keyword, val in deferred.items():
setattr(options, keyword, val)
if options.input_file == 'stream://input_file':
@@ -336,17 +349,15 @@ def ocr( # noqa: D417
plugins = list(plugins)
# No new variable names should be assigned until these two steps are run
create_options_kwargs = {k: v for k, v in locals().items() if k != 'kwargs'}
create_options_kwargs = {
k: v
for k, v in locals().items()
if k not in {'input_file', 'output_file', 'kwargs'}
}
create_options_kwargs.update(kwargs)
parser = get_parser()
create_options_kwargs['parser'] = parser
with _api_lock:
# We can't allow multiple ocrmypdf.ocr() threads to run in parallel, because
# they might install different plugins, and generally speaking we have areas
# of code that use global state.
if not plugin_manager:
plugin_manager = get_plugin_manager(plugins)
plugin_manager.hook.add_options(parser=parser) # pylint: disable=no-member
@@ -354,9 +365,173 @@ def ocr( # noqa: D417
if 'verbose' in kwargs:
warn("ocrmypdf.ocr(verbose=) is ignored. Use ocrmypdf.configure_logging().")
options = create_options(**create_options_kwargs)
options = create_options(
input_file=input_file,
output_file=output_file,
parser=parser,
**create_options_kwargs,
)
check_options(options, plugin_manager)
return run_pipeline(options=options, plugin_manager=plugin_manager, api=True)
return run_pipeline(options=options, plugin_manager=plugin_manager)
def pdf_to_hocr( # noqa: D417
input_pdf: Path,
output_folder: Path,
*,
language: Iterable[str] | None = None,
image_dpi: int | None = None,
jobs: int | None = None,
use_threads: bool | None = None,
title: str | None = None,
author: str | None = None,
subject: str | None = None,
keywords: str | None = None,
rotate_pages: bool | None = None,
remove_background: bool | None = None,
deskew: bool | None = None,
clean: bool | None = None,
clean_final: bool | None = None,
unpaper_args: str | None = None,
oversample: int | None = None,
remove_vectors: bool | None = None,
force_ocr: bool | None = None,
skip_text: bool | None = None,
redo_ocr: bool | None = None,
skip_big: float | None = None,
pages: str | None = None,
max_image_mpixels: float | None = None,
tesseract_config: Iterable[str] | None = None,
tesseract_pagesegmode: int | None = None,
tesseract_oem: int | None = None,
tesseract_thresholding: int | None = None,
tesseract_timeout: float | None = None,
tesseract_non_ocr_timeout: float | None = None,
tesseract_downsample_above: int | None = None,
tesseract_downsample_large_images: bool | None = None,
rotate_pages_threshold: float | None = None,
user_words: os.PathLike | None = None,
user_patterns: os.PathLike | None = None,
continue_on_soft_render_error: bool | None = None,
invalidate_digital_signatures: bool | None = None,
plugin_manager=None,
plugins: Sequence[StrPath] | None = None,
keep_temporary_files: bool | None = None,
**kwargs,
):
"""Partially run OCRmyPDF and produces an output folder containing hOCR files.
Given a PDF file, this function will run OCRmyPDF up to the point where
the PDF is rasterized to images, OCRed, and the hOCR files are produced,
all of which are saved to the output folder. This is useful for applications
that want to provide an interface for users to edit the text before
rendering the final PDF.
Use :func:`hocr_to_ocr_pdf` to produce the final PDF.
For arguments not explicitly documented here, see documentation for the
equivalent command line parameter.
Args:
input_pdf: Input PDF file path.
output_folder: Output folder path.
**kwargs: Keyword arguments.
"""
# No new variable names should be assigned until these two steps are run
create_options_kwargs = {
k: v
for k, v in locals().items()
if k not in {'input_pdf', 'output_folder', 'kwargs'}
}
create_options_kwargs.update(kwargs)
parser = get_parser()
with _api_lock:
if not plugin_manager:
plugin_manager = get_plugin_manager(plugins)
plugin_manager.hook.add_options(parser=parser) # pylint: disable=no-member
cmdline, deferred = _kwargs_to_cmdline(
defer_kwargs={'input_pdf', 'output_folder', 'plugins'},
**create_options_kwargs,
)
cmdline.append(str(input_pdf))
cmdline.append(str(output_folder))
parser.enable_api_mode()
options = parser.parse_args(cmdline)
for keyword, val in deferred.items():
setattr(options, keyword, val)
delattr(options, 'output_file')
setattr(options, 'output_folder', output_folder)
return run_hocr_pipeline(options=options, plugin_manager=plugin_manager)
def hocr_to_ocr_pdf( # noqa: D417
work_folder: Path,
output_file: Path,
*,
jobs: int | None = None,
use_threads: bool | None = None,
optimize: int | None = None,
jpg_quality: int | None = None,
png_quality: int | None = None,
jbig2_lossy: bool | None = None,
jbig2_page_group_size: int | None = None,
jbig2_threshold: float | None = None,
pdfa_image_compression: str | None = None,
color_conversion_strategy: str | None = None,
fast_web_view: float | None = None,
plugin_manager=None,
plugins: Sequence[StrPath] | None = None,
**kwargs,
):
"""Run OCRmyPDF on a work folder and produce an output PDF.
After running :func:`pdf_to_hocr`, this function will run OCRmyPDF on the work
folder to produce an output PDF. This function consolidates any changes made
to the hOCR files in the work folder and produces a final PDF.
For arguments not explicitly documented here, see documentation for the
equivalent command line parameter.
Args:
work_folder: Work folder path, as generated by :func:`pdf_to_hocr`.
output_file: Output PDF file path.
**kwargs: Keyword arguments.
"""
# No new variable names should be assigned until these two steps are run
create_options_kwargs = {
k: v
for k, v in locals().items()
if k not in {'work_folder', 'output_pdf', 'kwargs'}
}
create_options_kwargs.update(kwargs)
parser = get_parser()
with _api_lock:
if not plugin_manager:
plugin_manager = get_plugin_manager(plugins)
plugin_manager.hook.add_options(parser=parser) # pylint: disable=no-member
cmdline, deferred = _kwargs_to_cmdline(
defer_kwargs={'work_folder', 'output_file', 'plugins'},
**create_options_kwargs,
)
cmdline.append(str(work_folder))
cmdline.append(str(output_file))
parser.enable_api_mode()
options = parser.parse_args(cmdline)
for keyword, val in deferred.items():
setattr(options, keyword, val)
delattr(options, 'input_file')
setattr(options, 'work_folder', work_folder)
return run_hocr_to_ocr_pdf_pipeline(
options=options, plugin_manager=plugin_manager
)
__all__ = [
@@ -368,5 +543,8 @@ __all__ = [
'get_parser',
'get_plugin_manager',
'ocr',
'pdf_to_hocr',
'run_pipeline',
'run_pipeline_cli',
'hocr_to_ocr_pdf',
]
src/ocrmypdf/builtin_plugins/concurrency.py
+10 -7
View File
@@ -20,7 +20,8 @@ from typing import Callable, Union
from rich.console import Console as RichConsole
from ocrmypdf import Executor, hookimpl
from ocrmypdf._logging import RichLoggingHandler, RichTqdmProgressAdapter
from ocrmypdf._logging import RichLoggingHandler
from ocrmypdf._progressbar import RichProgressBar
from ocrmypdf.exceptions import InputFileError
from ocrmypdf.helpers import remove_all_log_handlers
@@ -29,6 +30,8 @@ Queue = Union[multiprocessing.Queue, queue.Queue]
UserInit = Callable[[], None]
WorkerInit = Callable[[Queue, UserInit, int], None]
RichTqdmProgressAdapter = RichProgressBar # Deprecated shim; remove in OCRmyPDF 16
def log_listener(q: Queue):
"""Listen to the worker processes and forward the messages to logging.
@@ -101,7 +104,7 @@ class StandardExecutor(Executor):
*,
use_threads: bool,
max_workers: int,
tqdm_kwargs: dict,
progress_kwargs: dict,
worker_initializer: Callable,
task: Callable,
task_arguments: Iterable,
@@ -127,12 +130,12 @@ class StandardExecutor(Executor):
listener = threading.Thread(target=log_listener, args=(log_queue,))
listener.start()
with self.pbar_class(**tqdm_kwargs) as pbar, executor_class(
with self.pbar_class(**progress_kwargs) as pbar, executor_class(
max_workers=max_workers,
initializer=initializer,
initargs=(log_queue, worker_initializer, logging.getLogger("").level),
) as executor:
futures = [executor.submit(task, args) for args in task_arguments]
futures = [executor.submit(task, *args) for args in task_arguments]
try:
for future in as_completed(futures):
result = future.result()
@@ -172,10 +175,10 @@ RICH_CONSOLE = RichConsole(stderr=True)
def get_progressbar_class():
"""Return the default progress bar class."""
def partial_RichTqdmProgressAdapter(*args, **kwargs):
return RichTqdmProgressAdapter(*args, **kwargs, console=RICH_CONSOLE)
def partial_RichProgressBar(*args, **kwargs):
return RichProgressBar(*args, **kwargs, console=RICH_CONSOLE)
return partial_RichTqdmProgressAdapter
return partial_RichProgressBar
@hookimpl
src/ocrmypdf/cli.py
+2 -2
View File
@@ -193,8 +193,8 @@ Online documentation is located at:
help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for "
"long term archiving (default, recommended) but may not suitable "
"for users who want their file altered as little as possible. 'pdfa' "
"also has problems with full Unicode text. 'pdf' attempts to "
"preserve file contents as much as possible. 'pdf-a1' creates a "
"also has problems with full Unicode text. 'pdf' minimizes changes "
"to the input file. 'pdf-a1' creates a "
"PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'. 'pdf-a3' creates a "
"PDF/A3-b file. 'none' will produce no output, which may be helpful if "
"only the --sidecar is desired.",
src/ocrmypdf/extra_plugins/semfree.py
+5 -5
View File
@@ -94,7 +94,7 @@ def process_loop(
for args in task_args:
try:
result = task(args)
result = task(*args)
except Exception as e: # pylint: disable=broad-except
conn.send((MessageType.exception, e))
break
@@ -114,16 +114,16 @@ class LambdaExecutor(Executor):
*,
use_threads: bool,
max_workers: int,
tqdm_kwargs: dict,
progress_kwargs: dict,
worker_initializer: Callable,
task: Callable,
task_arguments: Iterable,
task_finished: Callable,
):
if use_threads and max_workers == 1:
with self.pbar_class(**tqdm_kwargs) as pbar:
with self.pbar_class(**progress_kwargs) as pbar:
for args in task_arguments:
result = task(args)
result = task(*args)
task_finished(result, pbar)
return
@@ -157,7 +157,7 @@ class LambdaExecutor(Executor):
for process in processes:
process.start()
with self.pbar_class(**tqdm_kwargs) as pbar:
with self.pbar_class(**progress_kwargs) as pbar:
while connections:
for result in wait(connections):
if not isinstance(result, Connection):
src/ocrmypdf/helpers.py
+6 -2
View File
@@ -319,12 +319,16 @@ def remove_all_log_handlers(logger: logging.Logger) -> None:
def pikepdf_enable_mmap() -> None:
"""Enable pikepdf mmap."""
"""Enable pikepdf memory mapping."""
try:
pikepdf._core.set_access_default_mmap(True)
log.debug(
"pikepdf mmap "
+ ('enabled' if pikepdf._core.get_access_default_mmap() else 'disabled')
+ (
'enabled'
if pikepdf._core.get_access_default_mmap() # type: ignore[attr-defined]
else 'disabled'
)
)
except AttributeError:
log.debug("pikepdf mmap not available")
src/ocrmypdf/languages.py
+849
View File
@@ -0,0 +1,849 @@
# SPDX-FileCopyrightText: 2023 James R. Barlow
# SPDX-License-Identifier: MPL-2.0
"""Language codes and names from ISO 639.
Derived from
https://www.loc.gov/standards/iso639-2/ascii_8bits.html
"""
from typing import NamedTuple
class ISOCodeData(NamedTuple):
"""Data for a single ISO 639 code."""
alt: str
alpha_2: str
english: str
french: str
ISO_639_3 = {
'aar': ISOCodeData('', 'aa', 'Afar', 'afar'),
'abk': ISOCodeData('', 'ab', 'Abkhazian', 'abkhaze'),
'ace': ISOCodeData('', '', 'Achinese', 'aceh'),
'ach': ISOCodeData('', '', 'Acoli', 'acoli'),
'ada': ISOCodeData('', '', 'Adangme', 'adangme'),
'ady': ISOCodeData('', '', 'Adyghe; Adygei', 'adyghé'),
'afa': ISOCodeData(
'',
'',
'Afro-Asiatic languages',
'afro-asiatiques, langues',
),
'afh': ISOCodeData('', '', 'Afrihili', 'afrihili'),
'afr': ISOCodeData('', 'af', 'Afrikaans', 'afrikaans'),
'ain': ISOCodeData('', '', 'Ainu', 'aïnou'),
'aka': ISOCodeData('', 'ak', 'Akan', 'akan'),
'akk': ISOCodeData('', '', 'Akkadian', 'akkadien'),
'alb': ISOCodeData('sqi', 'sq', 'Albanian', 'albanais'),
'ale': ISOCodeData('', '', 'Aleut', 'aléoute'),
'alg': ISOCodeData(
'',
'',
'Algonquian languages',
'algonquines, langues',
),
'alt': ISOCodeData('', '', 'Southern Altai', 'altai du Sud'),
'amh': ISOCodeData('', 'am', 'Amharic', 'amharique'),
'ang': ISOCodeData(
'',
'',
'English, Old (ca.450-1100)',
'anglo-saxon (ca.450-1100)',
),
'anp': ISOCodeData('', '', 'Angika', 'angika'),
'apa': ISOCodeData('', '', 'Apache languages', 'apaches, langues'),
'ara': ISOCodeData('', 'ar', 'Arabic', 'arabe'),
'arc': ISOCodeData(
'',
'',
'Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)',
"araméen d'empire (700-300 BCE)",
),
'arg': ISOCodeData('', 'an', 'Aragonese', 'aragonais'),
'arm': ISOCodeData('hye', 'hy', 'Armenian', 'arménien'),
'arn': ISOCodeData(
'',
'',
'Mapudungun; Mapuche',
'mapudungun; mapuche; mapuce',
),
'arp': ISOCodeData('', '', 'Arapaho', 'arapaho'),
'art': ISOCodeData(
'',
'',
'Artificial languages',
'artificielles, langues',
),
'arw': ISOCodeData('', '', 'Arawak', 'arawak'),
'asm': ISOCodeData('', 'as', 'Assamese', 'assamais'),
'ast': ISOCodeData(
'',
'',
'Asturian; Bable; Leonese; Asturleonese',
'asturien; bable; léonais; asturoléonais',
),
'ath': ISOCodeData(
'',
'',
'Athapascan languages',
'athapascanes, langues',
),
'aus': ISOCodeData(
'',
'',
'Australian languages',
'australiennes, langues',
),
'ava': ISOCodeData('', 'av', 'Avaric', 'avar'),
'ave': ISOCodeData('', 'ae', 'Avestan', 'avestique'),
'awa': ISOCodeData('', '', 'Awadhi', 'awadhi'),
'aym': ISOCodeData('', 'ay', 'Aymara', 'aymara'),
'aze': ISOCodeData('', 'az', 'Azerbaijani', 'azéri'),
'bad': ISOCodeData('', '', 'Banda languages', 'banda, langues'),
'bai': ISOCodeData('', '', 'Bamileke languages', 'bamiléké, langues'),
'bak': ISOCodeData('', 'ba', 'Bashkir', 'bachkir'),
'bal': ISOCodeData('', '', 'Baluchi', 'baloutchi'),
'bam': ISOCodeData('', 'bm', 'Bambara', 'bambara'),
'ban': ISOCodeData('', '', 'Balinese', 'balinais'),
'baq': ISOCodeData('eus', 'eu', 'Basque', 'basque'),
'bas': ISOCodeData('', '', 'Basa', 'basa'),
'bat': ISOCodeData('', '', 'Baltic languages', 'baltes, langues'),
'bej': ISOCodeData('', '', 'Beja; Bedawiyet', 'bedja'),
'bel': ISOCodeData('', 'be', 'Belarusian', 'biélorusse'),
'bem': ISOCodeData('', '', 'Bemba', 'bemba'),
'ben': ISOCodeData('', 'bn', 'Bengali', 'bengali'),
'ber': ISOCodeData('', '', 'Berber languages', 'berbères, langues'),
'bho': ISOCodeData('', '', 'Bhojpuri', 'bhojpuri'),
'bih': ISOCodeData('', 'bh', 'Bihari languages', 'langues biharis'),
'bik': ISOCodeData('', '', 'Bikol', 'bikol'),
'bin': ISOCodeData('', '', 'Bini; Edo', 'bini; edo'),
'bis': ISOCodeData('', 'bi', 'Bislama', 'bichlamar'),
'bla': ISOCodeData('', '', 'Siksika', 'blackfoot'),
'bnt': ISOCodeData('', '', 'Bantu languages', 'bantou, langues'),
'bos': ISOCodeData('', 'bs', 'Bosnian', 'bosniaque'),
'bra': ISOCodeData('', '', 'Braj', 'braj'),
'bre': ISOCodeData('', 'br', 'Breton', 'breton'),
'btk': ISOCodeData('', '', 'Batak languages', 'batak, langues'),
'bua': ISOCodeData('', '', 'Buriat', 'bouriate'),
'bug': ISOCodeData('', '', 'Buginese', 'bugi'),
'bul': ISOCodeData('', 'bg', 'Bulgarian', 'bulgare'),
'bur': ISOCodeData('mya', 'my', 'Burmese', 'birman'),
'byn': ISOCodeData('', '', 'Blin; Bilin', 'blin; bilen'),
'cad': ISOCodeData('', '', 'Caddo', 'caddo'),
'cai': ISOCodeData(
'',
'',
'Central American Indian languages',
"amérindiennes de L'Amérique centrale, langues",
),
'car': ISOCodeData('', '', 'Galibi Carib', 'karib; galibi; carib'),
'cat': ISOCodeData('', 'ca', 'Catalan; Valencian', 'catalan; valencien'),
'cau': ISOCodeData(
'',
'',
'Caucasian languages',
'caucasiennes, langues',
),
'ceb': ISOCodeData('', '', 'Cebuano', 'cebuano'),
'cel': ISOCodeData(
'',
'',
'Celtic languages',
'celtiques, langues; celtes, langues',
),
'cha': ISOCodeData('', 'ch', 'Chamorro', 'chamorro'),
'chb': ISOCodeData('', '', 'Chibcha', 'chibcha'),
'che': ISOCodeData('', 'ce', 'Chechen', 'tchétchène'),
'chg': ISOCodeData('', '', 'Chagatai', 'djaghataï'),
'chi': ISOCodeData('zho', 'zh', 'Chinese', 'chinois'),
'chk': ISOCodeData('', '', 'Chuukese', 'chuuk'),
'chm': ISOCodeData('', '', 'Mari', 'mari'),
'chn': ISOCodeData('', '', 'Chinook jargon', 'chinook, jargon'),
'cho': ISOCodeData('', '', 'Choctaw', 'choctaw'),
'chp': ISOCodeData('', '', 'Chipewyan; Dene Suline', 'chipewyan'),
'chr': ISOCodeData('', '', 'Cherokee', 'cherokee'),
'chu': ISOCodeData(
'',
'cu',
('Church Slavic; Old Slavonic; Church Slavonic;'
' Old Bulgarian; Old Church Slavonic'),
"slavon d'église; vieux slave; slavon liturgique; vieux bulgare",
),
'chv': ISOCodeData('', 'cv', 'Chuvash', 'tchouvache'),
'chy': ISOCodeData('', '', 'Cheyenne', 'cheyenne'),
'cmc': ISOCodeData('', '', 'Chamic languages', 'chames, langues'),
'cnr': ISOCodeData('', '', 'Montenegrin', 'monténégrin'),
'cop': ISOCodeData('', '', 'Coptic', 'copte'),
'cor': ISOCodeData('', 'kw', 'Cornish', 'cornique'),
'cos': ISOCodeData('', 'co', 'Corsican', 'corse'),
'cpe': ISOCodeData(
'',
'',
'Creoles and pidgins, English based',
"créoles et pidgins basés sur l'anglais",
),
'cpf': ISOCodeData(
'',
'',
'Creoles and pidgins, French-based',
'créoles et pidgins basés sur le français',
),
'cpp': ISOCodeData(
'',
'',
'Creoles and pidgins, Portuguese-based',
'créoles et pidgins basés sur le portugais',
),
'cre': ISOCodeData('', 'cr', 'Cree', 'cree'),
'crh': ISOCodeData(
'',
'',
'Crimean Tatar; Crimean Turkish',
'tatar de Crimé',
),
'crp': ISOCodeData('', '', 'Creoles and pidgins', 'créoles et pidgins'),
'csb': ISOCodeData('', '', 'Kashubian', 'kachoube'),
'cus': ISOCodeData('', '', 'Cushitic languages', 'couchitiques, langues'),
'cze': ISOCodeData('ces', 'cs', 'Czech', 'tchèque'),
'dak': ISOCodeData('', '', 'Dakota', 'dakota'),
'dan': ISOCodeData('', 'da', 'Danish', 'danois'),
'dar': ISOCodeData('', '', 'Dargwa', 'dargwa'),
'day': ISOCodeData('', '', 'Land Dayak languages', 'dayak, langues'),
'del': ISOCodeData('', '', 'Delaware', 'delaware'),
'den': ISOCodeData('', '', 'Slave (Athapascan)', 'esclave (athapascan)'),
'dgr': ISOCodeData('', '', 'Dogrib', 'dogrib'),
'din': ISOCodeData('', '', 'Dinka', 'dinka'),
'div': ISOCodeData('', 'dv', 'Divehi; Dhivehi; Maldivian', 'maldivien'),
'doi': ISOCodeData('', '', 'Dogri', 'dogri'),
'dra': ISOCodeData(
'',
'',
'Dravidian languages',
'dravidiennes, langues',
),
'dsb': ISOCodeData('', '', 'Lower Sorbian', 'bas-sorabe'),
'dua': ISOCodeData('', '', 'Duala', 'douala'),
'dum': ISOCodeData(
'',
'',
'Dutch, Middle (ca.1050-1350)',
'néerlandais moyen (ca. 1050-1350)',
),
'dut': ISOCodeData('nld', 'nl', 'Dutch; Flemish', 'néerlandais; flamand'),
'dyu': ISOCodeData('', '', 'Dyula', 'dioula'),
'dzo': ISOCodeData('', 'dz', 'Dzongkha', 'dzongkha'),
'efi': ISOCodeData('', '', 'Efik', 'efik'),
'egy': ISOCodeData('', '', 'Egyptian (Ancient)', 'égyptien'),
'eka': ISOCodeData('', '', 'Ekajuk', 'ekajuk'),
'elx': ISOCodeData('', '', 'Elamite', 'élamite'),
'eng': ISOCodeData('', 'en', 'English', 'anglais'),
'enm': ISOCodeData(
'',
'',
'English, Middle (1100-1500)',
'anglais moyen (1100-1500)',
),
'epo': ISOCodeData('', 'eo', 'Esperanto', 'espéranto'),
'est': ISOCodeData('', 'et', 'Estonian', 'estonien'),
'ewe': ISOCodeData('', 'ee', 'Ewe', 'éwé'),
'ewo': ISOCodeData('', '', 'Ewondo', 'éwondo'),
'fan': ISOCodeData('', '', 'Fang', 'fang'),
'fao': ISOCodeData('', 'fo', 'Faroese', 'féroïen'),
'fat': ISOCodeData('', '', 'Fanti', 'fanti'),
'fij': ISOCodeData('', 'fj', 'Fijian', 'fidjien'),
'fil': ISOCodeData('', '', 'Filipino; Pilipino', 'filipino; pilipino'),
'fin': ISOCodeData('', 'fi', 'Finnish', 'finnois'),
'fiu': ISOCodeData(
'',
'',
'Finno-Ugrian languages',
'finno-ougriennes, langues',
),
'fon': ISOCodeData('', '', 'Fon', 'fon'),
'fre': ISOCodeData('fra', 'fr', 'French', 'français'),
'frm': ISOCodeData(
'',
'',
'French, Middle (ca.1400-1600)',
'français moyen (1400-1600)',
),
'fro': ISOCodeData(
'',
'',
'French, Old (842-ca.1400)',
'français ancien (842-ca.1400)',
),
'frr': ISOCodeData('', '', 'Northern Frisian', 'frison septentrional'),
'frs': ISOCodeData('', '', 'Eastern Frisian', 'frison oriental'),
'fry': ISOCodeData('', 'fy', 'Western Frisian', 'frison occidental'),
'ful': ISOCodeData('', 'ff', 'Fulah', 'peul'),
'fur': ISOCodeData('', '', 'Friulian', 'frioulan'),
'gaa': ISOCodeData('', '', 'Ga', 'ga'),
'gay': ISOCodeData('', '', 'Gayo', 'gayo'),
'gba': ISOCodeData('', '', 'Gbaya', 'gbaya'),
'gem': ISOCodeData('', '', 'Germanic languages', 'germaniques, langues'),
'geo': ISOCodeData('kat', 'ka', 'Georgian', 'géorgien'),
'ger': ISOCodeData('deu', 'de', 'German', 'allemand'),
'gez': ISOCodeData('', '', 'Geez', 'guèze'),
'gil': ISOCodeData('', '', 'Gilbertese', 'kiribati'),
'gla': ISOCodeData(
'',
'gd',
'Gaelic; Scottish Gaelic',
'gaélique; gaélique écossais',
),
'gle': ISOCodeData('', 'ga', 'Irish', 'irlandais'),
'glg': ISOCodeData('', 'gl', 'Galician', 'galicien'),
'glv': ISOCodeData('', 'gv', 'Manx', 'manx; mannois'),
'gmh': ISOCodeData(
'',
'',
'German, Middle High (ca.1050-1500)',
'allemand, moyen haut (ca. 1050-1500)',
),
'goh': ISOCodeData(
'',
'',
'German, Old High (ca.750-1050)',
'allemand, vieux haut (ca. 750-1050)',
),
'gon': ISOCodeData('', '', 'Gondi', 'gond'),
'gor': ISOCodeData('', '', 'Gorontalo', 'gorontalo'),
'got': ISOCodeData('', '', 'Gothic', 'gothique'),
'grb': ISOCodeData('', '', 'Grebo', 'grebo'),
'grc': ISOCodeData(
'',
'',
'Greek, Ancient (to 1453)',
"grec ancien (jusqu'à 1453)",
),
'gre': ISOCodeData(
'ell',
'el',
'Greek, Modern (1453-)',
'grec moderne (après 1453)',
),
'grn': ISOCodeData('', 'gn', 'Guarani', 'guarani'),
'gsw': ISOCodeData(
'',
'',
'Swiss German; Alemannic; Alsatian',
'suisse alémanique; alémanique; alsacien',
),
'guj': ISOCodeData('', 'gu', 'Gujarati', 'goudjrati'),
'gwi': ISOCodeData('', '', "Gwich'in", "gwich'in"),
'hai': ISOCodeData('', '', 'Haida', 'haida'),
'hat': ISOCodeData(
'',
'ht',
'Haitian; Haitian Creole',
'haïtien; créole haïtien',
),
'hau': ISOCodeData('', 'ha', 'Hausa', 'haoussa'),
'haw': ISOCodeData('', '', 'Hawaiian', 'hawaïen'),
'heb': ISOCodeData('', 'he', 'Hebrew', 'hébreu'),
'her': ISOCodeData('', 'hz', 'Herero', 'herero'),
'hil': ISOCodeData('', '', 'Hiligaynon', 'hiligaynon'),
'him': ISOCodeData(
'',
'',
'Himachali languages; Western Pahari languages',
'langues himachalis; langues paharis occidentales',
),
'hin': ISOCodeData('', 'hi', 'Hindi', 'hindi'),
'hit': ISOCodeData('', '', 'Hittite', 'hittite'),
'hmn': ISOCodeData('', '', 'Hmong; Mong', 'hmong'),
'hmo': ISOCodeData('', 'ho', 'Hiri Motu', 'hiri motu'),
'hrv': ISOCodeData('', 'hr', 'Croatian', 'croate'),
'hsb': ISOCodeData('', '', 'Upper Sorbian', 'haut-sorabe'),
'hun': ISOCodeData('', 'hu', 'Hungarian', 'hongrois'),
'hup': ISOCodeData('', '', 'Hupa', 'hupa'),
'iba': ISOCodeData('', '', 'Iban', 'iban'),
'ibo': ISOCodeData('', 'ig', 'Igbo', 'igbo'),
'ice': ISOCodeData('isl', 'is', 'Icelandic', 'islandais'),
'ido': ISOCodeData('', 'io', 'Ido', 'ido'),
'iii': ISOCodeData('', 'ii', 'Sichuan Yi; Nuosu', 'yi de Sichuan'),
'ijo': ISOCodeData('', '', 'Ijo languages', 'ijo, langues'),
'iku': ISOCodeData('', 'iu', 'Inuktitut', 'inuktitut'),
'ile': ISOCodeData('', 'ie', 'Interlingue; Occidental', 'interlingue'),
'ilo': ISOCodeData('', '', 'Iloko', 'ilocano'),
'ina': ISOCodeData(
'',
'ia',
'Interlingua (International Auxiliary Language Association)',
'interlingua (langue auxiliaire internationale)',
),
'inc': ISOCodeData('', '', 'Indic languages', 'indo-aryennes, langues'),
'ind': ISOCodeData('', 'id', 'Indonesian', 'indonésien'),
'ine': ISOCodeData(
'',
'',
'Indo-European languages',
'indo-européennes, langues',
),
'inh': ISOCodeData('', '', 'Ingush', 'ingouche'),
'ipk': ISOCodeData('', 'ik', 'Inupiaq', 'inupiaq'),
'ira': ISOCodeData('', '', 'Iranian languages', 'iraniennes, langues'),
'iro': ISOCodeData('', '', 'Iroquoian languages', 'iroquoises, langues'),
'ita': ISOCodeData('', 'it', 'Italian', 'italien'),
'jav': ISOCodeData('', 'jv', 'Javanese', 'javanais'),
'jbo': ISOCodeData('', '', 'Lojban', 'lojban'),
'jpn': ISOCodeData('', 'ja', 'Japanese', 'japonais'),
'jpr': ISOCodeData('', '', 'Judeo-Persian', 'judéo-persan'),
'jrb': ISOCodeData('', '', 'Judeo-Arabic', 'judéo-arabe'),
'kaa': ISOCodeData('', '', 'Kara-Kalpak', 'karakalpak'),
'kab': ISOCodeData('', '', 'Kabyle', 'kabyle'),
'kac': ISOCodeData('', '', 'Kachin; Jingpho', 'kachin; jingpho'),
'kal': ISOCodeData('', 'kl', 'Kalaallisut; Greenlandic', 'groenlandais'),
'kam': ISOCodeData('', '', 'Kamba', 'kamba'),
'kan': ISOCodeData('', 'kn', 'Kannada', 'kannada'),
'kar': ISOCodeData('', '', 'Karen languages', 'karen, langues'),
'kas': ISOCodeData('', 'ks', 'Kashmiri', 'kashmiri'),
'kau': ISOCodeData('', 'kr', 'Kanuri', 'kanouri'),
'kaw': ISOCodeData('', '', 'Kawi', 'kawi'),
'kaz': ISOCodeData('', 'kk', 'Kazakh', 'kazakh'),
'kbd': ISOCodeData('', '', 'Kabardian', 'kabardien'),
'kha': ISOCodeData('', '', 'Khasi', 'khasi'),
'khi': ISOCodeData('', '', 'Khoisan languages', 'khoïsan, langues'),
'khm': ISOCodeData('', 'km', 'Central Khmer', 'khmer central'),
'kho': ISOCodeData('', '', 'Khotanese; Sakan', 'khotanais; sakan'),
'kik': ISOCodeData('', 'ki', 'Kikuyu; Gikuyu', 'kikuyu'),
'kin': ISOCodeData('', 'rw', 'Kinyarwanda', 'rwanda'),
'kir': ISOCodeData('', 'ky', 'Kirghiz; Kyrgyz', 'kirghiz'),
'kmb': ISOCodeData('', '', 'Kimbundu', 'kimbundu'),
'kok': ISOCodeData('', '', 'Konkani', 'konkani'),
'kom': ISOCodeData('', 'kv', 'Komi', 'kom'),
'kon': ISOCodeData('', 'kg', 'Kongo', 'kongo'),
'kor': ISOCodeData('', 'ko', 'Korean', 'coréen'),
'kos': ISOCodeData('', '', 'Kosraean', 'kosrae'),
'kpe': ISOCodeData('', '', 'Kpelle', 'kpellé'),
'krc': ISOCodeData('', '', 'Karachay-Balkar', 'karatchai balkar'),
'krl': ISOCodeData('', '', 'Karelian', 'carélien'),
'kro': ISOCodeData('', '', 'Kru languages', 'krou, langues'),
'kru': ISOCodeData('', '', 'Kurukh', 'kurukh'),
'kua': ISOCodeData('', 'kj', 'Kuanyama; Kwanyama', 'kuanyama; kwanyama'),
'kum': ISOCodeData('', '', 'Kumyk', 'koumyk'),
'kur': ISOCodeData('', 'ku', 'Kurdish', 'kurde'),
'kut': ISOCodeData('', '', 'Kutenai', 'kutenai'),
'lad': ISOCodeData('', '', 'Ladino', 'judéo-espagnol'),
'lah': ISOCodeData('', '', 'Lahnda', 'lahnda'),
'lam': ISOCodeData('', '', 'Lamba', 'lamba'),
'lao': ISOCodeData('', 'lo', 'Lao', 'lao'),
'lat': ISOCodeData('', 'la', 'Latin', 'latin'),
'lav': ISOCodeData('', 'lv', 'Latvian', 'letton'),
'lez': ISOCodeData('', '', 'Lezghian', 'lezghien'),
'lim': ISOCodeData(
'',
'li',
'Limburgan; Limburger; Limburgish',
'limbourgeois',
),
'lin': ISOCodeData('', 'ln', 'Lingala', 'lingala'),
'lit': ISOCodeData('', 'lt', 'Lithuanian', 'lituanien'),
'lol': ISOCodeData('', '', 'Mongo', 'mongo'),
'loz': ISOCodeData('', '', 'Lozi', 'lozi'),
'ltz': ISOCodeData(
'',
'lb',
'Luxembourgish; Letzeburgesch',
'luxembourgeois',
),
'lua': ISOCodeData('', '', 'Luba-Lulua', 'luba-lulua'),
'lub': ISOCodeData('', 'lu', 'Luba-Katanga', 'luba-katanga'),
'lug': ISOCodeData('', 'lg', 'Ganda', 'ganda'),
'lui': ISOCodeData('', '', 'Luiseno', 'luiseno'),
'lun': ISOCodeData('', '', 'Lunda', 'lunda'),
'luo': ISOCodeData(
'',
'',
'Luo (Kenya and Tanzania)',
'luo (Kenya et Tanzanie)',
),
'lus': ISOCodeData('', '', 'Lushai', 'lushai'),
'mac': ISOCodeData('mkd', 'mk', 'Macedonian', 'macédonien'),
'mad': ISOCodeData('', '', 'Madurese', 'madourais'),
'mag': ISOCodeData('', '', 'Magahi', 'magahi'),
'mah': ISOCodeData('', 'mh', 'Marshallese', 'marshall'),
'mai': ISOCodeData('', '', 'Maithili', 'maithili'),
'mak': ISOCodeData('', '', 'Makasar', 'makassar'),
'mal': ISOCodeData('', 'ml', 'Malayalam', 'malayalam'),
'man': ISOCodeData('', '', 'Mandingo', 'mandingue'),
'mao': ISOCodeData('mri', 'mi', 'Maori', 'maori'),
'map': ISOCodeData(
'',
'',
'Austronesian languages',
'austronésiennes, langues',
),
'mar': ISOCodeData('', 'mr', 'Marathi', 'marathe'),
'mas': ISOCodeData('', '', 'Masai', 'massaï'),
'may': ISOCodeData('msa', 'ms', 'Malay', 'malais'),
'mdf': ISOCodeData('', '', 'Moksha', 'moksa'),
'mdr': ISOCodeData('', '', 'Mandar', 'mandar'),
'men': ISOCodeData('', '', 'Mende', 'mendé'),
'mga': ISOCodeData(
'',
'',
'Irish, Middle (900-1200)',
'irlandais moyen (900-1200)',
),
'mic': ISOCodeData('', '', "Mi'kmaq; Micmac", "mi'kmaq; micmac"),
'min': ISOCodeData('', '', 'Minangkabau', 'minangkabau'),
'mis': ISOCodeData('', '', 'Uncoded languages', 'langues non codées'),
'mkh': ISOCodeData('', '', 'Mon-Khmer languages', 'môn-khmer, langues'),
'mlg': ISOCodeData('', 'mg', 'Malagasy', 'malgache'),
'mlt': ISOCodeData('', 'mt', 'Maltese', 'maltais'),
'mnc': ISOCodeData('', '', 'Manchu', 'mandchou'),
'mni': ISOCodeData('', '', 'Manipuri', 'manipuri'),
'mno': ISOCodeData('', '', 'Manobo languages', 'manobo, langues'),
'moh': ISOCodeData('', '', 'Mohawk', 'mohawk'),
'mon': ISOCodeData('', 'mn', 'Mongolian', 'mongol'),
'mos': ISOCodeData('', '', 'Mossi', 'moré'),
'mul': ISOCodeData('', '', 'Multiple languages', 'multilingue'),
'mun': ISOCodeData('', '', 'Munda languages', 'mounda, langues'),
'mus': ISOCodeData('', '', 'Creek', 'muskogee'),
'mwl': ISOCodeData('', '', 'Mirandese', 'mirandais'),
'mwr': ISOCodeData('', '', 'Marwari', 'marvari'),
'myn': ISOCodeData('', '', 'Mayan languages', 'maya, langues'),
'myv': ISOCodeData('', '', 'Erzya', 'erza'),
'nah': ISOCodeData('', '', 'Nahuatl languages', 'nahuatl, langues'),
'nai': ISOCodeData(
'',
'',
'North American Indian languages',
'nord-amérindiennes, langues',
),
'nap': ISOCodeData('', '', 'Neapolitan', 'napolitain'),
'nau': ISOCodeData('', 'na', 'Nauru', 'nauruan'),
'nav': ISOCodeData('', 'nv', 'Navajo; Navaho', 'navaho'),
'nbl': ISOCodeData(
'',
'nr',
'Ndebele, South; South Ndebele',
'ndébélé du Sud',
),
'nde': ISOCodeData(
'',
'nd',
'Ndebele, North; North Ndebele',
'ndébélé du Nord',
),
'ndo': ISOCodeData('', 'ng', 'Ndonga', 'ndonga'),
'nds': ISOCodeData(
'',
'',
'Low German; Low Saxon; German, Low; Saxon, Low',
'bas allemand; bas saxon; allemand, bas; saxon, bas',
),
'nep': ISOCodeData('', 'ne', 'Nepali', 'népalais'),
'new': ISOCodeData('', '', 'Nepal Bhasa; Newari', 'nepal bhasa; newari'),
'nia': ISOCodeData('', '', 'Nias', 'nias'),
'nic': ISOCodeData(
'',
'',
'Niger-Kordofanian languages',
'nigéro-kordofaniennes, langues',
),
'niu': ISOCodeData('', '', 'Niuean', 'niué'),
'nno': ISOCodeData(
'',
'nn',
'Norwegian Nynorsk; Nynorsk, Norwegian',
'norvégien nynorsk; nynorsk, norvégien',
),
'nob': ISOCodeData(
'',
'nb',
'Bokmål, Norwegian; Norwegian Bokmål',
'norvégien bokmål',
),
'nog': ISOCodeData('', '', 'Nogai', 'nogaï; nogay'),
'non': ISOCodeData('', '', 'Norse, Old', 'norrois, vieux'),
'nor': ISOCodeData('', 'no', 'Norwegian', 'norvégien'),
'nqo': ISOCodeData('', '', "N'Ko", "n'ko"),
'nso': ISOCodeData(
'',
'',
'Pedi; Sepedi; Northern Sotho',
'pedi; sepedi; sotho du Nord',
),
'nub': ISOCodeData('', '', 'Nubian languages', 'nubiennes, langues'),
'nwc': ISOCodeData(
'',
'',
'Classical Newari; Old Newari; Classical Nepal Bhasa',
'newari classique',
),
'nya': ISOCodeData(
'',
'ny',
'Chichewa; Chewa; Nyanja',
'chichewa; chewa; nyanja',
),
'nym': ISOCodeData('', '', 'Nyamwezi', 'nyamwezi'),
'nyn': ISOCodeData('', '', 'Nyankole', 'nyankolé'),
'nyo': ISOCodeData('', '', 'Nyoro', 'nyoro'),
'nzi': ISOCodeData('', '', 'Nzima', 'nzema'),
'oci': ISOCodeData(
'',
'oc',
'Occitan (post 1500)',
'occitan (après 1500)',
),
'oji': ISOCodeData('', 'oj', 'Ojibwa', 'ojibwa'),
'ori': ISOCodeData('', 'or', 'Oriya', 'oriya'),
'orm': ISOCodeData('', 'om', 'Oromo', 'galla'),
'osa': ISOCodeData('', '', 'Osage', 'osage'),
'oss': ISOCodeData('', 'os', 'Ossetian; Ossetic', 'ossète'),
'ota': ISOCodeData(
'',
'',
'Turkish, Ottoman (1500-1928)',
'turc ottoman (1500-1928)',
),
'oto': ISOCodeData('', '', 'Otomian languages', 'otomi, langues'),
'paa': ISOCodeData('', '', 'Papuan languages', 'papoues, langues'),
'pag': ISOCodeData('', '', 'Pangasinan', 'pangasinan'),
'pal': ISOCodeData('', '', 'Pahlavi', 'pahlavi'),
'pam': ISOCodeData('', '', 'Pampanga; Kapampangan', 'pampangan'),
'pan': ISOCodeData('', 'pa', 'Panjabi; Punjabi', 'pendjabi'),
'pap': ISOCodeData('', '', 'Papiamento', 'papiamento'),
'pau': ISOCodeData('', '', 'Palauan', 'palau'),
'peo': ISOCodeData(
'',
'',
'Persian, Old (ca.600-400 B.C.)',
'perse, vieux (ca. 600-400 av. J.-C.)',
),
'per': ISOCodeData('fas', 'fa', 'Persian', 'persan'),
'phi': ISOCodeData(
'',
'',
'Philippine languages',
'philippines, langues',
),
'phn': ISOCodeData('', '', 'Phoenician', 'phénicien'),
'pli': ISOCodeData('', 'pi', 'Pali', 'pali'),
'pol': ISOCodeData('', 'pl', 'Polish', 'polonais'),
'pon': ISOCodeData('', '', 'Pohnpeian', 'pohnpei'),
'por': ISOCodeData('', 'pt', 'Portuguese', 'portugais'),
'pra': ISOCodeData('', '', 'Prakrit languages', 'prâkrit, langues'),
'pro': ISOCodeData(
'',
'',
'Provençal, Old (to 1500); Occitan, Old (to 1500)',
"provençal ancien (jusqu'à 1500); occitan ancien (jusqu'à 1500)",
),
'pus': ISOCodeData('', 'ps', 'Pushto; Pashto', 'pachto'),
'qaa': ISOCodeData(
'',
'',
'Reserved for local use',
"réservée à l'usage local",
),
'que': ISOCodeData('', 'qu', 'Quechua', 'quechua'),
'raj': ISOCodeData('', '', 'Rajasthani', 'rajasthani'),
'rap': ISOCodeData('', '', 'Rapanui', 'rapanui'),
'rar': ISOCodeData(
'',
'',
'Rarotongan; Cook Islands Maori',
'rarotonga; maori des îles Cook',
),
'roa': ISOCodeData('', '', 'Romance languages', 'romanes, langues'),
'roh': ISOCodeData('', 'rm', 'Romansh', 'romanche'),
'rom': ISOCodeData('', '', 'Romany', 'tsigane'),
'rum': ISOCodeData(
'ron',
'ro',
'Romanian; Moldavian; Moldovan',
'roumain; moldave',
),
'run': ISOCodeData('', 'rn', 'Rundi', 'rundi'),
'rup': ISOCodeData(
'',
'',
'Aromanian; Arumanian; Macedo-Romanian',
'aroumain; macédo-roumain',
),
'rus': ISOCodeData('', 'ru', 'Russian', 'russe'),
'sad': ISOCodeData('', '', 'Sandawe', 'sandawe'),
'sag': ISOCodeData('', 'sg', 'Sango', 'sango'),
'sah': ISOCodeData('', '', 'Yakut', 'iakoute'),
'sai': ISOCodeData(
'',
'',
'South American Indian languages',
'sud-amérindiennes, langues',
),
'sal': ISOCodeData('', '', 'Salishan languages', 'salishennes, langues'),
'sam': ISOCodeData('', '', 'Samaritan Aramaic', 'samaritain'),
'san': ISOCodeData('', 'sa', 'Sanskrit', 'sanskrit'),
'sas': ISOCodeData('', '', 'Sasak', 'sasak'),
'sat': ISOCodeData('', '', 'Santali', 'santal'),
'scn': ISOCodeData('', '', 'Sicilian', 'sicilien'),
'sco': ISOCodeData('', '', 'Scots', 'écossais'),
'sel': ISOCodeData('', '', 'Selkup', 'selkoupe'),
'sem': ISOCodeData('', '', 'Semitic languages', 'sémitiques, langues'),
'sga': ISOCodeData(
'',
'',
'Irish, Old (to 900)',
"irlandais ancien (jusqu'à 900)",
),
'sgn': ISOCodeData('', '', 'Sign Languages', 'langues des signes'),
'shn': ISOCodeData('', '', 'Shan', 'chan'),
'sid': ISOCodeData('', '', 'Sidamo', 'sidamo'),
'sin': ISOCodeData('', 'si', 'Sinhala; Sinhalese', 'singhalais'),
'sio': ISOCodeData('', '', 'Siouan languages', 'sioux, langues'),
'sit': ISOCodeData(
'',
'',
'Sino-Tibetan languages',
'sino-tibétaines, langues',
),
'sla': ISOCodeData('', '', 'Slavic languages', 'slaves, langues'),
'slo': ISOCodeData('slk', 'sk', 'Slovak', 'slovaque'),
'slv': ISOCodeData('', 'sl', 'Slovenian', 'slovène'),
'sma': ISOCodeData('', '', 'Southern Sami', 'sami du Sud'),
'sme': ISOCodeData('', 'se', 'Northern Sami', 'sami du Nord'),
'smi': ISOCodeData('', '', 'Sami languages', 'sames, langues'),
'smj': ISOCodeData('', '', 'Lule Sami', 'sami de Lule'),
'smn': ISOCodeData('', '', 'Inari Sami', "sami d'Inari"),
'smo': ISOCodeData('', 'sm', 'Samoan', 'samoan'),
'sms': ISOCodeData('', '', 'Skolt Sami', 'sami skolt'),
'sna': ISOCodeData('', 'sn', 'Shona', 'shona'),
'snd': ISOCodeData('', 'sd', 'Sindhi', 'sindhi'),
'snk': ISOCodeData('', '', 'Soninke', 'soninké'),
'sog': ISOCodeData('', '', 'Sogdian', 'sogdien'),
'som': ISOCodeData('', 'so', 'Somali', 'somali'),
'son': ISOCodeData('', '', 'Songhai languages', 'songhai, langues'),
'sot': ISOCodeData('', 'st', 'Sotho, Southern', 'sotho du Sud'),
'spa': ISOCodeData('', 'es', 'Spanish; Castilian', 'espagnol; castillan'),
'srd': ISOCodeData('', 'sc', 'Sardinian', 'sarde'),
'srn': ISOCodeData('', '', 'Sranan Tongo', 'sranan tongo'),
'srp': ISOCodeData('', 'sr', 'Serbian', 'serbe'),
'srr': ISOCodeData('', '', 'Serer', 'sérère'),
'ssa': ISOCodeData(
'',
'',
'Nilo-Saharan languages',
'nilo-sahariennes, langues',
),
'ssw': ISOCodeData('', 'ss', 'Swati', 'swati'),
'suk': ISOCodeData('', '', 'Sukuma', 'sukuma'),
'sun': ISOCodeData('', 'su', 'Sundanese', 'soundanais'),
'sus': ISOCodeData('', '', 'Susu', 'soussou'),
'sux': ISOCodeData('', '', 'Sumerian', 'sumérien'),
'swa': ISOCodeData('', 'sw', 'Swahili', 'swahili'),
'swe': ISOCodeData('', 'sv', 'Swedish', 'suédois'),
'syc': ISOCodeData('', '', 'Classical Syriac', 'syriaque classique'),
'syr': ISOCodeData('', '', 'Syriac', 'syriaque'),
'tah': ISOCodeData('', 'ty', 'Tahitian', 'tahitien'),
'tai': ISOCodeData('', '', 'Tai languages', 'tai, langues'),
'tam': ISOCodeData('', 'ta', 'Tamil', 'tamoul'),
'tat': ISOCodeData('', 'tt', 'Tatar', 'tatar'),
'tel': ISOCodeData('', 'te', 'Telugu', 'télougou'),
'tem': ISOCodeData('', '', 'Timne', 'temne'),
'ter': ISOCodeData('', '', 'Tereno', 'tereno'),
'tet': ISOCodeData('', '', 'Tetum', 'tetum'),
'tgk': ISOCodeData('', 'tg', 'Tajik', 'tadjik'),
'tgl': ISOCodeData('', 'tl', 'Tagalog', 'tagalog'),
'tha': ISOCodeData('', 'th', 'Thai', 'thaï'),
'tib': ISOCodeData('bod', 'bo', 'Tibetan', 'tibétain'),
'tig': ISOCodeData('', '', 'Tigre', 'tigré'),
'tir': ISOCodeData('', 'ti', 'Tigrinya', 'tigrigna'),
'tiv': ISOCodeData('', '', 'Tiv', 'tiv'),
'tkl': ISOCodeData('', '', 'Tokelau', 'tokelau'),
'tlh': ISOCodeData('', '', 'Klingon; tlhIngan-Hol', 'klingon'),
'tli': ISOCodeData('', '', 'Tlingit', 'tlingit'),
'tmh': ISOCodeData('', '', 'Tamashek', 'tamacheq'),
'tog': ISOCodeData('', '', 'Tonga (Nyasa)', 'tonga (Nyasa)'),
'ton': ISOCodeData(
'',
'to',
'Tonga (Tonga Islands)',
'tongan (Îles Tonga)',
),
'tpi': ISOCodeData('', '', 'Tok Pisin', 'tok pisin'),
'tsi': ISOCodeData('', '', 'Tsimshian', 'tsimshian'),
'tsn': ISOCodeData('', 'tn', 'Tswana', 'tswana'),
'tso': ISOCodeData('', 'ts', 'Tsonga', 'tsonga'),
'tuk': ISOCodeData('', 'tk', 'Turkmen', 'turkmène'),
'tum': ISOCodeData('', '', 'Tumbuka', 'tumbuka'),
'tup': ISOCodeData('', '', 'Tupi languages', 'tupi, langues'),
'tur': ISOCodeData('', 'tr', 'Turkish', 'turc'),
'tut': ISOCodeData('', '', 'Altaic languages', 'altaïques, langues'),
'tvl': ISOCodeData('', '', 'Tuvalu', 'tuvalu'),
'twi': ISOCodeData('', 'tw', 'Twi', 'twi'),
'tyv': ISOCodeData('', '', 'Tuvinian', 'touva'),
'udm': ISOCodeData('', '', 'Udmurt', 'oudmourte'),
'uga': ISOCodeData('', '', 'Ugaritic', 'ougaritique'),
'uig': ISOCodeData('', 'ug', 'Uighur; Uyghur', 'ouïgour'),
'ukr': ISOCodeData('', 'uk', 'Ukrainian', 'ukrainien'),
'umb': ISOCodeData('', '', 'Umbundu', 'umbundu'),
'und': ISOCodeData('', '', 'Undetermined', 'indéterminée'),
'urd': ISOCodeData('', 'ur', 'Urdu', 'ourdou'),
'uzb': ISOCodeData('', 'uz', 'Uzbek', 'ouszbek'),
'vai': ISOCodeData('', '', 'Vai', 'vaï'),
'ven': ISOCodeData('', 've', 'Venda', 'venda'),
'vie': ISOCodeData('', 'vi', 'Vietnamese', 'vietnamien'),
'vol': ISOCodeData('', 'vo', 'Volapük', 'volapük'),
'vot': ISOCodeData('', '', 'Votic', 'vote'),
'wak': ISOCodeData('', '', 'Wakashan languages', 'wakashanes, langues'),
'wal': ISOCodeData('', '', 'Wolaitta; Wolaytta', 'wolaitta; wolaytta'),
'war': ISOCodeData('', '', 'Waray', 'waray'),
'was': ISOCodeData('', '', 'Washo', 'washo'),
'wel': ISOCodeData('cym', 'cy', 'Welsh', 'gallois'),
'wen': ISOCodeData('', '', 'Sorbian languages', 'sorabes, langues'),
'wln': ISOCodeData('', 'wa', 'Walloon', 'wallon'),
'wol': ISOCodeData('', 'wo', 'Wolof', 'wolof'),
'xal': ISOCodeData('', '', 'Kalmyk; Oirat', 'kalmouk; oïrat'),
'xho': ISOCodeData('', 'xh', 'Xhosa', 'xhosa'),
'yao': ISOCodeData('', '', 'Yao', 'yao'),
'yap': ISOCodeData('', '', 'Yapese', 'yapois'),
'yid': ISOCodeData('', 'yi', 'Yiddish', 'yiddish'),
'yor': ISOCodeData('', 'yo', 'Yoruba', 'yoruba'),
'ypk': ISOCodeData('', '', 'Yupik languages', 'yupik, langues'),
'zap': ISOCodeData('', '', 'Zapotec', 'zapotèque'),
'zbl': ISOCodeData(
'',
'',
'Blissymbols; Blissymbolics; Bliss',
'symboles Bliss; Bliss',
),
'zen': ISOCodeData('', '', 'Zenaga', 'zenaga'),
'zgh': ISOCodeData(
'',
'',
'Standard Moroccan Tamazight',
'amazighe standard marocain',
),
'zha': ISOCodeData('', 'za', 'Zhuang; Chuang', 'zhuang; chuang'),
'znd': ISOCodeData('', '', 'Zande languages', 'zandé, langues'),
'zul': ISOCodeData('', 'zu', 'Zulu', 'zoulou'),
'zun': ISOCodeData('', '', 'Zuni', 'zuni'),
'zxx': ISOCodeData(
'',
'',
'No linguistic content; Not applicable',
'pas de contenu linguistique; non applicable',
),
'zza': ISOCodeData(
'',
'',
'Zaza; Dimili; Dimli; Kirdki; Kirmanjki; Zazaki',
'zaza; dimili; dimli; kirdki; kirmanjki; zazaki',
),
}
def iso_639_2_from_3(iso3: str) -> str:
"""Convert ISO 639-3 code to ISO 639-2 code."""
if iso3 in ISO_639_3:
return ISO_639_3[iso3].alpha_2
else:
return ""
src/ocrmypdf/optimize.py
+31 -25
View File
@@ -14,7 +14,8 @@ from collections import defaultdict
from collections.abc import Iterator, MutableSet, Sequence
from os import fspath
from pathlib import Path
from typing import Callable, NamedTuple, NewType
from typing import Any, Callable, NamedTuple, NewType
from warnings import warn
from zlib import compress
import img2pdf
@@ -34,6 +35,7 @@ from PIL import Image
from ocrmypdf._concurrent import Executor, SerialExecutor
from ocrmypdf._exec import jbig2enc, pngquant
from ocrmypdf._jobcontext import PdfContext
from ocrmypdf._progressbar import ProgressBar
from ocrmypdf.exceptions import OutputFileAccessError
from ocrmypdf.helpers import IMG2PDF_KWARGS, safe_symlink
@@ -69,11 +71,14 @@ def jpg_name(root: Path, xref: Xref) -> Path:
def extract_image_filter(
pdf: Pdf, root: Path, image: Stream, xref: Xref
image: Stream, xref: Xref, *args
) -> tuple[PdfImage, tuple[Name, Object]] | None:
"""Determine if an image is extractable."""
del pdf # unused args
del root
if isinstance(image, Pdf):
# Support deprecated old function signature
# TODO Remove for v16 and drop *args from current function signature
image, xref = args[0], args[1]
warn("extract_image_filter: pdf, root parameters ignored", DeprecationWarning)
if image.Subtype != Name.Image:
return None
@@ -132,7 +137,7 @@ def extract_image_jbig2(
"""Extract an image, saving it as a JBIG2 file."""
del options # unused arg
result = extract_image_filter(pdf, root, image, xref)
result = extract_image_filter(image, xref)
if result is None:
return None
pim, filtdp = result
@@ -172,7 +177,7 @@ def extract_image_generic(
*, pdf: Pdf, root: Path, image: Stream, xref: Xref, options
) -> XrefExt | None:
"""Generic image extraction."""
result = extract_image_filter(pdf, root, image, xref)
result = extract_image_filter(image, xref)
if result is None:
return None
pim, filtdp = result
@@ -283,7 +288,7 @@ def _find_image_xrefs(pdf: Pdf):
for pageno, page in enumerate(pdf.pages):
_find_image_xrefs_container(
pdf, page, pageno, include_xrefs, exclude_xrefs, pageno_for_xref
pdf, page.obj, pageno, include_xrefs, exclude_xrefs, pageno_for_xref
)
working_xrefs = include_xrefs - exclude_xrefs
@@ -386,15 +391,15 @@ def _produce_jbig2_images(
if options.jbig2_page_group_size > 1:
jbig2_args = jbig2_group_args
jbig2_convert = jbig2enc.convert_group_mp
jbig2_convert = jbig2enc.convert_group
else:
jbig2_args = jbig2_single_args
jbig2_convert = jbig2enc.convert_single_mp
jbig2_convert = jbig2enc.convert_single
executor(
use_threads=True,
max_workers=options.jobs,
tqdm_kwargs=dict(
progress_kwargs=dict(
total=len(jbig2_groups),
desc="JBIG2",
unit='item',
@@ -450,9 +455,9 @@ def convert_to_jbig2(
)
def _optimize_jpeg(args: tuple[Xref, Path, Path, int]) -> tuple[Xref, Path | None]:
xref, in_jpg, opt_jpg, jpeg_quality = args
def _optimize_jpeg(
xref: Xref, in_jpg: Path, opt_jpg: Path, jpeg_quality: int
) -> tuple[Xref, Path | None]:
with Image.open(in_jpg) as im:
im.save(opt_jpg, optimize=True, quality=jpeg_quality)
@@ -474,7 +479,7 @@ def transcode_jpegs(
opt_jpg = in_jpg.with_suffix('.opt.jpg')
yield xref, in_jpg, opt_jpg, options.jpeg_quality
def finish_jpeg(result: tuple[Xref, Path | None], pbar):
def finish_jpeg(result: tuple[Xref, Path | None], pbar: ProgressBar):
xref, opt_jpg = result
if opt_jpg:
compdata = opt_jpg.read_bytes() # JPEG can inserted into PDF as is
@@ -485,7 +490,7 @@ def transcode_jpegs(
executor(
use_threads=True, # Processes are significantly slower at this task
max_workers=options.jobs,
tqdm_kwargs=dict(
progress_kwargs=dict(
desc="Recompressing JPEGs",
total=len(jpegs),
unit='image',
@@ -500,7 +505,7 @@ def transcode_jpegs(
def _find_deflatable_jpeg(
*, pdf: Pdf, root: Path, image: Stream, xref: Xref, options
) -> XrefExt | None:
result = extract_image_filter(pdf, root, image, xref)
result = extract_image_filter(image, xref)
if result is None:
return None
_pim, filtdp = result
@@ -511,8 +516,9 @@ def _find_deflatable_jpeg(
return None
def _deflate_jpeg(args: tuple[Pdf, threading.Lock, Xref, int]) -> tuple[Xref, bytes]:
pdf, lock, xref, complevel = args
def _deflate_jpeg(
pdf: Pdf, lock: threading.Lock, xref: Xref, complevel: int
) -> tuple[Xref, bytes]:
with lock:
xobj = pdf.get_object(xref, 0)
try:
@@ -547,7 +553,7 @@ def deflate_jpegs(pdf: Pdf, root: Path, options, executor: Executor) -> None:
for xref in jpegs:
yield pdf, lock, xref, complevel
def finish(result, pbar):
def finish(result: tuple[Xref, bytes], pbar: ProgressBar):
xref, compdata = result
if len(compdata) > 0:
with lock:
@@ -558,7 +564,7 @@ def deflate_jpegs(pdf: Pdf, root: Path, options, executor: Executor) -> None:
executor(
use_threads=True, # We're sharing the pdf directly, must use threads
max_workers=options.jobs,
tqdm_kwargs=dict(
progress_kwargs=dict(
desc="Deflating JPEGs",
total=len(jpegs),
unit='image',
@@ -617,7 +623,7 @@ def transcode_pngs(
image_name_fn: Callable[[Path, Xref], Path],
root: Path,
options,
executor,
executor: Executor,
) -> None:
"""Apply lossy transcoding to PNGs."""
modified: MutableSet[Xref] = set()
@@ -641,13 +647,13 @@ def transcode_pngs(
executor(
use_threads=True,
max_workers=options.jobs,
tqdm_kwargs=dict(
progress_kwargs=dict(
desc="PNGs",
total=len(images),
unit='image',
disable=not options.progress_bar,
),
task=pngquant.quantize_mp,
task=pngquant.quantize,
task_arguments=pngquant_args(),
)
@@ -662,8 +668,8 @@ DEFAULT_EXECUTOR = SerialExecutor()
def optimize(
input_file: Path,
output_file: Path,
context,
save_settings,
context: PdfContext,
save_settings: dict[str, Any],
executor: Executor = DEFAULT_EXECUTOR,
) -> Path:
"""Optimize images in a PDF file."""
src/ocrmypdf/pdfinfo/info.py
+76 -41
View File
@@ -13,7 +13,7 @@ import statistics
import sys
from collections import defaultdict
from collections.abc import Container, Iterable, Iterator, Mapping, Sequence
from contextlib import ExitStack
from contextlib import contextmanager
from decimal import Decimal
from enum import Enum, auto
from functools import partial
@@ -32,11 +32,13 @@ from pikepdf import (
PdfImage,
PdfInlineImage,
PdfMatrix,
Stream,
UnsupportedImageTypeError,
parse_content_stream,
)
from ocrmypdf._concurrent import Executor, SerialExecutor
from ocrmypdf._progressbar import ProgressBar
from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
from ocrmypdf.helpers import Resolution, available_cpu_count, pikepdf_enable_mmap
from ocrmypdf.pdfinfo.layout import LTStateAwareChar, get_page_analysis, get_text_boxes
@@ -357,7 +359,7 @@ class ImageInfo:
if inline is not None:
self._origin = 'inline'
pim = inline
elif pdfimage is not None:
elif pdfimage is not None and isinstance(pdfimage, Stream):
self._origin = 'xobject'
pim = PdfImage(pdfimage)
else:
@@ -386,33 +388,43 @@ class ImageInfo:
if self._enc == Encoding.jpeg2000:
self._color = Colorspace.jpeg2000
if self._color == Colorspace.icc:
# Check the ICC profile to determine actual colorspace
try:
pim_icc = pim.icc
if pim_icc.profile.xcolor_space == 'GRAY':
self._comp = 1
elif pim_icc.profile.xcolor_space == 'CMYK':
self._comp = 4
else:
self._comp = 3
except (AttributeError, UnsupportedImageTypeError) as ex:
self._comp = None
logger.warning(
f"An image with a corrupt or unreadable ICC profile was found. "
f"The output PDF may not match the input PDF visually: {ex}. {self}"
)
self._comp = None
if self._color == Colorspace.icc and isinstance(pim, PdfImage):
self._comp = self._init_icc(pim)
else:
if isinstance(self._color, Colorspace):
self._comp = FRIENDLY_COMP.get(self._color)
else:
self._comp = None
# Bit of a hack... infer grayscale if component count is uncertain
# but encoding only supports monochrome.
if self._comp is None and self._enc in (Encoding.ccitt, Encoding.jbig2):
self._comp = FRIENDLY_COMP[Colorspace.gray]
def _init_icc(self, pim: PdfImage):
try:
icc = pim.icc
except UnsupportedImageTypeError as e:
logger.warning(
f"An image with a corrupt or unreadable ICC profile was found. "
f"Output PDF may not match the input PDF visually: {e}. {self}"
)
return None
# Check the ICC profile to determine actual colorspace
if icc is None or not hasattr(icc, 'profile'):
logger.warning(
f"An image with an ICC profile but no ICC profile data was found. "
f"The output PDF may not match the input PDF visually. {self}"
)
return None
try:
if icc.profile.xcolor_space == 'GRAY':
return 1
elif icc.profile.xcolor_space == 'CMYK':
return 4
else:
return 3
except AttributeError:
return None
@property
def name(self):
"""Name of the image as it appears in the PDF."""
@@ -693,29 +705,41 @@ def _pdf_pageinfo_sync_init(pdf: Pdf, infile: Path, pdfminer_loglevel):
atexit.register(on_process_close)
def _pdf_pageinfo_sync(args):
pageno, thread_pdf, infile, check_pages, detailed_analysis = args
pdf = thread_pdf if thread_pdf is not None else worker_pdf
with ExitStack() as stack:
if not pdf: # When called with SerialExecutor
pdf = stack.enter_context(Pdf.open(infile))
page = PageInfo(pdf, pageno, infile, check_pages, detailed_analysis)
return page
@contextmanager
def _pdf_pageinfo_sync_pdf(thread_pdf: Pdf | None, infile: Path):
if thread_pdf is not None:
yield thread_pdf
elif worker_pdf is not None:
yield worker_pdf
else:
with Pdf.open(infile) as pdf:
yield pdf
def _pdf_pageinfo_sync(
pageno: int,
thread_pdf: Pdf | None,
infile: Path,
check_pages: Container[int],
detailed_analysis: bool,
) -> PageInfo:
with _pdf_pageinfo_sync_pdf(thread_pdf, infile) as pdf:
return PageInfo(pdf, pageno, infile, check_pages, detailed_analysis)
def _pdf_pageinfo_concurrent(
pdf,
executor: Executor,
max_workers: int,
use_threads: bool,
infile,
progbar,
max_workers,
check_pages,
detailed_analysis=False,
detailed_analysis: bool = False,
) -> Sequence[PageInfo | None]:
pages: Sequence[PageInfo | None] = [None] * len(pdf.pages)
pages: list[PageInfo | None] = [None] * len(pdf.pages)
def update_pageinfo(result, pbar):
page = result
def update_pageinfo(page: PageInfo, pbar: ProgressBar):
if not page:
raise InputFileError("Could read a page in the PDF")
pages[page.pageno] = page
@@ -726,13 +750,17 @@ def _pdf_pageinfo_concurrent(
total = len(pdf.pages)
use_threads = False # No performance gain if threaded due to GIL
n_workers = min(1 + len(pages) // 4, max_workers)
if n_workers == 1:
# But if we decided on only one worker, there is no point in using
# If we decided on only one worker, there is no point in using
# a separate process.
use_threads = True
if use_threads and n_workers > 1:
# If we are using threads, there is no point in using more than one
# worker thread - they will just fight over the GIL.
n_workers = 1
# If we use a thread, we can pass the already-open Pdf for them to use
# If we use processes, we pass a None which tells the init function to open its
# own
@@ -742,10 +770,15 @@ def _pdf_pageinfo_concurrent(
(n, initial_pdf, infile, check_pages, detailed_analysis) for n in range(total)
)
assert n_workers == 1 if use_threads else n_workers >= 1, "Not multithreadable"
logger.debug(
f"Gathering info with {n_workers} "
+ ('thread' if use_threads else 'process')
+ " workers"
)
executor(
use_threads=use_threads,
max_workers=n_workers,
tqdm_kwargs=dict(
progress_kwargs=dict(
total=total, desc="Scanning contents", unit='page', disable=not progbar
),
worker_initializer=partial(
@@ -817,7 +850,7 @@ class PageInfo:
detailed_analysis: bool,
):
page: Page = pdf.pages[pageno]
mediabox = [Decimal(d) for d in page.MediaBox.as_list()]
mediabox = [Decimal(d) for d in page.mediabox.as_list()]
width_pt = mediabox[2] - mediabox[0]
height_pt = mediabox[3] - mediabox[1]
@@ -1050,11 +1083,12 @@ class PdfInfo:
def __init__(
self,
infile,
infile: Path,
*,
detailed_analysis: bool = False,
progbar: bool = False,
max_workers: int | None = None,
use_threads: bool = True,
check_pages=None,
executor: Executor = DEFAULT_EXECUTOR,
):
@@ -1069,9 +1103,10 @@ class PdfInfo:
self._pages = _pdf_pageinfo_concurrent(
pdf,
executor,
max_workers,
use_threads,
infile,
progbar,
max_workers,
check_pages=check_pages,
detailed_analysis=detailed_analysis,
)
@@ -1146,7 +1181,7 @@ class PdfInfo:
return f"<PdfInfo('...'), page count={len(self)}>"
def main():
def main(): # pragma: no cover
"""Run as a script."""
import argparse # pylint: disable=import-outside-toplevel
from pprint import pprint # pylint: disable=import-outside-toplevel
src/ocrmypdf/pluginspec.py
+38 -45
View File
@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, NamedTuple
import pluggy
from ocrmypdf import Executor, PdfContext
from ocrmypdf._progressbar import ProgressBar
from ocrmypdf.helpers import Resolution
if TYPE_CHECKING:
@@ -102,7 +103,6 @@ def check_options(options: Namespace) -> None:
and the application should terminate gracefully with an informative
message and error code.
Note:
This hook will be called from the main process, and may modify global state
before child worker processes are forked.
@@ -110,7 +110,7 @@ def check_options(options: Namespace) -> None:
@hookspec(firstresult=True)
def get_executor(progressbar_class) -> Executor:
def get_executor(progressbar_class: type[ProgressBar]) -> Executor:
"""Called to obtain an object that manages parallel execution.
This may be used to replace OCRmyPDF's default parallel execution system
@@ -132,41 +132,24 @@ def get_executor(progressbar_class) -> Executor:
This hook will be called from the main process, and may modify global state
before child worker processes are forked.
Note:
This is a :ref:`firstresult hook<firstresult>`.
"""
@hookspec(firstresult=True)
def get_progressbar_class():
def get_progressbar_class() -> type[ProgressBar]:
"""Called to obtain a class that can be used to monitor progress.
A progress bar is assumed, but this could be used for any type of monitoring.
The class should follow a tqdm-like protocol. Calling the class should return
a new progress bar object, which is activated with ``__enter__`` and terminated
``__exit__``. An update method is called whenever the progress bar is updated.
Progress bar objects will not be reused; a new one will be created for each
group of tasks.
The progress bar is held in the main process/thread and not updated by child
process/threads. When a child notifies the parent of completed work, the
parent updates the progress bar.
The arguments are the same as `tqdm <https://github.com/tqdm/tqdm>`_ accepts.
Progress bars should never write to ``sys.stdout``, or they will corrupt the
output if OCRmyPDF writes a PDF to standard output.
The type of events that OCRmyPDF reports to a progress bar may change in
minor releases.
OCRmyPDF will call this function when it wants to display a progress bar.
The class returned by this function must be compatible with the
:class:`ProgressBar` protocol.
Here is how OCRmyPDF will use the progress bar:
Example:
pbar_class = pm.hook.get_progressbar_class()
with pbar_class(**tqdm_kwargs) as pbar:
with pbar_class(**progress_kwargs) as pbar:
...
pbar.update(1)
"""
@@ -187,7 +170,6 @@ def validate(pdfinfo: PdfInfo, options: Namespace) -> None:
and the application should terminate gracefully with an informative
message and error code.
Note:
This hook will be called from the main process, and may modify global state
before child worker processes are forked.
@@ -231,6 +213,7 @@ def rasterize_pdf_page(
Returns:
Path: output_file if successful
Note:
This hook will be called from child processes. Modifying global state
will not affect the main process or other child processes.
@@ -270,7 +253,6 @@ def filter_ocr_image(page: PageContext, image: Image.Image) -> Image.Image:
This hook will be called from child processes. Modifying global state
will not affect the main process or other child processes.
Note:
This is a :ref:`firstresult hook<firstresult>`.
"""
@@ -308,7 +290,6 @@ def filter_page_image(page: PageContext, image_filename: Path) -> Path:
This hook will be called from child processes. Modifying global state
will not affect the main process or other child processes.
Note:
This is a :ref:`firstresult hook<firstresult>`.
"""
@@ -428,21 +409,42 @@ class OcrEngine(ABC):
def generate_hocr(
input_file: Path, output_hocr: Path, output_text: Path, options: Namespace
) -> None:
"""Called to produce a hOCR file and sidecar text file."""
"""Called to produce a hOCR file from a page image and sidecar text file.
A hOCR file is an HTML-like file that describes the position of text on a
page. OCRmyPDF can create a text only PDF from the hOCR file and graft it
onto the output PDF.
This function executes in a worker thread or worker process. OCRmyPDF
automatically parallelizes OCR over pages. The OCR engine should not
introduce more parallelism.
Args:
input_file: A page image on which to perform OCR.
output_hocr: The expected name of the output hOCR file.
output_text: The expected name of a text file containing the
recognized text.
options: The command line options.
"""
@staticmethod
@abstractmethod
def generate_pdf(
input_file: Path, output_pdf: Path, output_text: Path, options: Namespace
) -> None:
"""Called to produce a text only PDF.
"""Called to produce a text only PDF from a page image.
A text only PDF should contain no visible material of any kind, as it
will be grafted onto the input PDF page. It must be sized to the
exact dimensions of the input image.
This function executes in a worker thread or worker process. OCRmyPDF
automatically parallelizes OCR over pages. The OCR engine should not
introduce more parallelism.
Args:
input_file: A page image on which to perform OCR.
output_pdf: The expected name of the output PDF, which must be
a single page PDF with no visible content of any kind, sized
to the dimensions implied by the input_file's width, height
and DPI. The image will be grafted onto the input PDF page.
output_pdf: The expected name of the output PDF.
output_text: The expected name of a text file containing the
recognized text.
options: The command line options.
@@ -469,7 +471,7 @@ def generate_pdfa(
context: PdfContext,
pdf_version: str,
pdfa_part: str,
progressbar_class,
progressbar_class: type[ProgressBar] | None,
stop_on_soft_error: bool,
) -> Path:
"""Generate a PDF/A.
@@ -489,14 +491,8 @@ def generate_pdfa(
At its own discretion, the PDF/A generator may raise the version,
but should not lower it.
pdfa_part: The desired PDF/A compliance level, such as ``'2B'``.
progressbar_class: The class of a progress bar with a tqdm-like API. An
instance of this class will be initialized when PDF/A conversion
begins, using
``instance = progressbar_class(total: int, desc: str, unit:str)``,
defining the number of work units, a user-visible description,
and the name of the work units ("page"). Then ``instance.update()``
will be called when a work unit is completed. If ``None``, no
progress information is reported.
progressbar_class: The class of a progress bar, which must implement
the ProgressBar protocol. If None, no progress is reported.
stop_on_soft_error: If there is an "soft error" such that PDF/A generation
can proceed and produce a valid PDF/A, but output may be invalid or
may not visually resemble the original, the implementer of this hook
@@ -514,9 +510,6 @@ def generate_pdfa(
Before version 15.0.0, the ``context`` was not provided and ``compression``
was provided instead. Plugins should now read the context object to determine
if compression is requested.
See Also:
https://github.com/tqdm/tqdm
"""