menekis/OCRmyPDF
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove lossy JBIG2 support, retain lossless JBIG2 only

Browse Source 
Lossy JBIG2 has been removed due to well-documented risks of character
substitution errors (e.g., 6/8 confusion). The --jbig2-lossy and
--jbig2-page-group-size arguments are now deprecated and ignored with
a warning.

Changes:
- Remove jbig2_lossy and jbig2_page_group_size from OCROptions
- Simplify optimize.py to use single-image JBIG2 encoding only
  (no symbol dictionaries/JBIG2Globals)
- Remove convert_group() from jbig2enc.py
- Deprecate CLI args with warnings for backward compatibility
- Update documentation to explain lossless-only JBIG2
This commit is contained in:
James R. Barlow
2025-12-23 02:45:07 -08:00
parent 9ebba91466
commit 16c2604a07
12 changed files with 99 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ocrmypdf/_exec/jbig2enc.py
-18
View File
@@ -31,24 +31,6 @@ def available():
return True
def convert_group(cwd, infiles, out_prefix, threshold):
args = [
'jbig2',
'-b',
out_prefix,
'--symbol-mode', # symbol mode (lossy)
'-t',
str(threshold), # threshold
# '-r', # refinement mode (lossless symbol mode, currently disabled in
# jbig2)
'--pdf',
]
args.extend(infiles)
proc = run(args, cwd=cwd, stdout=PIPE, stderr=PIPE)
proc.check_returncode()
return proc
def convert_single(cwd, infile, outfile, threshold):
args = ['jbig2', '--pdf', '-t', str(threshold), infile]
with open(outfile, 'wb') as fstdout:
src/ocrmypdf/_options.py
-4
View File
@@ -129,8 +129,6 @@ class OCROptions(BaseModel):
optimize: int = 1
jpg_quality: int | None = None
png_quality: int | None = None
jbig2_lossy: bool | None = None
jbig2_page_group_size: int | None = None
jbig2_threshold: float = 0.85
# Compatibility alias for plugins that expect jpeg_quality
@@ -169,8 +167,6 @@ class OCROptions(BaseModel):
color_conversion_strategy: str = "LeaveColorUnchanged"
# Optimize/JBIG2 options - also accessible via options.optimize.<field>
jbig2_lossy: bool | None = None
jbig2_page_group_size: int | None = None
jbig2_threshold: float = 0.85
# Plugin system
src/ocrmypdf/_validation_coordinator.py
+1 -2
View File
@@ -87,12 +87,11 @@ class ValidationCoordinator:
"""Validate optimization options."""
# Check optimization consistency
if options.optimize == 0 and any([
options.jbig2_lossy,
options.png_quality and options.png_quality > 0,
options.jpeg_quality and options.jpeg_quality > 0
]):
log.warning(
"The arguments --jbig2-lossy, --png-quality, and --jpeg-quality "
"The arguments --png-quality and --jpeg-quality "
"will be ignored because --optimize=0."
)
src/ocrmypdf/api.py
+26 -4
View File
@@ -318,8 +318,8 @@ def ocr( # noqa: D417
optimize: int | None = None,
jpg_quality: int | None = None,
png_quality: int | None = None,
jbig2_lossy: bool | None = None,
jbig2_page_group_size: int | None = None,
jbig2_lossy: bool | None = None, # Deprecated, ignored
jbig2_page_group_size: int | None = None, # Deprecated, ignored
jbig2_threshold: float | None = None,
pages: str | None = None,
max_image_mpixels: float | None = None,
@@ -437,6 +437,17 @@ def ocr( # noqa: D417
if 'verbose' in kwargs:
warn("ocrmypdf.ocr(verbose=) is ignored. Use ocrmypdf.configure_logging().")
# Warn about deprecated jbig2 options and remove from kwargs
if jbig2_lossy:
warn(
"jbig2_lossy is deprecated and will be ignored. "
"Lossy JBIG2 has been removed due to character substitution risks."
)
create_options_kwargs.pop('jbig2_lossy', None)
if jbig2_page_group_size:
warn("jbig2_page_group_size is deprecated and will be ignored.")
create_options_kwargs.pop('jbig2_page_group_size', None)
options = create_options(
input_file=input_file,
output_file=output_file,
@@ -588,8 +599,8 @@ def _hocr_to_ocr_pdf( # noqa: D417
optimize: int | None = None,
jpg_quality: int | None = None,
png_quality: int | None = None,
jbig2_lossy: bool | None = None,
jbig2_page_group_size: int | None = None,
jbig2_lossy: bool | None = None, # Deprecated, ignored
jbig2_page_group_size: int | None = None, # Deprecated, ignored
jbig2_threshold: float | None = None,
pdfa_image_compression: str | None = None,
color_conversion_strategy: str | None = None,
@@ -647,6 +658,17 @@ def _hocr_to_ocr_pdf( # noqa: D417
# Remove None values to let OCROptions use its defaults
options_kwargs = {k: v for k, v in options_kwargs.items() if v is not None}
# Warn about deprecated jbig2 options and remove from kwargs
if jbig2_lossy:
warn(
"jbig2_lossy is deprecated and will be ignored. "
"Lossy JBIG2 has been removed due to character substitution risks."
)
options_kwargs.pop('jbig2_lossy', None)
if jbig2_page_group_size:
warn("jbig2_page_group_size is deprecated and will be ignored.")
options_kwargs.pop('jbig2_page_group_size', None)
# Add work_folder to options_kwargs since it's now a proper field
options_kwargs['work_folder'] = work_folder
src/ocrmypdf/builtin_plugins/optimize.py
+19 -23
View File
@@ -40,17 +40,6 @@ class OptimizeOptions(BaseModel):
png_quality: Annotated[
int, Field(ge=0, le=100, description="PNG quality level for optimization")
] = 0
jbig2_lossy: Annotated[
bool, Field(description="Enable JBIG2 lossy compression")
] = False
jbig2_page_group_size: Annotated[
int,
Field(
ge=0,
le=10000,
description="Number of pages to consider for JBIG2 compression (0=disabled)",
),
] = 0
jbig2_threshold: Annotated[
float,
Field(ge=0.4, le=0.9, description="JBIG2 symbol classification threshold"),
@@ -112,22 +101,18 @@ class OptimizeOptions(BaseModel):
"Values have same meaning as with --jpeg-quality"
),
)
# Deprecated arguments - kept for backward compatibility, emit warnings
optimizing.add_argument(
'--jbig2-lossy',
action='store_true',
help=(
"Enable JBIG2 lossy mode (better compression, not suitable for some "
"use cases - see documentation). Only takes effect if --optimize 1 or "
"higher is also enabled."
),
help=argparse.SUPPRESS, # Deprecated, hidden from help
)
optimizing.add_argument(
'--jbig2-page-group-size',
type=numeric(int, 1, 10000),
default=0,
metavar='N',
# Adjust number of pages to consider at once for JBIG2 compression
help=argparse.SUPPRESS,
help=argparse.SUPPRESS, # Deprecated, hidden from help
)
optimizing.add_argument(
'--jbig2-threshold',
@@ -144,12 +129,11 @@ class OptimizeOptions(BaseModel):
def validate_optimization_consistency(self):
"""Validate optimization options are consistent."""
if self.level == 0 and any([
self.jbig2_lossy,
self.png_quality > 0,
self.png_quality > 0,
self.jpeg_quality > 0
]):
log.warning(
"The arguments --jbig2-lossy, --png-quality, and --jpeg-quality "
"The arguments --png-quality and --jpeg-quality "
"will be ignored because --optimize=0."
)
return self
@@ -186,6 +170,18 @@ def add_options(parser):
@hookimpl
def check_options(options):
"""Check external dependencies for optimization."""
# Warn about deprecated options
if getattr(options, 'jbig2_lossy', False):
log.warning(
"The --jbig2-lossy option is deprecated and will be ignored. "
"Lossy JBIG2 compression has been removed due to risks of "
"character substitution errors."
)
if getattr(options, 'jbig2_page_group_size', 0) not in (0, None):
log.warning(
"The --jbig2-page-group-size option is deprecated and will be ignored."
)
if options.optimize >= 2:
check_external_program(
program='pngquant',
@@ -203,8 +199,8 @@ def check_options(options):
package='jbig2enc',
version_checker=jbig2enc.version,
need_version='0.28',
required_for='--optimize {2,3} | --jbig2-lossy',
recommended=True if not options.jbig2_lossy else False,
required_for='--optimize {2,3}',
recommended=True,
)
src/ocrmypdf/optimize.py
+32 -94
View File
@@ -9,7 +9,6 @@ import logging
import sys
import tempfile
import threading
from collections import defaultdict
from collections.abc import Callable, Iterator, MutableSet, Sequence
from os import fspath
from pathlib import Path
@@ -245,11 +244,9 @@ def extract_image_generic(
not pim.indexed
and pim.colorspace == Name.ICCBased
and pim.bits_per_component == 1
and not options.jbig2_lossy
):
# We can losslessly optimize 1-bit images to CCITT or JBIG2 without
# paying any attention to the ICC profile, provided we're not doing
# lossy JBIG2
# paying any attention to the ICC profile
pim.as_pil_image().save(png_name(root, xref))
return XrefExt(xref, '.png')
@@ -372,120 +369,65 @@ def extract_images_generic(
return jpegs, pngs
def _get_effective_jbig2_page_group_size(options) -> int:
"""Calculate the effective JBIG2 page group size based on options."""
jbig2_page_group_size = options.jbig2_page_group_size
if jbig2_page_group_size is None or jbig2_page_group_size == 0:
return 10 if options.jbig2_lossy else 1
return jbig2_page_group_size
def extract_images_jbig2(pdf: Pdf, root: Path, options) -> dict[int, list[XrefExt]]:
def extract_images_jbig2(pdf: Pdf, root: Path, options) -> list[XrefExt]:
"""Extract any bitonal image that we think we can improve as JBIG2."""
jbig2_page_group_size = _get_effective_jbig2_page_group_size(options)
jbig2_images = []
for _pageno, xref_ext in extract_images(pdf, root, options, extract_image_jbig2):
jbig2_images.append(xref_ext)
jbig2_groups = defaultdict(list)
for pageno, xref_ext in extract_images(pdf, root, options, extract_image_jbig2):
group = pageno // jbig2_page_group_size
jbig2_groups[group].append(xref_ext)
log.debug(f"Optimizable images: JBIG2 groups: {len(jbig2_groups)}")
return jbig2_groups
log.debug(f"Optimizable images: JBIG2: {len(jbig2_images)}")
return jbig2_images
def _produce_jbig2_images(
jbig2_groups: dict[int, list[XrefExt]], root: Path, options, executor: Executor
jbig2_images: list[XrefExt], root: Path, options, executor: Executor
) -> None:
"""Produce JBIG2 images from their groups."""
"""Produce JBIG2 images using lossless single-image encoding."""
def jbig2_group_args(root: Path, groups: dict[int, list[XrefExt]]):
for group, xref_exts in groups.items():
prefix = f'group{group:08d}'
def jbig2_args():
for xref_ext in jbig2_images:
xref, ext = xref_ext
yield (
fspath(root), # =cwd
(img_name(root, xref, ext) for xref, ext in xref_exts), # =infiles
prefix, # =out_prefix
fspath(root),
img_name(root, xref, ext),
root / f'{xref:08d}.jbig2',
options.jbig2_threshold,
)
def jbig2_single_args(root: Path, groups: dict[int, list[XrefExt]]):
for group, xref_exts in groups.items():
prefix = f'group{group:08d}'
# Second loop is to ensure multiple images per page are unpacked
for n, xref_ext in enumerate(xref_exts):
xref, ext = xref_ext
yield (
fspath(root),
img_name(root, xref, ext),
root / f'{prefix}.{n:04d}',
options.jbig2_threshold,
)
effective_group_size = _get_effective_jbig2_page_group_size(options)
if effective_group_size > 1:
jbig2_args = jbig2_group_args
jbig2_convert = jbig2enc.convert_group
else:
jbig2_args = jbig2_single_args
jbig2_convert = jbig2enc.convert_single
executor(
use_threads=True,
max_workers=options.jobs,
progress_kwargs=dict(
total=len(jbig2_groups),
total=len(jbig2_images),
desc="JBIG2",
unit='item',
unit='image',
disable=not options.progress_bar,
),
task=jbig2_convert,
task_arguments=jbig2_args(root, jbig2_groups),
task=jbig2enc.convert_single,
task_arguments=jbig2_args(),
)
def convert_to_jbig2(
pdf: Pdf,
jbig2_groups: dict[int, list[XrefExt]],
jbig2_images: list[XrefExt],
root: Path,
options,
executor: Executor,
) -> None:
"""Convert images to JBIG2 and insert into PDF.
When the JBIG2 page group size is > 1 we do several JBIG2 images at once
and build a symbol dictionary that will span several pages. Each JBIG2
image must reference to its symbol dictionary. If too many pages shared the
same dictionary JBIG2 encoding becomes more expensive and less efficient.
The default value of 10 was determined through testing. Currently this
must be lossy encoding since jbig2enc does not support refinement coding.
When the JBIG2 symbolic coder is not used, each JBIG2 stands on its own
and needs no dictionary. Currently this must be lossless JBIG2.
Each JBIG2 image is encoded independently using lossless compression.
No symbol dictionary (JBIG2Globals) is used.
"""
jbig2_globals_dict: Dictionary | None
_produce_jbig2_images(jbig2_images, root, options, executor)
_produce_jbig2_images(jbig2_groups, root, options, executor)
for group, xref_exts in jbig2_groups.items():
prefix = f'group{group:08d}'
jbig2_symfile = root / (prefix + '.sym')
if jbig2_symfile.exists():
jbig2_globals_data = jbig2_symfile.read_bytes()
jbig2_globals = Stream(pdf, jbig2_globals_data)
jbig2_globals_dict = Dictionary(JBIG2Globals=jbig2_globals)
elif _get_effective_jbig2_page_group_size(options) == 1:
jbig2_globals_dict = None
else:
raise FileNotFoundError(jbig2_symfile)
for n, xref_ext in enumerate(xref_exts):
xref, _ = xref_ext
jbig2_im_file = root / (prefix + f'.{n:04d}')
jbig2_im_data = jbig2_im_file.read_bytes()
im_obj = pdf.get_object(xref, 0)
im_obj.write(
jbig2_im_data, filter=Name.JBIG2Decode, decode_parms=jbig2_globals_dict
)
for xref_ext in jbig2_images:
xref, _ = xref_ext
jbig2_im_file = root / f'{xref:08d}.jbig2'
jbig2_im_data = jbig2_im_file.read_bytes()
im_obj = pdf.get_object(xref, 0)
im_obj.write(jbig2_im_data, filter=Name.JBIG2Decode, decode_parms=None)
def _optimize_jpeg(
@@ -730,8 +672,6 @@ def optimize(
options.jpg_quality = DEFAULT_JPEG_QUALITY if options.optimize < 3 else 40
if options.png_quality == 0:
options.png_quality = DEFAULT_PNG_QUALITY if options.optimize < 3 else 30
if options.jbig2_page_group_size == 0:
options.jbig2_page_group_size = 10 if options.jbig2_lossy else 1
with Pdf.open(input_file) as pdf:
root = output_file.parent / 'images'
@@ -745,8 +685,8 @@ def optimize(
# transcode_pngs(pdf, jpegs, jpg_name, root, options)
transcode_pngs(pdf, pngs, png_name, root, options, executor)
jbig2_groups = extract_images_jbig2(pdf, root, options)
convert_to_jbig2(pdf, jbig2_groups, root, options, executor)
jbig2_images = extract_images_jbig2(pdf, root, options)
convert_to_jbig2(pdf, jbig2_images, root, options, executor)
target_file = output_file.with_suffix('.opt.pdf')
pdf.remove_unreferenced_resources()
@@ -793,15 +733,13 @@ def main(infile, outfile, level, jobs=1):
optimize=int(level),
jpg_quality=0, # Use default
png_quality=0,
jbig2_page_group_size=0,
jbig2_lossy=False,
jbig2_threshold=0.85,
quiet=True,
progress_bar=False,
)
with TemporaryDirectory() as tmpdir:
context = PdfContext(options, tmpdir, infile, None, None)
context = PdfContext(options, Path(tmpdir), infile, None, None)
tmpout = Path(tmpdir) / 'out.pdf'
optimize(
infile,