diff --git a/docs/jbig2.md b/docs/jbig2.md index 9271339c..5b1d8e09 100644 --- a/docs/jbig2.md +++ b/docs/jbig2.md @@ -43,33 +43,21 @@ be required depending on your system. [sudo] apt install autotools-dev automake libtool libleptonica-dev pkg-config ::: -{#jbig2-lossy} +## JBIG2 Compression -## Lossy mode JBIG2 +OCRmyPDF uses JBIG2 lossless compression for bitonal (black and white) +images. This provides excellent compression ratios compared to the older +CCITT G4 standard, while preserving the exact pixel content of the +original image. -OCRmyPDF provides lossy mode JBIG2 as an advanced and potentially -dangerous feature. Users should [review the technical concerns with -JBIG2 in lossy mode](https://en.wikipedia.org/wiki/JBIG2#Disadvantages) -and decide if this feature is acceptable for their use case. In general, -this mode should not be used for archival purposes, should not be used -when the original document is not available or will be destroyed, and -should not be used when numbers present in the document are important, -because there is a risk of 6/8 and 8/6 substitution errors. +You can adjust the threshold for JBIG2 compression with +`--jbig2-threshold`. The default is 0.85. -JBIG2 lossy mode does achieve higher compression ratios than any other -monochrome (bitonal) compression technology; for large text documents -the savings are considerable. JBIG2 lossless still gives great -compression ratios and is a major improvement over the older CCITT G4 -standard. - -To turn on JBIG2 lossy mode, add the argument `--jbig2-lossy`. -`--optimize {1,2,3}` are necessary for the argument to take effect also -required. Also, a JBIG2 encoder must be installed as described in the -previous section. - -You can adjust the threshold for JBIG2 compression with the -`--jbig2-threshold`. The default is 0.85, meaning that if two symbols -are 85% similar, they will be compressed together. - -*Due to an oversight, ocrmypdf v7.0 and v7.1 used lossy mode by -default.* +:::{note} +Previous versions of OCRmyPDF supported a lossy JBIG2 mode +(`--jbig2-lossy`). This feature has been removed due to the well-known +risk of character substitution errors (e.g., 6/8 confusion). See +[JBIG2 disadvantages](https://en.wikipedia.org/wiki/JBIG2#Disadvantages) +for more information on why lossy JBIG2 is problematic. The `--jbig2-lossy` +and `--jbig2-page-group-size` arguments are now ignored with a warning. +::: diff --git a/docs/optimizer.md b/docs/optimizer.md index 5c7b28cd..6eba6cc8 100644 --- a/docs/optimizer.md +++ b/docs/optimizer.md @@ -28,9 +28,6 @@ header-rows: 1 - Enables lossless optimizations, such as transcoding images to more efficient formats. Also compress other uncompressed objects in the PDF and enables the more efficient "object streams" within the PDF. - (If ``--jbig2-lossy`` is issued, then lossy JBIG2 optimization is used. - The decision to use lossy JBIG2 is separate from standard optimization - settings.) * - ``--optimize 2`` - ``-O2`` - All of the above, and enables lossy optimizations and color quantization. @@ -105,7 +102,3 @@ quality image may be suitable for storage after OCR. It is not possible to optimize all image types. Uncommon image types may be skipped by the optimizer. - -OCRmyPDF provides `lossy mode JBIG2 `{.interpreted-text -role="ref"} as an advanced feature that additional requires the argument -`--jbig2-lossy`. diff --git a/misc/_webservice.py b/misc/_webservice.py index 8ddcacd5..6016080e 100644 --- a/misc/_webservice.py +++ b/misc/_webservice.py @@ -96,8 +96,7 @@ with st.expander("Optimization after OCR"): png_quality = st.slider( "PNG quality", min_value=0, max_value=100, value=75, key="png_quality" ) - jbig2_lossy = st.checkbox("JBIG2 lossy (dangerous)", value=False, key="jbig2_lossy") - jbig2_threshold = st.number_input("JBIG2 threshold", value=0, key="jbig2_threshold") + jbig2_threshold = st.number_input("JBIG2 threshold", value=0.85, key="jbig2_threshold") with st.expander("Advanced options"): jobs = st.slider( @@ -189,8 +188,6 @@ if uploaded: args.append(f"--jpeg-quality={jpeg_quality}") if optimize > '0' and png_quality: args.append(f"--png-quality={png_quality}") - if jbig2_lossy: - args.append("--jbig2-lossy") if jbig2_threshold: args.append(f"--jbig2-threshold={jbig2_threshold}") if jobs: diff --git a/src/ocrmypdf/_exec/jbig2enc.py b/src/ocrmypdf/_exec/jbig2enc.py index 1c6dd5fe..736de67e 100644 --- a/src/ocrmypdf/_exec/jbig2enc.py +++ b/src/ocrmypdf/_exec/jbig2enc.py @@ -31,24 +31,6 @@ def available(): return True -def convert_group(cwd, infiles, out_prefix, threshold): - args = [ - 'jbig2', - '-b', - out_prefix, - '--symbol-mode', # symbol mode (lossy) - '-t', - str(threshold), # threshold - # '-r', # refinement mode (lossless symbol mode, currently disabled in - # jbig2) - '--pdf', - ] - args.extend(infiles) - proc = run(args, cwd=cwd, stdout=PIPE, stderr=PIPE) - proc.check_returncode() - return proc - - def convert_single(cwd, infile, outfile, threshold): args = ['jbig2', '--pdf', '-t', str(threshold), infile] with open(outfile, 'wb') as fstdout: diff --git a/src/ocrmypdf/_options.py b/src/ocrmypdf/_options.py index b1b1152f..f126486f 100644 --- a/src/ocrmypdf/_options.py +++ b/src/ocrmypdf/_options.py @@ -129,8 +129,6 @@ class OCROptions(BaseModel): optimize: int = 1 jpg_quality: int | None = None png_quality: int | None = None - jbig2_lossy: bool | None = None - jbig2_page_group_size: int | None = None jbig2_threshold: float = 0.85 # Compatibility alias for plugins that expect jpeg_quality @@ -169,8 +167,6 @@ class OCROptions(BaseModel): color_conversion_strategy: str = "LeaveColorUnchanged" # Optimize/JBIG2 options - also accessible via options.optimize. - jbig2_lossy: bool | None = None - jbig2_page_group_size: int | None = None jbig2_threshold: float = 0.85 # Plugin system diff --git a/src/ocrmypdf/_validation_coordinator.py b/src/ocrmypdf/_validation_coordinator.py index fd2a5a33..538072b3 100644 --- a/src/ocrmypdf/_validation_coordinator.py +++ b/src/ocrmypdf/_validation_coordinator.py @@ -87,12 +87,11 @@ class ValidationCoordinator: """Validate optimization options.""" # Check optimization consistency if options.optimize == 0 and any([ - options.jbig2_lossy, options.png_quality and options.png_quality > 0, options.jpeg_quality and options.jpeg_quality > 0 ]): log.warning( - "The arguments --jbig2-lossy, --png-quality, and --jpeg-quality " + "The arguments --png-quality and --jpeg-quality " "will be ignored because --optimize=0." ) diff --git a/src/ocrmypdf/api.py b/src/ocrmypdf/api.py index a1622edf..0fb9aa6b 100644 --- a/src/ocrmypdf/api.py +++ b/src/ocrmypdf/api.py @@ -318,8 +318,8 @@ def ocr( # noqa: D417 optimize: int | None = None, jpg_quality: int | None = None, png_quality: int | None = None, - jbig2_lossy: bool | None = None, - jbig2_page_group_size: int | None = None, + jbig2_lossy: bool | None = None, # Deprecated, ignored + jbig2_page_group_size: int | None = None, # Deprecated, ignored jbig2_threshold: float | None = None, pages: str | None = None, max_image_mpixels: float | None = None, @@ -437,6 +437,17 @@ def ocr( # noqa: D417 if 'verbose' in kwargs: warn("ocrmypdf.ocr(verbose=) is ignored. Use ocrmypdf.configure_logging().") + # Warn about deprecated jbig2 options and remove from kwargs + if jbig2_lossy: + warn( + "jbig2_lossy is deprecated and will be ignored. " + "Lossy JBIG2 has been removed due to character substitution risks." + ) + create_options_kwargs.pop('jbig2_lossy', None) + if jbig2_page_group_size: + warn("jbig2_page_group_size is deprecated and will be ignored.") + create_options_kwargs.pop('jbig2_page_group_size', None) + options = create_options( input_file=input_file, output_file=output_file, @@ -588,8 +599,8 @@ def _hocr_to_ocr_pdf( # noqa: D417 optimize: int | None = None, jpg_quality: int | None = None, png_quality: int | None = None, - jbig2_lossy: bool | None = None, - jbig2_page_group_size: int | None = None, + jbig2_lossy: bool | None = None, # Deprecated, ignored + jbig2_page_group_size: int | None = None, # Deprecated, ignored jbig2_threshold: float | None = None, pdfa_image_compression: str | None = None, color_conversion_strategy: str | None = None, @@ -647,6 +658,17 @@ def _hocr_to_ocr_pdf( # noqa: D417 # Remove None values to let OCROptions use its defaults options_kwargs = {k: v for k, v in options_kwargs.items() if v is not None} + # Warn about deprecated jbig2 options and remove from kwargs + if jbig2_lossy: + warn( + "jbig2_lossy is deprecated and will be ignored. " + "Lossy JBIG2 has been removed due to character substitution risks." + ) + options_kwargs.pop('jbig2_lossy', None) + if jbig2_page_group_size: + warn("jbig2_page_group_size is deprecated and will be ignored.") + options_kwargs.pop('jbig2_page_group_size', None) + # Add work_folder to options_kwargs since it's now a proper field options_kwargs['work_folder'] = work_folder diff --git a/src/ocrmypdf/builtin_plugins/optimize.py b/src/ocrmypdf/builtin_plugins/optimize.py index 707ca909..4729ab33 100644 --- a/src/ocrmypdf/builtin_plugins/optimize.py +++ b/src/ocrmypdf/builtin_plugins/optimize.py @@ -40,17 +40,6 @@ class OptimizeOptions(BaseModel): png_quality: Annotated[ int, Field(ge=0, le=100, description="PNG quality level for optimization") ] = 0 - jbig2_lossy: Annotated[ - bool, Field(description="Enable JBIG2 lossy compression") - ] = False - jbig2_page_group_size: Annotated[ - int, - Field( - ge=0, - le=10000, - description="Number of pages to consider for JBIG2 compression (0=disabled)", - ), - ] = 0 jbig2_threshold: Annotated[ float, Field(ge=0.4, le=0.9, description="JBIG2 symbol classification threshold"), @@ -112,22 +101,18 @@ class OptimizeOptions(BaseModel): "Values have same meaning as with --jpeg-quality" ), ) + # Deprecated arguments - kept for backward compatibility, emit warnings optimizing.add_argument( '--jbig2-lossy', action='store_true', - help=( - "Enable JBIG2 lossy mode (better compression, not suitable for some " - "use cases - see documentation). Only takes effect if --optimize 1 or " - "higher is also enabled." - ), + help=argparse.SUPPRESS, # Deprecated, hidden from help ) optimizing.add_argument( '--jbig2-page-group-size', type=numeric(int, 1, 10000), default=0, metavar='N', - # Adjust number of pages to consider at once for JBIG2 compression - help=argparse.SUPPRESS, + help=argparse.SUPPRESS, # Deprecated, hidden from help ) optimizing.add_argument( '--jbig2-threshold', @@ -144,12 +129,11 @@ class OptimizeOptions(BaseModel): def validate_optimization_consistency(self): """Validate optimization options are consistent.""" if self.level == 0 and any([ - self.jbig2_lossy, - self.png_quality > 0, + self.png_quality > 0, self.jpeg_quality > 0 ]): log.warning( - "The arguments --jbig2-lossy, --png-quality, and --jpeg-quality " + "The arguments --png-quality and --jpeg-quality " "will be ignored because --optimize=0." ) return self @@ -186,6 +170,18 @@ def add_options(parser): @hookimpl def check_options(options): """Check external dependencies for optimization.""" + # Warn about deprecated options + if getattr(options, 'jbig2_lossy', False): + log.warning( + "The --jbig2-lossy option is deprecated and will be ignored. " + "Lossy JBIG2 compression has been removed due to risks of " + "character substitution errors." + ) + if getattr(options, 'jbig2_page_group_size', 0) not in (0, None): + log.warning( + "The --jbig2-page-group-size option is deprecated and will be ignored." + ) + if options.optimize >= 2: check_external_program( program='pngquant', @@ -203,8 +199,8 @@ def check_options(options): package='jbig2enc', version_checker=jbig2enc.version, need_version='0.28', - required_for='--optimize {2,3} | --jbig2-lossy', - recommended=True if not options.jbig2_lossy else False, + required_for='--optimize {2,3}', + recommended=True, ) diff --git a/src/ocrmypdf/optimize.py b/src/ocrmypdf/optimize.py index 99cbc55f..93f0c105 100644 --- a/src/ocrmypdf/optimize.py +++ b/src/ocrmypdf/optimize.py @@ -9,7 +9,6 @@ import logging import sys import tempfile import threading -from collections import defaultdict from collections.abc import Callable, Iterator, MutableSet, Sequence from os import fspath from pathlib import Path @@ -245,11 +244,9 @@ def extract_image_generic( not pim.indexed and pim.colorspace == Name.ICCBased and pim.bits_per_component == 1 - and not options.jbig2_lossy ): # We can losslessly optimize 1-bit images to CCITT or JBIG2 without - # paying any attention to the ICC profile, provided we're not doing - # lossy JBIG2 + # paying any attention to the ICC profile pim.as_pil_image().save(png_name(root, xref)) return XrefExt(xref, '.png') @@ -372,120 +369,65 @@ def extract_images_generic( return jpegs, pngs -def _get_effective_jbig2_page_group_size(options) -> int: - """Calculate the effective JBIG2 page group size based on options.""" - jbig2_page_group_size = options.jbig2_page_group_size - if jbig2_page_group_size is None or jbig2_page_group_size == 0: - return 10 if options.jbig2_lossy else 1 - return jbig2_page_group_size - - -def extract_images_jbig2(pdf: Pdf, root: Path, options) -> dict[int, list[XrefExt]]: +def extract_images_jbig2(pdf: Pdf, root: Path, options) -> list[XrefExt]: """Extract any bitonal image that we think we can improve as JBIG2.""" - jbig2_page_group_size = _get_effective_jbig2_page_group_size(options) + jbig2_images = [] + for _pageno, xref_ext in extract_images(pdf, root, options, extract_image_jbig2): + jbig2_images.append(xref_ext) - jbig2_groups = defaultdict(list) - for pageno, xref_ext in extract_images(pdf, root, options, extract_image_jbig2): - group = pageno // jbig2_page_group_size - jbig2_groups[group].append(xref_ext) - - log.debug(f"Optimizable images: JBIG2 groups: {len(jbig2_groups)}") - return jbig2_groups + log.debug(f"Optimizable images: JBIG2: {len(jbig2_images)}") + return jbig2_images def _produce_jbig2_images( - jbig2_groups: dict[int, list[XrefExt]], root: Path, options, executor: Executor + jbig2_images: list[XrefExt], root: Path, options, executor: Executor ) -> None: - """Produce JBIG2 images from their groups.""" + """Produce JBIG2 images using lossless single-image encoding.""" - def jbig2_group_args(root: Path, groups: dict[int, list[XrefExt]]): - for group, xref_exts in groups.items(): - prefix = f'group{group:08d}' + def jbig2_args(): + for xref_ext in jbig2_images: + xref, ext = xref_ext yield ( - fspath(root), # =cwd - (img_name(root, xref, ext) for xref, ext in xref_exts), # =infiles - prefix, # =out_prefix + fspath(root), + img_name(root, xref, ext), + root / f'{xref:08d}.jbig2', options.jbig2_threshold, ) - def jbig2_single_args(root: Path, groups: dict[int, list[XrefExt]]): - for group, xref_exts in groups.items(): - prefix = f'group{group:08d}' - # Second loop is to ensure multiple images per page are unpacked - for n, xref_ext in enumerate(xref_exts): - xref, ext = xref_ext - yield ( - fspath(root), - img_name(root, xref, ext), - root / f'{prefix}.{n:04d}', - options.jbig2_threshold, - ) - - effective_group_size = _get_effective_jbig2_page_group_size(options) - if effective_group_size > 1: - jbig2_args = jbig2_group_args - jbig2_convert = jbig2enc.convert_group - else: - jbig2_args = jbig2_single_args - jbig2_convert = jbig2enc.convert_single - executor( use_threads=True, max_workers=options.jobs, progress_kwargs=dict( - total=len(jbig2_groups), + total=len(jbig2_images), desc="JBIG2", - unit='item', + unit='image', disable=not options.progress_bar, ), - task=jbig2_convert, - task_arguments=jbig2_args(root, jbig2_groups), + task=jbig2enc.convert_single, + task_arguments=jbig2_args(), ) def convert_to_jbig2( pdf: Pdf, - jbig2_groups: dict[int, list[XrefExt]], + jbig2_images: list[XrefExt], root: Path, options, executor: Executor, ) -> None: """Convert images to JBIG2 and insert into PDF. - When the JBIG2 page group size is > 1 we do several JBIG2 images at once - and build a symbol dictionary that will span several pages. Each JBIG2 - image must reference to its symbol dictionary. If too many pages shared the - same dictionary JBIG2 encoding becomes more expensive and less efficient. - The default value of 10 was determined through testing. Currently this - must be lossy encoding since jbig2enc does not support refinement coding. - - When the JBIG2 symbolic coder is not used, each JBIG2 stands on its own - and needs no dictionary. Currently this must be lossless JBIG2. + Each JBIG2 image is encoded independently using lossless compression. + No symbol dictionary (JBIG2Globals) is used. """ - jbig2_globals_dict: Dictionary | None + _produce_jbig2_images(jbig2_images, root, options, executor) - _produce_jbig2_images(jbig2_groups, root, options, executor) - - for group, xref_exts in jbig2_groups.items(): - prefix = f'group{group:08d}' - jbig2_symfile = root / (prefix + '.sym') - if jbig2_symfile.exists(): - jbig2_globals_data = jbig2_symfile.read_bytes() - jbig2_globals = Stream(pdf, jbig2_globals_data) - jbig2_globals_dict = Dictionary(JBIG2Globals=jbig2_globals) - elif _get_effective_jbig2_page_group_size(options) == 1: - jbig2_globals_dict = None - else: - raise FileNotFoundError(jbig2_symfile) - - for n, xref_ext in enumerate(xref_exts): - xref, _ = xref_ext - jbig2_im_file = root / (prefix + f'.{n:04d}') - jbig2_im_data = jbig2_im_file.read_bytes() - im_obj = pdf.get_object(xref, 0) - im_obj.write( - jbig2_im_data, filter=Name.JBIG2Decode, decode_parms=jbig2_globals_dict - ) + for xref_ext in jbig2_images: + xref, _ = xref_ext + jbig2_im_file = root / f'{xref:08d}.jbig2' + jbig2_im_data = jbig2_im_file.read_bytes() + im_obj = pdf.get_object(xref, 0) + im_obj.write(jbig2_im_data, filter=Name.JBIG2Decode, decode_parms=None) def _optimize_jpeg( @@ -730,8 +672,6 @@ def optimize( options.jpg_quality = DEFAULT_JPEG_QUALITY if options.optimize < 3 else 40 if options.png_quality == 0: options.png_quality = DEFAULT_PNG_QUALITY if options.optimize < 3 else 30 - if options.jbig2_page_group_size == 0: - options.jbig2_page_group_size = 10 if options.jbig2_lossy else 1 with Pdf.open(input_file) as pdf: root = output_file.parent / 'images' @@ -745,8 +685,8 @@ def optimize( # transcode_pngs(pdf, jpegs, jpg_name, root, options) transcode_pngs(pdf, pngs, png_name, root, options, executor) - jbig2_groups = extract_images_jbig2(pdf, root, options) - convert_to_jbig2(pdf, jbig2_groups, root, options, executor) + jbig2_images = extract_images_jbig2(pdf, root, options) + convert_to_jbig2(pdf, jbig2_images, root, options, executor) target_file = output_file.with_suffix('.opt.pdf') pdf.remove_unreferenced_resources() @@ -793,15 +733,13 @@ def main(infile, outfile, level, jobs=1): optimize=int(level), jpg_quality=0, # Use default png_quality=0, - jbig2_page_group_size=0, - jbig2_lossy=False, jbig2_threshold=0.85, quiet=True, progress_bar=False, ) with TemporaryDirectory() as tmpdir: - context = PdfContext(options, tmpdir, infile, None, None) + context = PdfContext(options, Path(tmpdir), infile, None, None) tmpout = Path(tmpdir) / 'out.pdf' optimize( infile, diff --git a/tests/test_api.py b/tests/test_api.py index 234244bf..ad7b0936 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -120,14 +120,12 @@ def test_nested_plugin_option_access(): tesseract_timeout=120.0, tesseract_oem=1, optimize=2, - jbig2_lossy=True, ) # Test flat access still works assert options.tesseract_timeout == 120.0 assert options.tesseract_oem == 1 assert options.optimize == 2 - assert options.jbig2_lossy is True # Test nested access for tesseract tesseract = options.tesseract diff --git a/tests/test_optimize.py b/tests/test_optimize.py index afdf4a0e..603dbc19 100644 --- a/tests/test_optimize.py +++ b/tests/test_optimize.py @@ -81,8 +81,8 @@ def test_jpg_png_params(resources, outpdf): @needs_jbig2enc -@pytest.mark.parametrize('lossy', [False, True]) -def test_jbig2_lossy(lossy, resources, outpdf): +def test_jbig2_lossless(resources, outpdf): + """Test that JBIG2 lossless encoding works without JBIG2Globals.""" args = [ resources / 'ccitt.pdf', outpdf, @@ -99,19 +99,14 @@ def test_jbig2_lossy(lossy, resources, outpdf): '--jbig2-threshold', '0.7', ] - if lossy: - args.append('--jbig2-lossy') check_ocrmypdf(*args) with pikepdf.open(outpdf) as pdf: pim = pikepdf.PdfImage(next(iter(pdf.pages[0].images.values()))) assert pim.filters[0] == '/JBIG2Decode' - - if lossy: - assert '/JBIG2Globals' in pim.decode_parms[0] - else: - assert len(pim.decode_parms) == 0 + # Lossless JBIG2 has no JBIG2Globals (no shared symbol dictionary) + assert len(pim.decode_parms) == 0 @needs_pngquant diff --git a/tests/test_validation.py b/tests/test_validation.py index 7154f6e2..319f598d 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -89,7 +89,7 @@ def test_mutex_options(): def test_optimizing(caplog): vd.check_options( - *make_opts_pm(optimize=0, jbig2_lossy=True, png_quality=18, jpeg_quality=10) + *make_opts_pm(optimize=0, png_quality=18, jpeg_quality=10) ) assert 'will be ignored because' in caplog.text