Compare commits

..
17 changed files with 73 additions and 252 deletions
+4 -18
View File
@@ -17,32 +17,16 @@ FROM base as builder
# Note we need leptonica here to build jbig2
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential autoconf automake libtool \
libleptonica-dev \
zlib1g-dev \
python3-dev \
python3-distutils \
libffi-dev \
ca-certificates \
curl \
git \
libcairo2-dev \
pkg-config
# Get the latest pip (Ubuntu version doesn't support manylinux2010)
RUN \
curl https://bootstrap.pypa.io/get-pip.py | python3
# Compile and install jbig2
# Needs libleptonica-dev, zlib1g-dev
RUN \
mkdir jbig2 \
&& curl -L https://github.com/agl/jbig2enc/archive/ea6a40a.tar.gz | \
tar xz -C jbig2 --strip-components=1 \
&& cd jbig2 \
&& ./autogen.sh && ./configure && make && make install \
&& cd .. \
&& rm -rf jbig2
# Get the latest pip
RUN curl https://bootstrap.pypa.io/get-pip.py | python3
COPY . /app
@@ -56,11 +40,13 @@ FROM base
RUN apt-get update && apt-get install -y --no-install-recommends \
software-properties-common gpg-agent
RUN add-apt-repository -y ppa:alex-p/tesseract-ocr-devel
RUN add-apt-repository -y ppa:alex-p/jbig2enc
RUN apt-get update && apt-get install -y --no-install-recommends \
ghostscript \
fonts-droid-fallback \
jbig2dec \
jbig2 \
img2pdf \
libsm6 libxext6 libxrender-dev \
pngquant \
-26
View File
@@ -255,32 +255,6 @@ jobs:
password: ${{ secrets.TOKEN_PYPI }}
# repository_url: https://test.pypi.org/legacy/
create_release:
name: Create GitHub release
needs: [wheel_sdist_linux, test_linux, test_macos, test_windows]
runs-on: ubuntu-latest
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
permissions:
# Required to create a release
contents: write
steps:
- uses: actions/download-artifact@v3
with:
name: artifact
path: dist
- name: Create Release
id: create-release
uses: shogo82148/actions-create-release@v1
- name: Upload Assets
uses: shogo82148/actions-upload-release-asset@v1
with:
upload_url: ${{ steps.create-release.outputs.upload_url }}
asset_path: |
./dist/*.whl
./dist/*.tar.gz
docker:
name: Build Docker images
needs: [wheel_sdist_linux, test_linux, test_macos, test_windows]
-14
View File
@@ -372,17 +372,3 @@ Some users may consider enabling lossy JBIG2. See: :ref:`jbig2-lossy`.
Image processing and PDF/A conversion can also introduce lossy transformations
to your PDF images, even when ``--optimize 1`` is in use.
Digitally signed PDFs
=====================
OCRmyPDF cannot preserve digital signatures in PDFs and also add to OCR to them.
By default, it will refuse to modify a signed PDF regardless of other settings. You can
override this behavior with ``--invalidate-digital-signatures``; as the name suggests,
any digital signatures will be invalidated.
OCRmyPDF cannot open documents that are encrypted with a digital certificate.
Versions of OCRmyPDF prior to 14.4.0 would invalidate existing digital signatures
without warning.
+8
View File
@@ -24,6 +24,14 @@ On macOS, Homebrew packages jbig2enc and OCRmyPDF includes it by
default. The Docker image for OCRmyPDF also builds its own JBIG2 encoder
from source.
On Ubuntu, you can install the JBIG2 encoder using the following PPA:
.. code-block:: bash
sudo add-apt-repository ppa:alex-p/jbig2enc
sudo apt update
sudo apt install jbig2enc
For all other Linux, you must build a JBIG2 encoder from source:
.. code-block:: bash
-13
View File
@@ -28,19 +28,6 @@ tagged yet.
.. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
v14.4.0
=======
- Digitally signed PDFs are now detected. If the PDF is signed, OCRmyPDF will
refuse to modify it. Previously, only encrypted PDFs were detected, not
those that were signed but not encrypted. :issue:`1040`
- In addition, `--invalidate-digital-signatures` can be used to override the
above behavior and modify the PDF anyway. :issue:`1040`
- tqdm progress bars replaced with "rich" progress bars. The rich library is
a new dependency. Certain APIs that used tqdm are now deprecated and will
be removed in the next major release.
- Improved integration with GitHub Releases. Thanks to @stumpylog.
v14.3.0
=======
-1
View File
@@ -31,7 +31,6 @@ __ocrmypdf_arguments()
--force-ocr (OCR documents that already have printable text)
--skip-text (skip OCR on any pages that already contain text)
--redo-ocr (redo OCR on any pages that seem to have OCR already)
--invalidate-digital-signatures (remove digital signatures from PDF)
--skip-big (skip OCR on pages larger than this many MPixels)
--optimize (select optimization level)
--jpeg-quality (JPEG quality [0..100])
-1
View File
@@ -16,7 +16,6 @@ complete -c ocrmypdf -l remove-vectors -d "don't send vector objects to OCR"
complete -c ocrmypdf -s f -l force-ocr -d "OCR documents that already have printable text"
complete -c ocrmypdf -s s -l skip-ocr -d "skip OCR on pages that text, otherwise try OCR"
complete -c ocrmypdf -l redo-ocr -d "redo OCR on any pages that seem to have OCR already"
complete -c ocrmypdf -l invalidate-digital-signatures -d "invalidate digital signatures and allow OCR to proceed"
complete -c ocrmypdf -s k -l keep-temporary-files -d "keep temporary files (debug)"
+1 -1
View File
@@ -17,6 +17,7 @@ license = {text = "MPL-2.0"}
requires-python = ">=3.8"
dependencies = [
"Pillow>=8.2.0",
"coloredlogs>=14.0",
"deprecation>=2.1.0",
"img2pdf>=0.3.0", # pure Python
"packaging>=20",
@@ -24,7 +25,6 @@ dependencies = [
"pikepdf>=5.0.1",
"pluggy>=0.13.0",
"reportlab>=3.5.66",
"rich>=13",
"tqdm>=4",
"importlib-resources>=5;python_version<'3.9'", # until Python 3.9
"typing-extensions>=4;python_version<'3.10'",
-67
View File
@@ -8,17 +8,6 @@ from __future__ import annotations
import logging
from contextlib import suppress
from rich.console import Console
from rich.logging import RichHandler
from rich.progress import (
BarColumn,
MofNCompleteColumn,
Progress,
TaskProgressColumn,
TextColumn,
TimeRemainingColumn,
)
from rich.table import Column
from tqdm import tqdm
@@ -52,59 +41,3 @@ class TqdmConsole:
def flush(self):
with suppress(AttributeError):
self.file.flush()
class RichLoggingHandler(RichHandler):
def __init__(self, console: Console, **kwargs):
super().__init__(
console=console, show_level=False, show_time=False, markup=True, **kwargs
)
class RichTqdmProgressAdapter:
"""Adapt tqdm API to rich progress bar."""
def __init__(
self,
*,
console: Console,
desc: str,
total: float | None = None,
unit: str | None = None,
unit_scale: float | None = 1.0,
disable: bool = False,
**kwargs,
):
self.progress = Progress(
TextColumn(
"[progress.description]{task.description}",
table_column=Column(min_width=20),
),
BarColumn(),
TaskProgressColumn(),
MofNCompleteColumn(),
TimeRemainingColumn(),
console=console,
auto_refresh=True,
redirect_stderr=True,
redirect_stdout=False,
disable=disable,
**kwargs,
)
self.unit_scale = unit_scale
self.progress_bar = self.progress.add_task(
desc, total=total * self.unit_scale, unit=unit
)
def __enter__(self):
self.progress.start()
return self
def __exit__(self, exc_type, exc_value, traceback):
self.progress.refresh()
self.progress.stop()
return False
def update(self, value=None):
advance = self.unit_scale if value is None else value
self.progress.update(self.progress_bar, advance=advance)
+14 -17
View File
@@ -27,7 +27,6 @@ from ocrmypdf._jobcontext import PageContext, PdfContext
from ocrmypdf._version import PROGRAM_NAME
from ocrmypdf._version import __version__ as VERSION
from ocrmypdf.exceptions import (
DigitalSignatureError,
DpiError,
EncryptedPdfError,
InputFileError,
@@ -81,32 +80,34 @@ def triage_image_file(input_file: Path, output_file: Path, options) -> None:
if im.info['dpi'] <= (96, 96) and not options.image_dpi:
log.info("Image size: (%d, %d)", *im.size)
log.info("Image resolution: (%d, %d)", *im.info['dpi'])
raise DpiError(
log.error(
"Input file is an image, but the resolution (DPI) is "
"not credible. Estimate the resolution at which the "
"image was scanned and specify it using --image-dpi."
)
raise DpiError()
elif not options.image_dpi:
log.info("Image size: (%d, %d)", *im.size)
raise DpiError(
log.error(
"Input file is an image, but has no resolution (DPI) "
"in its metadata. Estimate the resolution at which "
"image was scanned and specify it using --image-dpi."
)
raise DpiError()
if im.mode in ('RGBA', 'LA'):
raise UnsupportedImageFormatError(
log.error(
"The input image has an alpha channel. Remove the alpha "
"channel first."
)
raise UnsupportedImageFormatError()
if 'iccprofile' not in im.info:
if im.mode == 'RGB':
log.info("Input image has no ICC profile, assuming sRGB")
elif im.mode == 'CMYK':
raise UnsupportedImageFormatError(
"Input CMYK image has no ICC profile, not usable"
)
log.error("Input CMYK image has no ICC profile, not usable")
raise UnsupportedImageFormatError()
try:
log.info("Image seems valid. Try converting to PDF...")
@@ -124,6 +125,7 @@ def triage_image_file(input_file: Path, output_file: Path, options) -> None:
)
log.info("Successfully converted to PDF, processing...")
except img2pdf.ImageOpenError as e:
log.error(e)
raise UnsupportedImageFormatError() from e
@@ -193,21 +195,18 @@ def validate_pdfinfo_options(context: PdfContext) -> None:
options = context.options
if pdfinfo.needs_rendering:
raise InputFileError(
log.error(
"This PDF contains dynamic XFA forms created by Adobe LiveCycle "
"Designer and can only be read by Adobe Acrobat or Adobe Reader."
)
if pdfinfo.has_signature:
if options.invalidate_digital_signatures:
log.warning("All digital signatures will be invalidated")
else:
raise DigitalSignatureError()
raise InputFileError()
if pdfinfo.has_acroform:
if options.redo_ocr:
raise InputFileError(
log.error(
"This PDF has a user fillable form. --redo-ocr is not "
"currently possible on such files."
)
raise InputFileError()
else:
log.warning(
"This PDF has a fillable form. "
@@ -854,9 +853,7 @@ def metadata_fixup(working_file: Path, context: PdfContext) -> Path:
with pikepdf.open(context.origin) as original, pikepdf.open(working_file) as pdf:
docinfo = get_docinfo(original, context)
with pdf.open_metadata() as meta_pdf:
meta_pdf.load_from_docinfo(
docinfo, delete_missing=False, raise_failure=False
)
meta_pdf.load_from_docinfo(docinfo, delete_missing=False, raise_failure=False)
# If xmp:CreateDate is missing, set it to the modify date to
# ensure consistency with Ghostscript.
if 'xmp:CreateDate' not in meta_pdf:
+10
View File
@@ -15,6 +15,9 @@ from pathlib import Path
from typing import AnyStr, BinaryIO, Iterable, Union
from warnings import warn
import coloredlogs
from humanfriendly.terminal import enable_ansi_support
from ocrmypdf._logging import PageNumberFilter, TqdmConsole
from ocrmypdf._plugin_manager import get_plugin_manager
from ocrmypdf._sync import run_pipeline
@@ -109,7 +112,14 @@ def configure_logging(
else:
fmt = '%(pageno)s%(message)s'
use_colors = progress_bar_friendly
formatter = None
if use_colors:
use_colors = enable_ansi_support()
if use_colors:
use_colors = coloredlogs.terminal_supports_colors()
if use_colors:
formatter = coloredlogs.ColoredFormatter(fmt=fmt)
if not formatter:
formatter = logging.Formatter(fmt=fmt)
+4 -11
View File
@@ -16,10 +16,10 @@ from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_compl
from contextlib import suppress
from typing import Callable, Iterable, Type, Union
from rich.console import Console as RichConsole
from tqdm import tqdm
from ocrmypdf import Executor, hookimpl
from ocrmypdf._logging import RichLoggingHandler, RichTqdmProgressAdapter
from ocrmypdf._logging import TqdmConsole
from ocrmypdf.exceptions import InputFileError
from ocrmypdf.helpers import remove_all_log_handlers
@@ -168,20 +168,13 @@ def get_executor(progressbar_class):
return StandardExecutor(pbar_class=progressbar_class)
RICH_CONSOLE = RichConsole(stderr=True)
@hookimpl
def get_progressbar_class():
"""Return the default progress bar class."""
def partial_RichTqdmProgressAdapter(*args, **kwargs):
return RichTqdmProgressAdapter(*args, **kwargs, console=RICH_CONSOLE)
return partial_RichTqdmProgressAdapter
return tqdm
@hookimpl
def get_logging_console():
"""Return the default logging console handler."""
return RichLoggingHandler(console=RICH_CONSOLE)
return logging.StreamHandler(stream=TqdmConsole(sys.stderr))
-7
View File
@@ -359,13 +359,6 @@ Online documentation is located at:
help="Skip OCR on pages larger than the specified amount of megapixels, "
"but include skipped pages in final output",
)
ocrsettings.add_argument(
'--invalidate-digital-signatures',
action='store_true',
help="Normally, OCRmyPDF will refuse to OCR a PDF that has a digital "
"signature. This option allows OCR to proceed, but the digital signature "
"will be invalidated.",
)
advanced = parser.add_argument_group(
"Advanced", "Advanced options to control OCRmyPDF"
-12
View File
@@ -108,18 +108,6 @@ class EncryptedPdfError(ExitCodeException):
)
class DigitalSignatureError(ExitCodeException):
"""PDF has a digital signature."""
exit_code = ExitCode.input_file
message = dedent(
"""\
Input PDF has a digital signature. OCR would alter the document,
invalidating the signature.
"""
)
class TesseractConfigError(ExitCodeException):
"""Tesseract config can't be parsed."""
+1 -1
View File
@@ -12,7 +12,7 @@ worker communicates only with the main process.
This is not without drawbacks. If the tasks are not "even" in size, which cannot
be guaranteed, some workers may end up with too much work while others are idle.
It is less efficient than the standard implementation, so not the default.
It is less efficient than the standard implementation, so not th edefault.
"""
from __future__ import annotations
+28 -38
View File
@@ -21,9 +21,7 @@ from typing import Container, Iterable, Iterator, Mapping, NamedTuple, Sequence,
from warnings import warn
from pikepdf import (
Name,
Object,
Page,
Pdf,
PdfImage,
PdfInlineImage,
@@ -493,15 +491,15 @@ def _image_xobjects(container) -> Iterator[tuple[Object, str]]:
since the object does not know its own name.
"""
if Name.Resources not in container:
if '/Resources' not in container:
return
resources = container[Name.Resources]
if Name.XObject not in resources:
resources = container['/Resources']
if '/XObject' not in resources:
return
for key, candidate in resources[Name.XObject].items():
if candidate is None or Name.Subtype not in candidate:
for key, candidate in resources['/XObject'].items():
if candidate is None or '/Subtype' not in candidate:
continue
if candidate[Name.Subtype] == Name.Image:
if candidate['/Subtype'] == '/Image':
pdfimage = candidate
yield (pdfimage, key)
@@ -537,15 +535,15 @@ def _find_form_xobject_images(pdf: Pdf, container: Object, contentsinfo: Content
The container may be a page, or a parent Form XObject.
"""
if Name.Resources not in container:
if '/Resources' not in container:
return
resources = container[Name.Resources]
if Name.XObject not in resources:
resources = container['/Resources']
if '/XObject' not in resources:
return
xobjs = resources[Name.XObject].as_dict()
xobjs = resources['/XObject'].as_dict()
for xobj in xobjs:
candidate = xobjs[xobj]
if candidate is None or candidate[Name.Subtype] != Name.Form:
if candidate is None or candidate['/Subtype'] != '/Form':
continue
form_xobject = candidate
@@ -583,19 +581,16 @@ def _process_content_streams(
downsampling.
"""
if container.get(Name.Type) == Name.Page and Name.Contents in container:
if container.get('/Type') == '/Page' and '/Contents' in container:
initial_shorthand = shorthand or UNIT_SQUARE
elif (
container.get(Name.Type) == Name.XObject
and container[Name.Subtype] == Name.Form
):
elif container.get('/Type') == '/XObject' and container['/Subtype'] == '/Form':
# Set the CTM to the state it was when the "Do" operator was
# encountered that is drawing this instance of the Form XObject
ctm = PdfMatrix(shorthand) if shorthand else PdfMatrix.identity()
# A Form XObject may provide its own matrix to map form space into
# user space. Get this if one exists
form_shorthand = container.get(Name.Matrix, PdfMatrix.identity())
form_shorthand = container.get('/Matrix', PdfMatrix.identity())
form_matrix = PdfMatrix(form_shorthand)
# Concatenate form matrix with CTM to ensure CTM is correct for
@@ -776,7 +771,7 @@ class PageInfo:
check_pages: Container[int],
detailed_analysis: bool,
):
page: Page = pdf.pages[pageno]
page = pdf.pages[pageno]
mediabox = [Decimal(d) for d in page.MediaBox.as_list()]
width_pt = mediabox[2] - mediabox[0]
height_pt = mediabox[3] - mediabox[1]
@@ -784,7 +779,7 @@ class PageInfo:
check_this_page = pageno in check_pages
if check_this_page and detailed_analysis:
pscript5_mode = str(pdf.docinfo.get(Name.Creator)).startswith('PScript5')
pscript5_mode = str(pdf.docinfo.get('/Creator')).startswith('PScript5')
miner = get_page_analysis(infile, pageno, pscript5_mode)
self._textboxes = list(simplify_textboxes(miner, get_text_boxes))
bboxes = (box.bbox for box in self._textboxes)
@@ -794,13 +789,17 @@ class PageInfo:
self._textboxes = []
self._has_text = None # i.e. "no information"
userunit = page.get(Name.UserUnit, Decimal(1.0))
userunit = page.get('/UserUnit', Decimal(1.0))
if not isinstance(userunit, Decimal):
userunit = Decimal(userunit)
self._userunit = userunit
self._width_inches = width_pt * userunit / Decimal(72.0)
self._height_inches = height_pt * userunit / Decimal(72.0)
self._rotate = int(getattr(page.obj, 'Rotate', 0))
try:
self._rotate = int(page['/Rotate'])
except KeyError:
self._rotate = 0
userunit_shorthand = (userunit, 0, 0, userunit, 0, 0)
@@ -954,10 +953,6 @@ DEFAULT_EXECUTOR = SerialExecutor()
class PdfInfo:
"""Get summary information about a PDF."""
_has_acroform: bool = False
_has_signature: bool = False
_needs_rendering: bool = False
def __init__(
self,
infile,
@@ -985,13 +980,13 @@ class PdfInfo:
check_pages=check_pages,
detailed_analysis=detailed_analysis,
)
self._needs_rendering = pdf.Root.get(Name.NeedsRendering, False)
if Name.AcroForm in pdf.Root:
if len(pdf.Root.AcroForm.get(Name.Fields, [])) > 0:
self._needs_rendering = pdf.Root.get('/NeedsRendering', False)
self._has_acroform = False
if '/AcroForm' in pdf.Root:
if len(pdf.Root.AcroForm.get('/Fields', [])) > 0:
self._has_acroform = True
elif Name.XFA in pdf.Root.AcroForm:
elif '/XFA' in pdf.Root.AcroForm:
self._has_acroform = True
self._has_signature = bool(pdf.Root.AcroForm.get(Name.SigFlags, 0) & 1)
@property
def pages(self) -> Sequence[PageInfo | None]:
@@ -1011,14 +1006,9 @@ class PdfInfo:
@property
def has_acroform(self) -> bool:
"""Return True if the document catalog has an AcroForm."""
"""Return True if any page has an AcroForm."""
return self._has_acroform
@property
def has_signature(self) -> bool:
"""Return True if the document annotations has a digital signature."""
return self._has_signature
@property
def filename(self) -> str | Path:
"""Return filename of PDF."""
+3 -25
View File
@@ -5,7 +5,6 @@ from __future__ import annotations
import logging
import pikepdf
import pytest
import ocrmypdf
@@ -20,11 +19,10 @@ def acroform(resources):
return resources / 'acroform.pdf'
def test_acroform_and_redo(acroform, no_outpdf):
with pytest.raises(
ocrmypdf.exceptions.InputFileError, match='--redo-ocr is not currently possible'
):
def test_acroform_and_redo(acroform, caplog, no_outpdf):
with pytest.raises(ocrmypdf.exceptions.InputFileError):
check_ocrmypdf(acroform, no_outpdf, '--redo-ocr')
assert '--redo-ocr is not currently possible' in caplog.text
def test_acroform_message(acroform, caplog, outpdf):
@@ -32,23 +30,3 @@ def test_acroform_message(acroform, caplog, outpdf):
check_ocrmypdf(acroform, outpdf, '--plugin', 'tests/plugins/tesseract_noop.py')
assert 'fillable form' in caplog.text
assert '--force-ocr' in caplog.text
@pytest.fixture
def digitally_signed(acroform, outdir):
out = outdir / 'acroform_signed.pdf'
with pikepdf.open(acroform) as pdf:
pdf.Root.AcroForm.SigFlags = 3
pdf.save(out)
yield out
def test_digital_signature(digitally_signed, no_outpdf):
with pytest.raises(ocrmypdf.exceptions.DigitalSignatureError):
check_ocrmypdf(digitally_signed, no_outpdf)
def test_digital_signature_invalidate(digitally_signed, no_outpdf):
check_ocrmypdf(
digitally_signed, no_outpdf, '--force-ocr', '--invalidate-digital-signatures'
)