Delinting

This commit is contained in:
James R. Barlow
2020-05-03 00:53:29 -07:00
parent 5dbc080fa0
commit c85278b31d
18 changed files with 79 additions and 120 deletions
+1 -1
View File
@@ -56,7 +56,7 @@ def run(args=None):
configure_logging(
verbosity, progress_bar_friendly=options.progress_bar, manage_root_logger=True
)
log.debug('ocrmypdf ' + __version__)
log.debug('ocrmypdf %s', __version__)
try:
check_options(options)
except ValueError as e:
-1
View File
@@ -15,7 +15,6 @@
# You should have received a copy of the GNU General Public License
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
import logging
import os
import shutil
import sys
+18 -24
View File
@@ -20,30 +20,29 @@ import os
import re
import sys
from datetime import datetime, timezone
from pathlib import Path
from shutil import copyfileobj
import img2pdf
import pikepdf
from pikepdf.models.metadata import encode_pdf_date
from PIL import Image
from PIL import Image, ImageColor, ImageDraw
from . import leptonica
from ._version import PROGRAM_NAME
from ._version import __version__ as VERSION
from .exceptions import (
from ocrmypdf import leptonica
from ocrmypdf._version import PROGRAM_NAME
from ocrmypdf._version import __version__ as VERSION
from ocrmypdf.exceptions import (
DpiError,
EncryptedPdfError,
InputFileError,
PriorOcrFoundError,
UnsupportedImageFormatError,
)
from .exec import ghostscript, tesseract
from .helpers import Resolution, safe_symlink
from .hocrtransform import HocrTransform
from .optimize import optimize
from .pdfa import generate_pdfa_ps
from .pdfinfo import Colorspace, Encoding, PdfInfo
from ocrmypdf.exec import ghostscript, tesseract, unpaper
from ocrmypdf.helpers import Resolution, safe_symlink
from ocrmypdf.hocrtransform import HocrTransform
from ocrmypdf.optimize import optimize
from ocrmypdf.pdfa import generate_pdfa_ps
from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
log = logging.getLogger(__name__)
@@ -63,8 +62,8 @@ def triage_image_file(input_file, output_file, options):
log.info("Input file is an image")
if 'dpi' in im.info:
if im.info['dpi'] <= (96, 96) and not options.image_dpi:
log.info("Image size: (%d, %d)" % im.size)
log.info("Image resolution: (%d, %d)" % im.info['dpi'])
log.info("Image size: (%d, %d)", *im.size)
log.info("Image resolution: (%d, %d)", *im.info['dpi'])
log.error(
"Input file is an image, but the resolution (DPI) is "
"not credible. Estimate the resolution at which the "
@@ -72,7 +71,7 @@ def triage_image_file(input_file, output_file, options):
)
raise DpiError()
elif not options.image_dpi:
log.info("Image size: (%d, %d)" % im.size)
log.info("Image size: (%d, %d)", *im.size)
log.error(
"Input file is an image, but has no resolution (DPI) "
"in its metadata. Estimate the resolution at which "
@@ -261,7 +260,7 @@ def is_ocr_required(page_context):
ocr_required = True
elif options.redo_ocr:
if pageinfo.has_corrupt_text:
log.warn(
log.warning(
"some text on this page cannot be mapped to characters: "
"consider using --force-ocr instead"
)
@@ -288,7 +287,7 @@ def is_ocr_required(page_context):
)
elif options.force_ocr:
# Warn the user they might not want to do this
log.warn(
log.warning(
"page has no images - "
"all vector content will be "
f"rasterized at {VECTOR_PAGE_DPI} DPI, losing some resolution and likely "
@@ -308,7 +307,7 @@ def is_ocr_required(page_context):
pixel_count = pageinfo.width_pixels * pageinfo.height_pixels
if pixel_count > (options.skip_big * 1_000_000):
ocr_required = False
log.warn(
log.warning(
"page too big, skipping OCR "
f"({(pixel_count / 1_000_000):.1f} MPixels > {options.skip_big:.1f} MPixels --skip-big)"
)
@@ -464,8 +463,6 @@ def preprocess_deskew(input_file, page_context):
def preprocess_clean(input_file, page_context):
from .exec import unpaper
output_file = page_context.get_path('pp_clean.png')
dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
unpaper.clean(input_file, output_file, dpi.x, page_context.options.unpaper_args)
@@ -480,9 +477,6 @@ def create_ocr_image(image, page_context):
output_file = page_context.get_path('ocr.png')
options = page_context.options
with Image.open(image) as im:
from PIL import ImageColor
from PIL import ImageDraw
white = ImageColor.getcolor('#ffffff', im.mode)
# pink = ImageColor.getcolor('#ff0080', im.mode)
draw = ImageDraw.ImageDraw(im)
@@ -811,7 +805,7 @@ def merge_sidecars(txt_files, context):
return output_file
def copy_final(input_file, output_file, context):
def copy_final(input_file, output_file, _context):
log.debug('%s -> %s', input_file, output_file)
with open(input_file, 'rb') as input_stream:
if output_file == '-':
+1 -5
View File
@@ -15,7 +15,6 @@
# You should have received a copy of the GNU General Public License
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
import importlib
import logging
import logging.handlers
import os
@@ -27,13 +26,10 @@ from pathlib import Path
from tempfile import mkdtemp
import PIL
import pluggy
from ocrmypdf import pluginspec
from ocrmypdf._concurrent import exec_progress_pool
from ocrmypdf._graft import OcrGrafter
from ocrmypdf._jobcontext import PdfContext, cleanup_working_files
from ocrmypdf._logging import PageNumberFilter
from ocrmypdf._pipeline import (
convert_to_pdfa,
copy_final,
@@ -372,7 +368,7 @@ def run_pipeline(options, *, plugin_manager, api=False):
else:
log.error(type(e).__name__)
return e.exit_code
except (Exception if not api else NeverRaise) as e:
except (Exception if not api else NeverRaise) as e: # pylint: disable=broad-except
log.exception("An exception occurred while executing the pipeline")
return ExitCode.other_error
finally:
+1 -2
View File
@@ -21,6 +21,7 @@ import locale
import logging
import os
import sys
import unicodedata
from pathlib import Path
from shutil import copyfileobj
@@ -284,8 +285,6 @@ def check_options_advanced(options):
def check_options_metadata(options):
import unicodedata
docinfo = [options.title, options.author, options.keywords, options.subject]
for s in (m for m in docinfo if m):
for c in s:
+1 -2
View File
@@ -19,7 +19,6 @@ import logging
import os
import sys
from argparse import ArgumentParser
from contextlib import suppress
from enum import IntEnum
from pathlib import Path
from typing import Dict, Iterable
@@ -28,7 +27,7 @@ from ocrmypdf._logging import PageNumberFilter, TqdmConsole
from ocrmypdf._plugin_manager import get_plugin_manager
from ocrmypdf._sync import run_pipeline
from ocrmypdf._validation import check_options
from ocrmypdf.cli import get_parser, plugins_only_parser
from ocrmypdf.cli import get_parser
try:
import coloredlogs
+10 -11
View File
@@ -25,13 +25,15 @@ from contextlib import suppress
from os import fspath
from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired
from ..exceptions import (
from PIL import Image
from ocrmypdf.exceptions import (
MissingDependencyError,
SubprocessOutputError,
TesseractConfigError,
)
from ..helpers import page_number, safe_symlink
from . import get_version, run
from ocrmypdf.exec import get_version, run
from ocrmypdf.helpers import safe_symlink
log = logging.getLogger(__name__)
@@ -133,7 +135,7 @@ def languages(tesseract_env=None):
for line in output.splitlines():
if line.startswith('Error'):
raise MissingDependencyError(lang_error(output))
header, *rest = output.splitlines()
_header, *rest = output.splitlines()
return set(lang.strip() for lang in rest)
@@ -227,18 +229,15 @@ def tesseract_log_output(stdout, input_file):
tlog.info(line.strip())
def page_timedout(input_file, timeout):
def page_timedout(timeout):
if timeout == 0:
return
prefix = f"{(page_number(input_file)):4d}: [tesseract] "
log.warning(prefix + " took too long to OCR - skipping")
log.warning("[tesseract] took too long to OCR - skipping")
def _generate_null_hocr(output_hocr, output_sidecar, image):
"""Produce a .hocr file that reports no text detected on a page that is
the same size as the input image."""
from PIL import Image
with Image.open(image) as im:
w, h = im.size
@@ -293,7 +292,7 @@ def generate_hocr(
# Generate a HOCR file with no recognized text if tesseract times out
# Temporary workaround to hocrTransform not being able to function if
# it does not have a valid hOCR file.
page_timedout(input_file, timeout)
page_timedout(timeout)
_generate_null_hocr(output_hocr, output_sidecar, input_file)
except CalledProcessError as e:
tesseract_log_output(e.output, input_file)
@@ -389,7 +388,7 @@ def generate_pdf(
if os.path.exists(prefix + '.txt'):
shutil.move(prefix + '.txt', output_text)
except TimeoutExpired:
page_timedout(input_image, timeout)
page_timedout(timeout)
use_skip_page(text_only, skip_pdf, output_pdf, output_text)
except CalledProcessError as e:
tesseract_log_output(e.output, input_image)
+1 -1
View File
@@ -25,7 +25,7 @@ from collections.abc import Iterable
from contextlib import suppress
from functools import wraps
from io import StringIO
from math import inf, isclose
from math import isclose
from pathlib import Path
import pikepdf
+4 -14
View File
@@ -29,7 +29,7 @@ from collections.abc import Sequence
from contextlib import suppress
from ctypes.util import find_library
from functools import lru_cache
from io import BytesIO
from io import BytesIO, UnsupportedOperation
from os import fspath
from tempfile import TemporaryFile
@@ -96,7 +96,6 @@ class _LeptonicaErrorTrap:
self.no_stderr = False
def __enter__(self):
from io import UnsupportedOperation
self.tmpfile = TemporaryFile()
@@ -351,7 +350,7 @@ class Pix(LeptonicaObject):
py_file.write(buffer)
@classmethod
def frompil(self, pillow_image):
def frompil(cls, pillow_image):
"""Create a copy of a PIL.Image from this Pix"""
bio = BytesIO()
pillow_image.save(bio, format='png', compress_level=1)
@@ -363,7 +362,7 @@ class Pix(LeptonicaObject):
def topil(self):
"""Returns a PIL.Image version of this Pix"""
from PIL import Image
from PIL import Image # pylint: disable=import-outside-toplevel
# Leptonica manages data in words, so it implicitly does an endian
# swap. Tell Pillow about this when it reads the data.
@@ -534,16 +533,7 @@ class Pix(LeptonicaObject):
)
return Pix(thresh_pix)
def crop_to_foreground(
self,
threshold=128,
mindist=70,
erasedist=30,
pagenum=0,
showmorph=0,
display=0,
pdfdir=ffi.NULL,
):
def crop_to_foreground(self, threshold=128, mindist=70, erasedist=30, showmorph=0):
if get_leptonica_version() < 'leptonica-1.76':
# Leptonica 1.76 changed the API for pixFindPageForeground; we don't
# support the old version
+7 -7
View File
@@ -217,7 +217,7 @@ def extract_images(pike, root, options, extract_fn):
result = extract_fn(
pike=pike, root=root, image=image, xref=xref, options=options
)
except Exception as e:
except Exception as e: # pylint: disable=broad-except
log.debug("Image xref %s, error %s", xref, repr(e))
errors += 1
else:
@@ -422,12 +422,12 @@ def transcode_pngs(pike, images, image_name_fn, root, options):
)
continue
if compdata.type == leptonica.lept.L_FLATE_ENCODE:
return rewrite_png(pike, im_obj, compdata, log)
return rewrite_png(pike, im_obj, compdata)
elif compdata.type == leptonica.lept.L_G4_ENCODE:
return rewrite_png_as_g4(pike, im_obj, compdata, log)
return rewrite_png_as_g4(pike, im_obj, compdata)
def rewrite_png_as_g4(pike, im_obj, compdata, log):
def rewrite_png_as_g4(pike, im_obj, compdata):
im_obj.BitsPerComponent = 1
im_obj.Width = compdata.w
im_obj.Height = compdata.h
@@ -447,7 +447,7 @@ def rewrite_png_as_g4(pike, im_obj, compdata, log):
return
def rewrite_png(pike, im_obj, compdata, log):
def rewrite_png(pike, im_obj, compdata):
# When a PNG is inserted into a PDF, we more or less copy the IDAT section from
# the PDF and transfer the rest of the PNG headers to PDF image metadata.
# One thing we have to do is tell the PDF reader whether a predictor was used
@@ -553,8 +553,8 @@ def optimize(input_file, output_file, context, save_settings):
def main(infile, outfile, level, jobs=1):
from tempfile import TemporaryDirectory
from shutil import copy
from tempfile import TemporaryDirectory # pylint: disable=import-outside-toplevel
from shutil import copy # pylint: disable=import-outside-toplevel
class OptimizeOptions:
"""Emulate ocrmypdf's options"""
+3 -3
View File
@@ -17,14 +17,13 @@
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
import logging
import os
import re
from collections import defaultdict, namedtuple
from decimal import Decimal
from enum import Enum
from functools import partial
from math import hypot, isclose
from os import PathLike, fspath
from os import PathLike
from pathlib import Path
from warnings import warn
@@ -821,13 +820,14 @@ class PdfInfo:
def main():
# pylint: disable=import-outside-toplevel
import argparse
from pprint import pprint
parser = argparse.ArgumentParser()
parser.add_argument('infile')
args = parser.parse_args()
pagesinfo, pdfinfo = _pdf_get_all_pageinfo(args.infile)
from pprint import pprint
pprint(pdfinfo)
for page in pagesinfo:
+10 -8
View File
@@ -24,7 +24,8 @@ from subprocess import PIPE, run
import pytest
from ocrmypdf import api, cli
from ocrmypdf import api, cli, pdfinfo
from ocrmypdf.exec import unpaper
pytest_plugins = ['helpers_namespace']
@@ -62,10 +63,8 @@ def running_in_travis():
@pytest.helpers.register
def have_unpaper():
try:
from ocrmypdf.exec import unpaper
unpaper.version()
except Exception:
except Exception: # pylint: disable=broad-except
return False
return True
@@ -95,7 +94,7 @@ assert ast.parse(WINDOWS_SHIM_TEMPLATE.format(spoofer=repr(r"C:\\Temp\\file.py")
@pytest.helpers.register
def spoof(tmp_path_factory, **kwargs):
"""Modify PATH to override subprocess executables
r"""Modify PATH to override subprocess executables
spoof(tmp_path_factory, program1='replacement', ...)
@@ -277,7 +276,12 @@ def run_ocrmypdf(input_file, output_file, *args, env=None, universal_newlines=Tr
env['COVERAGE_PROCESS_START'] = os.fspath(coverage_rc)
p = run(
p_args, stdout=PIPE, stderr=PIPE, universal_newlines=universal_newlines, env=env
p_args,
stdout=PIPE,
stderr=PIPE,
universal_newlines=universal_newlines,
env=env,
check=False,
)
# print(p.stderr)
return p, p.stdout, p.stderr
@@ -285,8 +289,6 @@ def run_ocrmypdf(input_file, output_file, *args, env=None, universal_newlines=Tr
@pytest.helpers.register
def first_page_dimensions(pdf):
from ocrmypdf import pdfinfo
info = pdfinfo.PdfInfo(pdf)
page0 = info[0]
return (page0.width_inches, page0.height_inches)
+10 -11
View File
@@ -29,8 +29,7 @@ from PIL import Image
import ocrmypdf
from ocrmypdf.exceptions import ExitCode, MissingDependencyError
from ocrmypdf.exec import ghostscript, tesseract
from ocrmypdf.helpers import check_pdf
from ocrmypdf.exec import get_version, ghostscript, tesseract
from ocrmypdf.pdfa import file_claims_pdfa
from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
@@ -311,7 +310,7 @@ def test_pagesegmode(renderer, spoof_tesseract_cache, resources, outpdf):
@pytest.mark.parametrize('renderer', RENDERERS)
def test_tesseract_crash(renderer, spoof_tesseract_crash, resources, no_outpdf, caplog):
def test_tesseract_crash(renderer, spoof_tesseract_crash, resources, no_outpdf):
p, _, err = run_ocrmypdf(
resources / 'ccitt.pdf',
no_outpdf,
@@ -410,7 +409,7 @@ def test_destination_not_writable(spoof_tesseract_noop, resources, outdir):
protected_file = outdir / 'protected.pdf'
protected_file.touch()
protected_file.chmod(0o400) # Read-only
p, out, err = run_ocrmypdf(
p, _out, _err = run_ocrmypdf(
resources / 'jbig2.pdf', protected_file, env=spoof_tesseract_noop
)
assert p.returncode == ExitCode.file_access_error, "Expected error"
@@ -448,7 +447,7 @@ THIS FILE IS INVALID
'''
)
p, out, err = run_ocrmypdf(
p, _out, err = run_ocrmypdf(
resources / 'ccitt.pdf',
outdir / 'out.pdf',
'--pdf-renderer',
@@ -568,6 +567,7 @@ def test_compression_preserved(
stdin=input_stream,
universal_newlines=True,
env=spoof_tesseract_noop,
check=False,
)
if im.mode in ('RGBA', 'LA'):
@@ -629,6 +629,7 @@ def test_compression_changed(
stdin=input_stream,
universal_newlines=True,
env=spoof_tesseract_noop,
check=False,
)
assert p.returncode == ExitCode.ok, p.stderr
@@ -711,10 +712,10 @@ def test_pdfa_n(spoof_tesseract_cache, pdfa_level, resources, outpdf):
)
@pytest.mark.slow
def test_decompression_bomb(resources, outpdf):
p, out, err = run_ocrmypdf(resources / 'hugemono.pdf', outpdf)
p, _out, err = run_ocrmypdf(resources / 'hugemono.pdf', outpdf)
assert 'decompression bomb' in err
p, out, err = run_ocrmypdf(
p, _out, err = run_ocrmypdf(
resources / 'hugemono.pdf', outpdf, '--max-image-mpixels', '2000'
)
assert p.returncode == 0
@@ -736,7 +737,7 @@ def test_text_curves(spoof_tesseract_noop, resources, outpdf):
def test_output_is_dir(spoof_tesseract_noop, resources, outdir):
p, out, err = run_ocrmypdf(
p, _out, err = run_ocrmypdf(
resources / 'trivial.pdf', outdir, '--force-ocr', env=spoof_tesseract_noop
)
assert p.returncode == ExitCode.file_access_error
@@ -747,7 +748,7 @@ def test_output_is_dir(spoof_tesseract_noop, resources, outdir):
def test_output_is_symlink(spoof_tesseract_noop, resources, outdir):
sym = Path(outdir / 'this_is_a_symlink')
sym.symlink_to(outdir / 'out.pdf')
p, out, err = run_ocrmypdf(
p, _out, err = run_ocrmypdf(
resources / 'trivial.pdf', sym, '--force-ocr', env=spoof_tesseract_noop
)
assert p.returncode == ExitCode.ok, err
@@ -761,8 +762,6 @@ def test_livecycle(resources, no_outpdf):
def test_version_check():
from ocrmypdf.exec import get_version
with pytest.raises(MissingDependencyError):
get_version('NOT_FOUND_UNLIKELY_ON_PATH')
+5 -11
View File
@@ -17,21 +17,18 @@
import datetime
import logging
import mmap
import os
from datetime import timezone
from os import fspath
from pathlib import Path
from shutil import copyfile, move
from unittest.mock import MagicMock, patch
from shutil import copyfile
from unittest.mock import patch
import pikepdf
import pytest
from pikepdf.models.metadata import decode_pdf_date
from ocrmypdf._jobcontext import PdfContext
from ocrmypdf._pipeline import convert_to_pdfa
from ocrmypdf._pipeline import convert_to_pdfa, metadata_fixup
from ocrmypdf.cli import get_parser
from ocrmypdf.exceptions import ExitCode
from ocrmypdf.pdfa import SRGB_ICC_PROFILE, file_claims_pdfa, generate_pdfa_ps
@@ -192,9 +189,8 @@ def test_xml_metadata_preserved(spoof_tesseract_noop, output_type, resources, ou
input_file = resources / 'graph.pdf'
try:
from libxmp import consts
from libxmp.utils import file_to_dict
except Exception:
from libxmp.utils import file_to_dict # pylint: disable=import-outside-toplevel
except Exception: # pylint: disable=broad-except
pytest.skip("libxmp not available or libexempi3 not installed")
before = file_to_dict(str(input_file))
@@ -290,8 +286,6 @@ def test_kodak_toc(resources, outpdf, spoof_tesseract_noop):
def test_metadata_fixup_warning(resources, outdir, caplog):
from ocrmypdf._pipeline import metadata_fixup
options = get_parser().parse_args(
args=['--output-type', 'pdfa-2', 'graph.pdf', 'out.pdf']
)
-1
View File
@@ -15,7 +15,6 @@
# You should have received a copy of the GNU General Public License
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
import logging
from os import fspath
from pathlib import Path
+3 -5
View File
@@ -18,7 +18,6 @@
import logging
import os
import subprocess
from contextlib import contextmanager
from os import fspath
from pathlib import Path
@@ -60,8 +59,8 @@ def test_skip_pages_does_not_replicate(resources, basename, outdir):
for page in info:
assert len(page.images) == 1, "skipped page was replicated"
for n in range(len(info_in)):
assert info[n].width_inches == info_in[n].width_inches
for n, info_out_n in enumerate(info):
assert info_out_n.width_inches == info_in[n].width_inches
def test_content_preservation(resources, outpdf):
@@ -131,8 +130,7 @@ def test_image_too_large_pdf(monkeypatch, resources, outdir):
def test_timeout(caplog):
tesseract.page_timedout('123456.png', 5)
assert "123456" in caplog.text
tesseract.page_timedout(5)
assert "took too long" in caplog.text
+2 -11
View File
@@ -23,24 +23,15 @@ import pytest
from ocrmypdf._validation import check_options
from ocrmypdf.cli import get_parser
from ocrmypdf.exceptions import ExitCode, MissingDependencyError
from ocrmypdf.exec import unpaper
# pytest.helpers is dynamic
# pylint: disable=no-member
# pylint: disable=no-member,redefined-outer-name
# pylint: disable=w0612
check_ocrmypdf = pytest.helpers.check_ocrmypdf
run_ocrmypdf = pytest.helpers.run_ocrmypdf
spoof = pytest.helpers.spoof
def have_unpaper():
try:
unpaper.version()
except Exception:
return False
else:
return True
have_unpaper = pytest.helpers.have_unpaper
@pytest.fixture
+2 -2
View File
@@ -145,9 +145,9 @@ def test_report_file_size(tmp_path, caplog):
def test_false_action_store_true():
opts = make_opts(keep_temporary_files=True)
assert opts.keep_temporary_files == True
assert opts.keep_temporary_files
opts = make_opts(keep_temporary_files=False)
assert opts.keep_temporary_files == False
assert not opts.keep_temporary_files
@pytest.mark.parametrize('progress_bar', [True, False])