ci : add packages for PyPy build

Error is:
TypeError: from_buffer() got a 'memoryview' object, which supports the buffer interface but cannot be rendered as a plain raw address on PyPy

PyPy is happy to access a bytes() copy of the memoryview.

.github/workflows/build.yml

@@ -18,8 +18,24 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-18.04] #, ubuntu-20.04]
+        include:
-        python: ["3.6"] #, "3.7", "3.8", "3.9"]
+          - os: ubuntu-18.04
+            python: 3.6
+          - os: ubuntu-18.04
+            python: 3.7
+          - os: ubuntu-20.04
+            python: 3.8
+          - os: ubuntu-20.04
+            python: 3.9
+          - os: ubuntu-latest
+            python: 3.9
+          - os: ubuntu-20.04
+            python: "pypy-3.6"
+          - os: ubuntu-latest
+            python: "pypy-3.7"
+          - os: ubuntu-latest
+            python: 3.9
+            tesseract5: true
 
     env:
       OS: ${{ matrix.os }}
@@ -35,6 +51,11 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
+      - name: Install Tesseract 5
+        if: matrix.tesseract5
+        run: |
+          sudo add-apt-repository ppa:alex-p/tesseract-ocr-devel
+
       - name: Install common packages
         run: |
           sudo apt-get update
@@ -65,6 +86,14 @@ jobs:
           sudo apt-get install -y --no-install-recommends \
             libexempi8
 
+      - name: Install Ubuntu packages for PyPy
+        if: startsWith(matrix.python, 'pypy')
+        run: |
+          sudo apt-get install -y --no-install-recommends \
+            libxml2-dev \
+            libxslt1-dev \
+            pypy3-dev
+
       - name: Install Python packages
         run: |
           python -m pip install .[test]

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.0.1
     hooks:
       - id: check-case-conflict
       - id: check-merge-conflict
@@ -12,12 +12,30 @@ repos:
     hooks:
       - id: seed-isort-config
   - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.7.0 # pick the isort version you'd like to use from https://github.com/pre-commit/mirrors-isort/releases
+    rev: v5.9.3 # pick the isort version you'd like to use from https://github.com/pre-commit/mirrors-isort/releases
     hooks:
       - id: isort
   - repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 21.7b0
     hooks:
       - id: black
         language_version: python
         exclude: ^src/ocrmypdf/lib/_leptonica.py
+  - repo: https://github.com/asottile/setup-cfg-fmt
+    rev: v1.17.0
+    hooks:
+      - id: setup-cfg-fmt
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.24.0
+    hooks:
+      - id: pyupgrade
+        args: ["--py36-plus"]
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.910
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          - types-toml
+          - types-setuptools
+          - types-requests
+          - types-Pillow

README.md

@@ -91,6 +91,11 @@ brew install tesseract-lang
 
 You can then pass the `-l LANG` argument to OCRmyPDF to give a hint as to what languages it should search for. Multiple languages can be requested.
 
+OCRmyPDF supports Tesseract 4.0 and the beta versions of Tesseract 5.0. It will
+automatically use whichever version it finds first on the `PATH` environment
+variable. On Windows, if `PATH` does not provide a Tesseract binary, we use
+the highest version number that is installed according to the Windows Registry.
+
 ## Documentation and support
 
 Once OCRmyPDF is installed, the built-in help which explains the command syntax and options can be accessed via:
@@ -115,7 +120,7 @@ In addition to the required Python version (3.6+), OCRmyPDF requires external pr
 - [heise Open Source, 09/2014: Texterkennung mit OCRmyPDF](https://heise.de/-2356670)
 - [heise Durchsuchbare PDF-Dokumente mit OCRmyPDF erstellen](https://www.heise.de/ratgeber/Durchsuchbare-PDF-Dokumente-mit-OCRmyPDF-erstellen-4607592.html)
 - [Excellent Utilities: OCRmyPDF](https://www.linuxlinks.com/excellent-utilities-ocrmypdf-add-ocr-text-layer-scanned-pdfs/)
-- [LinuxUser Texterkennung mit OCRmyPDF und Scanbd automatisieren](https://www.linux-community.de/ausgaben/linuxuser/2021/06/texterkennung-mit-ocrmypdf-und-scanbd-automatisieren/)  
+- [LinuxUser Texterkennung mit OCRmyPDF und Scanbd automatisieren](https://www.linux-community.de/ausgaben/linuxuser/2021/06/texterkennung-mit-ocrmypdf-und-scanbd-automatisieren/)
 
 ## Business enquiries
 

docs/conf.py

@@ -275,7 +275,7 @@ htmlhelp_basename = 'ocrmypdfdoc'
 
 # -- Options for LaTeX output ---------------------------------------------
 
-latex_elements = {
+latex_elements = {  # type: ignore
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',

docs/jbig2.rst

@@ -9,11 +9,11 @@ encoding was patented for a long time. All known JBIG2 US patents have
 expired as of 2017, but it is possible that unknown patents exist.
 
 JBIG2 encoding is recommended for OCRmyPDF and is used to losslessly
-create smaller PDFs. If JBIG2 encoding not available, lower quality
+create smaller PDFs. If JBIG2 encoding is not available, lower quality
 encodings will be used.
 
 JBIG2 decoding is not patented and is performed automatically by most
-PDF viewers. It is widely supported has been part of the PDF
+PDF viewers. It is widely supported and has been part of the PDF
 specification since 2001.
 
 On macOS, Homebrew packages jbig2enc and OCRmyPDF includes it by
@@ -37,7 +37,7 @@ Lossy mode JBIG2
 
 OCRmyPDF provides lossy mode JBIG2 as an advanced feature. Users should
 `review the technical concerns with JBIG2 in lossy
-mode <https://abbyy.technology/en:kb:tip:jbig2_compression_and_ocr>`__
+mode <https://en.wikipedia.org/wiki/JBIG2#Disadvantages>`__
 and decide if this feature is acceptable for their use case.
 
 JBIG2 lossy mode does achieve higher compression ratios than any other

docs/release_notes.rst

@@ -12,6 +12,25 @@ may be unreliable. Use the API to depend on precise behavior.
 The public API may be useful in scripts that launch OCRmyPDF processes or that
 wish to use some of its features for working with PDFs.
 
+v12.4.0
+=======
+
+-  When grafting text layers, use pikepdf's ``unparse_content_stream`` if available.
+-  Confirmed support for pluggy 1.0. (Thanks @QuLogic.)
+-  Fixed some typing issues, improved pre-commit settings, and fixed issues
+   flagged by linters.
+-  PyPy 7.3.3 (=Python 3.6) is now supported. Note that PyPy does not necessarily
+   run faster, because the vast majority of OCRmyPDF's execution time is spent
+   running OCR or generally executing native code. However, PyPy may bring speed
+   improvements in some areas.
+
+v12.3.3
+=======
+
+-  watcher.py: fixed interpretation of boolean env vars (:issue:`821`).
+-  Adjust CI scripts to test Tesseract 5 betas.
+-  Document our support for the Tesseract 5 betas.
+
 v12.3.2
 =======
 

misc/watcher.py

@@ -37,14 +37,19 @@ import ocrmypdf
 
 # pylint: disable=logging-format-interpolation
 
+
+def getenv_bool(name: str, default: str = 'False'):
+    return os.getenv(name, default).lower() in ('true', 'yes', 'y', '1')
+
+
 INPUT_DIRECTORY = os.getenv('OCR_INPUT_DIRECTORY', '/input')
 OUTPUT_DIRECTORY = os.getenv('OCR_OUTPUT_DIRECTORY', '/output')
-OUTPUT_DIRECTORY_YEAR_MONTH = bool(os.getenv('OCR_OUTPUT_DIRECTORY_YEAR_MONTH', ''))
+OUTPUT_DIRECTORY_YEAR_MONTH = getenv_bool('OCR_OUTPUT_DIRECTORY_YEAR_MONTH')
-ON_SUCCESS_DELETE = bool(os.getenv('OCR_ON_SUCCESS_DELETE', ''))
+ON_SUCCESS_DELETE = getenv_bool('OCR_ON_SUCCESS_DELETE')
-DESKEW = bool(os.getenv('OCR_DESKEW', ''))
+DESKEW = getenv_bool('OCR_DESKEW')
 OCR_JSON_SETTINGS = json.loads(os.getenv('OCR_JSON_SETTINGS', '{}'))
 POLL_NEW_FILE_SECONDS = int(os.getenv('OCR_POLL_NEW_FILE_SECONDS', '1'))
-USE_POLLING = bool(os.getenv('OCR_USE_POLLING', ''))
+USE_POLLING = getenv_bool('OCR_USE_POLLING')
 LOGLEVEL = os.getenv('OCR_LOGLEVEL', 'INFO')
 PATTERNS = ['*.pdf', '*.PDF']
 

misc/webservice.py

@@ -37,7 +37,7 @@ app.secret_key = "secret"
 app.config['MAX_CONTENT_LENGTH'] = 50_000_000
 app.config.from_envvar("OCRMYPDF_WEBSERVICE_SETTINGS", silent=True)
 
-ALLOWED_EXTENSIONS = set(["pdf"])
+ALLOWED_EXTENSIONS = {"pdf"}
 
 
 def allowed_file(filename):

pyproject.toml

@@ -34,3 +34,41 @@ exclude = '''
   | src/ocrmypdf/lib/_leptonica.py
 )/
 '''
+
+[tool.coverage.run]
+branch = true
+parallel = true
+concurrency = ["multiprocessing"]
+
+[tool.coverage.paths]
+source = ["src/ocrmypdf"]
+
+[tool.coverage.report]
+# Regexes for lines to exclude from consideration
+exclude_lines = [
+    # Have to re-enable the standard pragma
+    "pragma: no cover",
+
+    # Don't complain if tests don't hit defensive assertion code:
+    "raise AssertionError",
+    "raise NotImplementedError",
+
+    # Don't complain if non-runnable code isn't run:
+    "if 0:",
+	  "if False:",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:"
+]
+
+[tool.isort]
+profile = "black"
+known_first_party = "ocrmypdf"
+known_third_party = ["PIL", "_cffi_backend", "cffi", "flask", "img2pdf", "ocrmypdf", "pdfminer", "pikepdf", "pkg_resources", "pluggy", "pytest", "reportlab", "setuptools", "sphinx_rtd_theme", "tqdm", "watchdog", "werkzeug"]
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+norecursedirs = ["lib", ".pc", ".git", "venv", "output", "cache", "resources"]
+testpaths = ["tests"]
+addopts = "-n auto"
+markers = ["slow"]
+filterwarnings = ["ignore:.*XMLParser.*:DeprecationWarning"]

setup.cfg

@@ -2,23 +2,15 @@
 name = ocrmypdf
 description = OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched
 long_description = file: README.md
-long_description_content_type = text/markdown; charset=UTF-8
+long_description_content_type = text/markdown
 url = https://github.com/jbarlow83/OCRmyPDF
 author = James R. Barlow
 author_email = james@purplerock.ca
+license = MPL-2.0
+license_file = LICENSE
 license_files =
     LICENSE
-keywords =
-    PDF
-    OCR
-    optical character recognition
-    PDF/A
-    scanning
 classifiers =
-    Programming Language :: Python :: 3.6
-    Programming Language :: Python :: 3.7
-    Programming Language :: Python :: 3.8
-    Programming Language :: Python :: 3.9
     Development Status :: 5 - Production/Stable
     Environment :: Console
     Intended Audience :: End Users/Desktop
@@ -30,68 +22,80 @@ classifiers =
     Operating System :: POSIX
     Operating System :: POSIX :: BSD
     Operating System :: POSIX :: Linux
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
     Topic :: Scientific/Engineering :: Image Recognition
     Topic :: Text Processing :: Indexing
     Topic :: Text Processing :: Linguistic
+keywords =
+    PDF
+    OCR
+    optical character recognition
+    PDF/A
+    scanning
 project_urls =
     Documentation = https://ocrmypdf.readthedocs.io/
     Source = https://github.com/jbarlow83/ocrmypdf
     Tracker = https://github.com/jbarlow83/ocrmypdf/issues
 
 [options]
-zip_safe = False
 packages = find:
+install_requires =
+    Pillow>=8.2.0
+    cffi>=1.9.1  # must be a setup and install requirement
+    coloredlogs>=14.0  # strictly optional
+    img2pdf>=0.3.0,<0.5  # pure Python
+    pdfminer.six!=20200720,>=20191110,<=20201018
+    pikepdf>=2.10.0
+    pluggy>=0.13.0,<2
+    reportlab>=3.5.66
+    setuptools
+    tqdm>=4
+python_requires = >=3.6
+include_package_data = True
 package_dir =
     =src
 platforms = any
-include_package_data=True
+setup_requires =
-install_requires =
+    cffi>=1.9.1  # to build the leptonica module
-    cffi >= 1.9.1  # must be a setup and install requirement
+    setuptools_scm
-    coloredlogs >= 14.0  # strictly optional
+    setuptools_scm_git_archive
-    img2pdf >= 0.3.0, < 0.5  # pure Python, so track HEAD closely
+zip_safe = False
-    pdfminer.six >= 20191110, != 20200720, <= 20201018
+
-    pikepdf >= 2.10.0
+[options.packages.find]
-    Pillow >= 8.2.0
+where = src
-    pluggy >= 0.13.0, < 1.0
+
-    reportlab >= 3.5.66
+[options.entry_points]
-    setuptools
+console_scripts =
-    tqdm >= 4
+    ocrmypdf = ocrmypdf.__main__:run
-python_requires = >= 3.6
+
-setup_requires =  # can be removed whenever we can drop pip 9 support
+[options.extras_require]
-    cffi >= 1.9.1  # to build the leptonica module
+docs =
-    setuptools_scm  # so that version will work
+    sphinx
-    setuptools_scm_git_archive  # enable version from github tarballs
+    sphinx-issues
+    sphinx-rtd-theme
+extended_test =
+    PyMuPDF==1.13.4
+test =
+    coverage[toml]>=5
+    pytest>=6.0.0
+    pytest-cov>=2.11.1
+    pytest-xdist>=2.2.0
+    python-xmp-toolkit==2.0.1  # also requires apt-get install libexempi3
+watcher =
+    watchdog>=1.0.2,<3
+webservice =
+    Flask>=1,<3
 
 [options.package_data]
 ocrmypdf =
     data/sRGB.icc
     py.typed
 
-[options.packages.find]
-where = src
-
-[options.extras_require]
-test =
-    pytest >= 6.0.0
-    pytest-xdist >= 2.2.0
-    pytest-cov >= 2.11.1
-    python-xmp-toolkit == 2.0.1  # also requires apt-get install libexempi3
-                                 # or brew install exempi
-docs =
-    sphinx
-    sphinx-rtd-theme
-    sphinx-issues
-extended_test =
-    PyMuPDF == 1.13.4
-watcher =
-    watchdog >= 1.0.2, < 3
-webservice =
-    Flask >= 1, < 3
-
-[options.entry_points]
-console_scripts =
-    ocrmypdf = ocrmypdf.__main__:run
-
 [bdist_wheel]
 python-tag = py36
 
@@ -100,48 +104,4 @@ test = pytest
 
 [check-manifest]
 ignore =
-	.github
+    .github
-
-[tool:pytest]
-norecursedirs = lib .pc .git output cache resources
-testpaths = tests
-filterwarnings =
-	ignore:.*XMLParser.*:DeprecationWarning
-markers =
-	slow
-addopts =
-	-n auto
-
-[isort]
-multi_line_output = 3
-include_trailing_comma = True
-force_grid_wrap = 0
-use_parentheses = True
-line_length = 88
-known_first_party = ocrmypdf
-known_third_party = PIL,_cffi_backend,cffi,flask,img2pdf,pdfminer,pikepdf,pkg_resources,pluggy,pytest,reportlab,setuptools,sphinx_rtd_theme,tqdm,watchdog,werkzeug
-
-[coverage:paths]
-source =
-	src/ocrmypdf
-
-[coverage:run]
-branch = true
-parallel = true
-concurrency = multiprocessing
-
-[coverage:report]
-# Regexes for lines to exclude from consideration
-exclude_lines =
-    # Have to re-enable the standard pragma
-    pragma: no cover
-
-    # Don't complain if tests don't hit defensive assertion code:
-    raise AssertionError
-    raise NotImplementedError
-
-    # Don't complain if non-runnable code isn't run:
-    if 0:
-	if False:
-    if __name__ == .__main__.:
-	if TYPE_CHECKING:

src/ocrmypdf/_exec/tesseract.py

@@ -20,6 +20,7 @@ from typing import List, Optional
 
 from PIL import Image
 
+from ocrmypdf.api import StrPath
 from ocrmypdf.exceptions import (
     MissingDependencyError,
     SubprocessOutputError,
@@ -117,7 +118,7 @@ def get_languages():
         if line.startswith('Error'):
             raise MissingDependencyError(lang_error(output))
     _header, *rest = output.splitlines()
-    return set(lang.strip() for lang in rest)
+    return {lang.strip() for lang in rest}
 
 
 def tess_base_args(langs: List[str], engine_mode: Optional[int]) -> List[str]:
@@ -250,7 +251,8 @@ def generate_hocr(
 
     # Reminder: test suite tesseract test plugins will break after any changes
     # to the number of order parameters here
-    args_tesseract.extend([input_file, prefix, 'hocr', 'txt'] + tessconfig)
+    args_tesseract.extend([os.fspath(input_file), os.fspath(prefix), 'hocr', 'txt'])
+    args_tesseract.extend(tessconfig)
     try:
         p = run(args_tesseract, stdout=PIPE, stderr=STDOUT, timeout=timeout, check=True)
         stdout = p.stdout
@@ -324,7 +326,8 @@ def generate_pdf(
     # Reminder: test suite tesseract test plugins might break after any changes
     # to the number of order parameters here
 
-    args_tesseract.extend([input_file, prefix, 'pdf', 'txt'] + tessconfig)
+    args_tesseract.extend([os.fspath(input_file), os.fspath(prefix), 'pdf', 'txt'])
+    args_tesseract.extend(tessconfig)
     try:
         p = run(args_tesseract, stdout=PIPE, stderr=STDOUT, timeout=timeout, check=True)
         stdout = p.stdout

src/ocrmypdf/_exec/unpaper.py

@@ -45,7 +45,7 @@ def _setup_unpaper_io(tmpdir: Path, input_file: Path) -> Tuple[Path, Path]:
                     im = im.convert(mode='1')
                 else:
                     im = im.convert(mode='RGB')
-            except IOError as e:
+            except OSError as e:
                 raise MissingDependencyError(
                     "Could not convert image with type " + im.mode
                 ) from e

src/ocrmypdf/_graft.py

@@ -73,17 +73,24 @@ def strip_invisible_text(pdf, page):
         except AttributeError:
             return str(op).encode('ascii')
 
-    lines = []
+    if hasattr(pikepdf, 'unparse_content_stream'):
+        content_stream = pikepdf.unparse_content_stream(stream)
+    else:
+        lines = []
 
-    for operands, operator in stream:
+        for operands, operator in stream:
-        if operator == pikepdf.Operator('INLINE IMAGE'):
+            if operator == pikepdf.Operator('INLINE IMAGE'):
-            iim = operands[0]
+                iim = operands[0]
-            line = iim.unparse()
+                line = iim.unparse()
-        else:
+            else:
-            line = b' '.join(convert(op) for op in operands) + b' ' + operator.unparse()
+                line = (
-        lines.append(line)
+                    b' '.join(convert(op) for op in operands)
+                    + b' '
+                    + operator.unparse()
+                )
+            lines.append(line)
 
-    content_stream = b'\n'.join(lines)
+        content_stream = b'\n'.join(lines)
     page.Contents = pikepdf.Stream(pdf, content_stream)
 
 

src/ocrmypdf/_pipeline.py

@@ -48,7 +48,7 @@ def triage_image_file(input_file, output_file, options):
     log.info("Input file is not a PDF, checking if it is an image...")
     try:
         im = Image.open(input_file)
-    except EnvironmentError as e:
+    except OSError as e:
         # Recover the original filename
         log.error(str(e).replace(str(input_file), str(options.input_file)))
         raise UnsupportedImageFormatError() from e
@@ -135,7 +135,7 @@ def triage(original_filename, input_file, output_file, options):
             # Origin file is a pdf create a symlink with pdf extension
             safe_symlink(input_file, output_file)
             return output_file
-    except EnvironmentError as e:
+    except OSError as e:
         log.debug(f"Temporary file was at: {input_file}")
         msg = str(e).replace(str(input_file), original_filename)
         raise InputFileError(msg) from e
@@ -521,13 +521,12 @@ def create_ocr_image(image: Path, page_context: PageContext):
                 # be None)
                 bbox = [float(v) for v in textarea]
                 xyscale = tuple(float(coord) / 72.0 for coord in im.info['dpi'])
-                pixcoords = [
+                pixcoords = (
                     bbox[0] * xyscale[0],
                     im.height - bbox[3] * xyscale[1],
                     bbox[2] * xyscale[0],
                     im.height - bbox[1] * xyscale[1],
-                ]
+                )
-                pixcoords = [int(round(c)) for c in pixcoords]
                 log.debug('blanking %r', pixcoords)
                 draw.rectangle(pixcoords, fill=white)
                 # draw.rectangle(pixcoords, outline=pink)
@@ -856,7 +855,7 @@ def merge_sidecars(txt_files: Iterable[Optional[Path]], context: PdfContext):
             if frm != 1:
                 stream.write('\f')  # Form feed between pages
             if txt_file:
-                with open(txt_file, 'r', encoding="utf-8") as in_:
+                with open(txt_file, encoding="utf-8") as in_:
                     txt = in_.read()
                     # Some OCR engines (e.g. Tesseract v4 alpha) add form feeds
                     # between pages, and some do not. For consistency, we ignore

src/ocrmypdf/_sync.py

@@ -421,7 +421,7 @@ def run_pipeline(options, *, plugin_manager, api=False):
             try:
                 debug_log_handler.close()
                 log.removeHandler(debug_log_handler)
-            except EnvironmentError as e:
+            except OSError as e:
                 print(e, file=sys.stderr)
         cleanup_working_files(work_folder, options)
 

src/ocrmypdf/_validation.py

@@ -13,7 +13,7 @@ import sys
 import unicodedata
 from pathlib import Path
 from shutil import copyfileobj
-from typing import List, Set, Tuple, Union
+from typing import List, Set, Tuple
 
 import pikepdf
 import PIL
@@ -142,8 +142,6 @@ def check_options_preprocessing(options):
 
 
 def _pages_from_ranges(ranges: str) -> Set[int]:
-    if is_iterable_notstr(ranges):
-        return set(ranges)
     pages: List[int] = []
     page_groups = ranges.replace(' ', '').split(',')
     for g in page_groups:
@@ -182,10 +180,8 @@ def _pages_from_ranges(ranges: str) -> Set[int]:
 
 def check_options_ocr_behavior(options):
     exclusive_options = sum(
-        [
+        (1 if opt else 0)
-            (1 if opt else 0)
+        for opt in (options.force_ocr, options.skip_text, options.redo_ocr)
-            for opt in (options.force_ocr, options.skip_text, options.redo_ocr)
-        ]
     )
     if exclusive_options >= 2:
         raise BadArgsError("Choose only one of --force-ocr, --skip-text, --redo-ocr.")
@@ -302,7 +298,7 @@ def check_closed_streams(options):  # pragma: no cover
         if options.input_file == '-':
             log.error("Trying to read from stdin but stdin seems closed")
             return False
-        sys.stdin = open(os.devnull, 'r')
+        sys.stdin = open(os.devnull)
 
     if sys.stdout is None:
         if options.output_file == '-':

src/ocrmypdf/api.py

@@ -31,7 +31,7 @@ except ModuleNotFoundError:
     coloredlogs = None
 
 
-StrPath = Union[os.PathLike, AnyStr]
+StrPath = Union[Path, AnyStr]
 PathOrIO = Union[BinaryIO, StrPath]
 
 _api_lock = threading.Lock()

src/ocrmypdf/builtin_plugins/concurrency.py

@@ -20,9 +20,8 @@ import signal
 import sys
 import threading
 from contextlib import suppress
-from multiprocessing import Pool as ProcessPool
+from multiprocessing.pool import Pool, ThreadPool
-from multiprocessing.pool import ThreadPool
+from typing import Callable, Iterable, Optional, Tuple, Type, Union
-from typing import Callable, Iterable, Union
 
 from tqdm import tqdm
 
@@ -31,7 +30,10 @@ from ocrmypdf._logging import TqdmConsole
 from ocrmypdf.exceptions import InputFileError
 from ocrmypdf.helpers import remove_all_log_handlers
 
+ProcessPool = Pool
 Queue = Union[multiprocessing.Queue, queue.Queue]
+UserInit = Callable[[], None]
+WorkerInit = Callable[[Queue, UserInit, int], None]
 
 
 def log_listener(q: Queue):
@@ -62,7 +64,7 @@ def process_sigbus(*args):
     raise InputFileError("A worker process lost access to an input file")
 
 
-def process_init(q: Queue, user_init: Callable[[], None], loglevel):
+def process_init(q: Queue, user_init: UserInit, loglevel) -> None:
     """Initialize a process pool worker"""
 
     # Ignore SIGINT (our parent process will kill us gracefully)
@@ -85,7 +87,7 @@ def process_init(q: Queue, user_init: Callable[[], None], loglevel):
     return
 
 
-def thread_init(_queue: Queue, user_init: Callable[[], None], _loglevel):
+def thread_init(q: Queue, user_init: UserInit, loglevel) -> None:
     # As a thread, block SIGBUS so the main thread deals with it...
     with suppress(AttributeError):
         signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGBUS})
@@ -107,9 +109,9 @@ class StandardExecutor(Executor):
         task_finished: Callable,
     ):
         if use_threads:
-            log_queue = queue.Queue(-1)
+            log_queue: Queue = queue.Queue(-1)
-            pool_class = ThreadPool
+            pool_class: Type[Pool] = ThreadPool
-            initializer = thread_init
+            initializer: WorkerInit = thread_init
         else:
             log_queue = multiprocessing.Queue(-1)
             pool_class = ProcessPool

src/ocrmypdf/cli.py

@@ -6,7 +6,7 @@
 
 
 import argparse
-from typing import Optional, Type, TypeVar
+from typing import Any, Callable, Optional, TypeVar
 
 from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME
 from ocrmypdf._version import __version__ as _VERSION
@@ -14,7 +14,9 @@ from ocrmypdf._version import __version__ as _VERSION
 T = TypeVar('T')
 
 
-def numeric(basetype: Type[T], min_: Optional[T] = None, max_: Optional[T] = None):
+def numeric(
+    basetype: Callable[[Any], T], min_: Optional[T] = None, max_: Optional[T] = None
+):
     """Validator for numeric params"""
     min_ = basetype(min_) if min_ is not None else None
     max_ = basetype(max_) if max_ is not None else None
@@ -22,7 +24,7 @@ def numeric(basetype: Type[T], min_: Optional[T] = None, max_: Optional[T] = Non
     def _numeric(string):
         value = basetype(string)
         if (min_ is not None and value < min_) or (max_ is not None and value > max_):
-            msg = "%r not in valid range %r" % (string, (min_, max_))
+            msg = f"{string!r} not in valid range {(min_, max_)!r}"
             raise argparse.ArgumentTypeError(msg)
         return value
 

src/ocrmypdf/extra_plugins/semfree.py

@@ -28,7 +28,7 @@ from enum import Enum, auto
 from itertools import islice, repeat, takewhile, zip_longest
 from multiprocessing import Pipe, Process
 from multiprocessing.connection import Connection, wait
-from typing import Callable, Iterable, Iterator
+from typing import Callable, Iterable, Iterator, List
 
 from ocrmypdf import Executor, hookimpl
 from ocrmypdf._concurrent import NullProgressBar
@@ -60,7 +60,9 @@ def process_sigbus(*args):
 
 class ConnectionLogHandler(logging.handlers.QueueHandler):
     def __init__(self, conn: Connection) -> None:
-        super().__init__(None)
+        # sets the parent's queue to None - parent only touches queue
+        # in enqueue() which we override
+        super().__init__(None)  # type: ignore
         self.conn = conn
 
     def enqueue(self, record):
@@ -126,8 +128,8 @@ class LambdaExecutor(Executor):
         if not grouped_args:
             return
 
-        processes = []
+        processes: List[Process] = []
-        connections = []
+        connections: List[Connection] = []
         for chunk in grouped_args:
             parent_conn, child_conn = Pipe()
 
@@ -152,6 +154,8 @@ class LambdaExecutor(Executor):
         with self.pbar_class(**tqdm_kwargs) as pbar:
             while connections:
                 for r in wait(connections):
+                    if not isinstance(r, Connection):
+                        raise NotImplementedError("We only support Connection()")
                     try:
                         msg_type, msg = r.recv()
                     except EOFError:

src/ocrmypdf/helpers.py

@@ -189,7 +189,7 @@ def is_file_writable(test_file: os.PathLike) -> bool:
                 with suppress(OSError):
                     p.unlink()
             return True
-    except (EnvironmentError, RuntimeError) as e:
+    except (OSError, RuntimeError) as e:
         log.debug(e)
         log.error(str(e))
         return False
@@ -273,7 +273,7 @@ def deprecated(func):
     def new_func(*args, **kwargs):
         warnings.simplefilter('always', DeprecationWarning)  # turn off filter
         warnings.warn(
-            "Call to deprecated function {}.".format(func.__name__),
+            f"Call to deprecated function {func.__name__}.",
             category=DeprecationWarning,
             stacklevel=2,
         )

src/ocrmypdf/hocrtransform.py

@@ -349,7 +349,7 @@ class HocrTransform:
         interword_spaces: bool,
         show_bounding_boxes: bool,
     ):
-        if not line:
+        if line is not None:
             return
         pxl_line_coords = self.element_coordinates(line)
         line_box = self.pt_from_pixel(pxl_line_coords)

src/ocrmypdf/leptonica.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # © 2013-16: jbarlow83 from Github (https://github.com/jbarlow83)
 #
@@ -13,6 +12,7 @@
 import argparse
 import logging
 import os
+import platform
 import sys
 import threading
 from collections import deque
@@ -23,6 +23,7 @@ from functools import lru_cache
 from io import BytesIO, UnsupportedOperation
 from os import fspath
 from tempfile import TemporaryFile
+from typing import ContextManager, Type
 from warnings import warn
 
 from ocrmypdf.exceptions import MissingDependencyError
@@ -67,7 +68,7 @@ if os.name == 'nt':
     # Loading zlib from other places could cause a version mismatch
     _zlib_path = os.path.join(os.path.dirname(_libpath), 'zlib1.dll')
     if not os.path.exists(_zlib_path):
-        _zlib_path = find_library('zlib')
+        _zlib_path = find_library('zlib') or ''
     try:
         zlib = ffi.dlopen(_zlib_path)
     except ffi.error as e:
@@ -86,7 +87,7 @@ except ffi.error as e:
     ) from e
 
 
-class _LeptonicaErrorTrap_Redirect:
+class _LeptonicaErrorTrap_Redirect(ContextManager):
     """
     Context manager to trap errors reported by Leptonica < 1.79 or on Apple Silicon.
 
@@ -132,7 +133,7 @@ class _LeptonicaErrorTrap_Redirect:
         except Exception:
             self.leptonica_lock.release()
             raise
-        return self
+        return
 
     def __exit__(self, exc_type, exc_value, traceback):
         # Restore old stderr
@@ -172,7 +173,7 @@ tls = threading.local()
 tls.trap = None
 
 
-class _LeptonicaErrorTrap_Queue:
+class _LeptonicaErrorTrap_Queue(ContextManager):
     def __init__(self):
         self.queue = deque()
 
@@ -226,7 +227,7 @@ except (ffi.error, MemoryError):
     # Pre-1.79 Leptonica does not have leptSetStderrHandler
     # And some platforms, notably Apple ARM 64, do not allow the write+execute
     # memory needed to set up the callback function.
-    _LeptonicaErrorTrap = _LeptonicaErrorTrap_Redirect
+    _LeptonicaErrorTrap: Type[ContextManager] = _LeptonicaErrorTrap_Redirect
 else:
     # 1.79 have this new symbol
     _LeptonicaErrorTrap = _LeptonicaErrorTrap_Queue
@@ -272,7 +273,7 @@ class LeptonicaObject:
         # Leptonica API uses double-pointers for its destroy APIs to prevent
         # dangling pointers. This means we need to put our single pointer,
         # cdata, in a temporary CDATA**.
-        pp = ffi.new('{} **'.format(cls.LEPTONICA_TYPENAME), cdata)
+        pp = ffi.new(f'{cls.LEPTONICA_TYPENAME} **', cdata)
         cls.cdata_destroy(pp)
 
 
@@ -439,6 +440,9 @@ class Pix(LeptonicaObject):
         bio = BytesIO()
         pillow_image.save(bio, format='png', compress_level=1)
         py_buffer = bio.getbuffer()
+        if platform.python_implementation() == 'PyPy':
+            # PyPy complains that it cannot do from_buffer(memoryview)
+            py_buffer = bytes(py_buffer)
         c_buffer = ffi.from_buffer(py_buffer)
         with _LeptonicaErrorTrap():
             pix = Pix(lept.pixReadMem(c_buffer, len(c_buffer)))
@@ -844,7 +848,7 @@ class Box(LeptonicaObject):
 
     def __repr__(self):
         if self._cdata:
-            return '<leptonica.Box x={0} y={1} w={2} h={3}>'.format(
+            return '<leptonica.Box x={} y={} w={} h={}>'.format(
                 self.x, self.y, self.w, self.h
             )
         return '<leptonica.Box NULL>'
@@ -916,7 +920,7 @@ class Sel(LeptonicaObject):
         lines = [line.strip() for line in selstr.split('\n') if line.strip()]
         h = len(lines)
         w = len(lines[0])
-        lengths = set(len(line) for line in lines)
+        lengths = {len(line) for line in lines}
         if len(lengths) != 1:
             raise ValueError("All lines in selstr must be same length")
 

src/ocrmypdf/optimize.py

@@ -365,6 +365,7 @@ def convert_to_jbig2(
     When the JBIG2 symbolic coder is not used, each JBIG2 stands on its own
     and needs no dictionary. Currently this must be lossless JBIG2.
     """
+    jbig2_globals_dict: Optional[Dictionary]
 
     _produce_jbig2_images(jbig2_groups, root, options, executor)
 

src/ocrmypdf/pdfinfo/info.py

@@ -17,11 +17,21 @@ from functools import partial
 from math import hypot, inf, isclose
 from os import PathLike
 from pathlib import Path
-from typing import Container, Iterator, Optional, Tuple, Union
+from typing import (
+    Container,
+    Dict,
+    Iterator,
+    List,
+    Mapping,
+    NamedTuple,
+    Optional,
+    Tuple,
+    Union,
+)
 from warnings import warn
 
 import pikepdf
-from pikepdf import Object, Pdf, PdfMatrix
+from pikepdf import Name, Object, Pdf, PdfInlineImage, PdfMatrix
 
 from ocrmypdf._concurrent import Executor, SerialExecutor
 from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
@@ -36,7 +46,7 @@ Encoding = Enum(
     'Encoding', 'ccitt jpeg jpeg2000 jbig2 asciihex ascii85 lzw flate runlength'
 )
 
-FRIENDLY_COLORSPACE = {
+FRIENDLY_COLORSPACE: Dict[str, Colorspace] = {
     '/DeviceGray': Colorspace.gray,
     '/CalGray': Colorspace.gray,
     '/DeviceRGB': Colorspace.rgb,
@@ -54,7 +64,7 @@ FRIENDLY_COLORSPACE = {
     '/I': Colorspace.index,
 }
 
-FRIENDLY_ENCODING = {
+FRIENDLY_ENCODING: Dict[str, Encoding] = {
     '/CCITTFaxDecode': Encoding.ccitt,
     '/DCTDecode': Encoding.jpeg,
     '/JPXDecode': Encoding.jpeg2000,
@@ -68,7 +78,7 @@ FRIENDLY_ENCODING = {
     '/RL': Encoding.runlength,
 }
 
-FRIENDLY_COMP = {
+FRIENDLY_COMP: Dict[Colorspace, int] = {
     Colorspace.gray: 1,
     Colorspace.rgb: 3,
     Colorspace.cmyk: 4,
@@ -86,16 +96,30 @@ def _is_unit_square(shorthand):
     return all(isclose(a, b, rel_tol=1e-3) for a, b in pairwise)
 
 
-XobjectSettings = namedtuple('XobjectSettings', ['name', 'shorthand', 'stack_depth'])
+class XobjectSettings(NamedTuple):
+    name: str
+    shorthand: Tuple[float, float, float, float, float, float]
+    stack_depth: int
 
-InlineSettings = namedtuple('InlineSettings', ['iimage', 'shorthand', 'stack_depth'])
 
-ContentsInfo = namedtuple(
+class InlineSettings(NamedTuple):
-    'ContentsInfo',
+    iimage: PdfInlineImage
-    ['xobject_settings', 'inline_images', 'found_vector', 'found_text', 'name_index'],
+    shorthand: Tuple[float, float, float, float, float, float]
-)
+    stack_depth: int
 
-TextboxInfo = namedtuple('TextboxInfo', ['bbox', 'is_visible', 'is_corrupt'])
+
+class ContentsInfo(NamedTuple):
+    xobject_settings: List[XobjectSettings]
+    inline_images: List[InlineSettings]
+    found_vector: bool
+    found_text: bool
+    name_index: Mapping[str, List[XobjectSettings]]
+
+
+class TextboxInfo(NamedTuple):
+    bbox: Tuple[float, float, float, float]
+    is_visible: bool
+    is_corrupt: bool
 
 
 class VectorMarker:
@@ -146,8 +170,8 @@ def _interpret_contents(contentstream: Object, initial_shorthand=UNIT_SQUARE):
 
     stack = []
     ctm = PdfMatrix(initial_shorthand)
-    xobject_settings = []
+    xobject_settings: List[XobjectSettings] = []
-    inline_images = []
+    inline_images: List[InlineSettings] = []
     name_index = defaultdict(lambda: [])
     found_vector = False
     found_text = False
@@ -185,7 +209,7 @@ def _interpret_contents(contentstream: Object, initial_shorthand=UNIT_SQUARE):
                 name=image_name, shorthand=ctm.shorthand, stack_depth=len(stack)
             )
             xobject_settings.append(settings)
-            name_index[image_name].append(settings)
+            name_index[str(image_name)].append(settings)
         elif operator == 'INLINE IMAGE':  # BI/ID/EI are grouped into this
             iimage = operands[0]
             inline = InlineSettings(
@@ -271,6 +295,9 @@ def _get_dpi(ctm_shorthand, image_size) -> Resolution:
 class ImageInfo:
     DPI_PREC = Decimal('1.000')
 
+    _comp: Optional[int]
+    _name: str
+
     def __init__(
         self,
         *,
@@ -303,14 +330,14 @@ class ImageInfo:
 
         self._bpc = int(pim.bits_per_component)
         try:
-            self._enc = FRIENDLY_ENCODING.get(pim.filters[0], 'image')
+            self._enc = FRIENDLY_ENCODING.get(pim.filters[0])
         except IndexError:
-            self._enc = '?'
+            self._enc = None
 
         try:
-            self._color = FRIENDLY_COLORSPACE.get(pim.colorspace, '?')
+            self._color = FRIENDLY_COLORSPACE.get(pim.colorspace)
         except NotImplementedError:
-            self._color = '?'
+            self._color = None
         if self._enc == Encoding.jpeg2000:
             self._color = Colorspace.jpeg2000
 
@@ -324,11 +351,14 @@ class ImageInfo:
             else:
                 self._comp = 3
         else:
-            self._comp = FRIENDLY_COMP.get(self._color, '?')
+            if isinstance(self._color, Colorspace):
+                self._comp = FRIENDLY_COMP.get(self._color)
+            else:
+                self._comp = None
 
             # Bit of a hack... infer grayscale if component count is uncertain
             # but encoding only supports monochrome.
-            if self._comp == '?' and self._enc in (Encoding.ccitt, Encoding.jbig2):
+            if self._comp is None and self._enc in (Encoding.ccitt, Encoding.jbig2):
                 self._comp = FRIENDLY_COMP[Colorspace.gray]
 
     @property
@@ -353,15 +383,15 @@ class ImageInfo:
 
     @property
     def color(self):
-        return self._color
+        return self._color if self._color is not None else '?'
 
     @property
     def comp(self):
-        return self._comp
+        return self._comp if self._comp is not None else '?'
 
     @property
     def enc(self):
-        return self._enc
+        return self._enc if self._enc is not None else 'image'
 
     @property
     def renderable(self):
@@ -661,6 +691,10 @@ def _pdf_pageinfo_concurrent(
 
 
 class PageInfo:
+    _has_text: Optional[bool]
+    _has_vector: Optional[bool]
+    _images: List[ImageInfo]
+
     def __init__(
         self,
         pdf: Pdf,
@@ -732,7 +766,7 @@ class PageInfo:
         else:
             self._has_vector = None  # i.e. "no information"
             self._has_text = None
-            self._images = None
+            self._images = []
 
         self._dpi = None
         if self._images:
@@ -749,7 +783,7 @@ class PageInfo:
 
     @property
     def has_text(self) -> bool:
-        return self._has_text
+        return bool(self._has_text)
 
     @property
     def has_corrupt_text(self) -> bool:
@@ -759,7 +793,7 @@ class PageInfo:
 
     @property
     def has_vector(self) -> bool:
-        return self._has_vector
+        return bool(self._has_vector)
 
     @property
     def width_inches(self) -> Decimal:

src/ocrmypdf/pdfinfo/layout.py

@@ -135,7 +135,7 @@ class LTStateAwareChar(LTChar):
         return self._text
 
     def __repr__(self):
-        return '<%s %s matrix=%s rendermode=%r font=%r adv=%s text=%r>' % (
+        return '<{} {} matrix={} rendermode={!r} font={!r} adv={} text={!r}>'.format(
             self.__class__.__name__,
             bbox2str(self.bbox),
             matrix2str(self.matrix),

src/ocrmypdf/pluginspec.py

@@ -5,7 +5,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
-from abc import ABC, abstractmethod, abstractstaticmethod
+from abc import ABC, abstractmethod
 from argparse import ArgumentParser, Namespace
 from collections import namedtuple
 from logging import Handler
@@ -197,7 +197,7 @@ def rasterize_pdf_page(
 
 
 @hookspec(firstresult=True)
-def filter_ocr_image(page: 'PageContext', image: 'Image') -> 'Image':
+def filter_ocr_image(page: 'PageContext', image: 'Image.Image') -> 'Image.Image':
     """Called to filter the image before it is sent to OCR.
 
     This is the image that OCR sees, not what the user sees when they view the
@@ -325,11 +325,13 @@ class OcrEngine(ABC):
     Tesseract OCR.
     """
 
-    @abstractstaticmethod
+    @staticmethod
+    @abstractmethod
     def version() -> str:
         """Returns the version of the OCR engine."""
 
-    @abstractstaticmethod
+    @staticmethod
+    @abstractmethod
     def creator_tag(options: Namespace) -> str:
         """Returns the creator tag to identify this software's role in creating the PDF.
 
@@ -349,24 +351,28 @@ class OcrEngine(ABC):
         to the user, usually in an error message.
         """
 
-    @abstractstaticmethod
+    @staticmethod
+    @abstractmethod
     def languages(options: Namespace) -> AbstractSet[str]:
         """Returns the set of all languages that are supported by the engine.
 
         Languages are typically given in 3-letter ISO 3166-1 codes, but actually
         can be any value understood by the OCR engine."""
 
-    @abstractstaticmethod
+    @staticmethod
+    @abstractmethod
     def get_orientation(input_file: Path, options: Namespace) -> OrientationConfidence:
         """Returns the orientation of the image."""
 
-    @abstractstaticmethod
+    @staticmethod
+    @abstractmethod
     def generate_hocr(
         input_file: Path, output_hocr: Path, output_text: Path, options: Namespace
     ) -> None:
         """Called to produce a hOCR file and sidecar text file."""
 
-    @abstractstaticmethod
+    @staticmethod
+    @abstractmethod
     def generate_pdf(
         input_file: Path, output_pdf: Path, output_text: Path, options: Namespace
     ) -> None:

tests/test_main.py

@@ -701,7 +701,7 @@ def test_sidecar_pagecount(resources, outpdf):
     pdfinfo = PdfInfo(resources / '3small.pdf')
     num_pages = len(pdfinfo)
 
-    with open(sidecar, 'r', encoding='utf-8') as f:
+    with open(sidecar, encoding='utf-8') as f:
         ocr_text = f.read()
 
     # There should a formfeed between each pair of pages, so the count of
@@ -722,7 +722,7 @@ def test_sidecar_nonempty(resources, outpdf):
         'tests/plugins/tesseract_cache.py',
     )
 
-    with open(sidecar, 'r', encoding='utf-8') as f:
+    with open(sidecar, encoding='utf-8') as f:
         ocr_text = f.read()
     assert 'the' in ocr_text
 

tests/test_page_numbers.py

@@ -50,10 +50,6 @@ def test_nonmonotonic_warning(caplog):
     assert 'out of order' in caplog.text
 
 
-def test_list_range():
-    assert _pages_from_ranges([0, 1, 2]) == {0, 1, 2}
-
-
 def test_limited_pages(resources, outpdf):
     multi = resources / 'multipage.pdf'
     ocrmypdf.ocr(

tests/test_pipeline.py

@@ -79,9 +79,7 @@ def test_dpi_needed(image, text, vector, result, rgb_image, outdir):
             # Input:
             ('', '', '', '', ''),
             # Output:
-            (
+            (((1, 5), None),),
-                ((1, 5), None),
-            ),
         ),
         (
             'no_empty_values',
@@ -147,4 +145,4 @@ def test_dpi_needed(image, text, vector, result, rgb_image, outdir):
     ),
 )
 def test_enumerate_compress_ranges(name, input, output):
-    assert output == tuple(_pipeline.enumerate_compress_ranges(input))
+    assert output == tuple(_pipeline.enumerate_compress_ranges(input))