v12.3.3 release notes

Closes #821

.coveragerc

@@ -1,24 +0,0 @@
-[paths]
-source =
-	src
-	*/site-packages
-
-[run]
-branch = true
-parallel = true
-concurrency =
-	thread
-	multiprocessing
-source =
-	src/ocrmypdf
-
-[report]
-exclude_lines =
-	pragma: no cover
-	def __repr__
-	raise AssertionError
-	raise NotImplementedError
-	if 0:
-	if False:
-	if __name__ == .__main__.:
-	if TYPE_CHECKING:

.docker/Dockerfile

@@ -1,17 +1,27 @@
 # OCRmyPDF
 #
-FROM ubuntu:20.04 as base
 
-FROM base as builder
+FROM ubuntu:21.04 as base
 
 ENV LANG=C.UTF-8
+ENV TZ=UTC
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  python3 \
+  libqpdf-dev \
+  zlib1g \
+  liblept5
+
+FROM base as builder
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
   build-essential autoconf automake libtool \
   libleptonica-dev \
   zlib1g-dev \
-  python3 \
+  python3-dev \
   python3-distutils \
+  libffi-dev \
   ca-certificates \
   curl \
   git
@@ -35,26 +45,15 @@ COPY . /app
 
 WORKDIR /app
 
-RUN pip3 install --no-cache-dir \
-  -r requirements/main.txt \
-  -r requirements/webservice.txt \
-  -r requirements/test.txt \
-  -r requirements/watcher.txt \
-  .
+RUN pip3 install --no-cache-dir .[test,webservice,watcher]
 
 FROM base
 
-ENV LANG=C.UTF-8
-
 RUN apt-get update && apt-get install -y --no-install-recommends \
   ghostscript \
   img2pdf \
-  liblept5 \
   libsm6 libxext6 libxrender-dev \
-  zlib1g \
   pngquant \
-  python3 \
-  qpdf \
   tesseract-ocr \
   tesseract-ocr-chi-sim \
   tesseract-ocr-deu \
@@ -62,7 +61,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   tesseract-ocr-fra \
   tesseract-ocr-por \
   tesseract-ocr-spa \
-  unpaper
+  unpaper \
+  && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 
@@ -76,6 +76,5 @@ COPY --from=builder /app/misc/watcher.py /app/
 COPY --from=builder /app/setup.cfg /app/setup.py /app/README.md /app/
 COPY --from=builder /app/requirements /app/requirements
 COPY --from=builder /app/tests /app/tests
-COPY --from=builder /app/src /app/src
 
 ENTRYPOINT ["/usr/local/bin/ocrmypdf"]

.github/ISSUE_TEMPLATE/1-general-issues.md

@@ -21,8 +21,12 @@ If applicable, add screenshots to help explain your problem.
 
 **System (please complete the following information):**
  - OS:
- - Python version: 
+ - Python version:
  - OCRmyPDF version:
 
+**Installation**
+How did you install OCRmyPDF? Did you install it from your operating system's
+package manager, or using pip?
+
 **Additional context**
 Add any other context about the problem here.

.github/ISSUE_TEMPLATE/3-feature_request.md

@@ -14,7 +14,14 @@ A clear and concise description of what the problem is. Ex. I'm always frustrate
 A clear and concise description of what you want to happen.
 
 **Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
+A clear and concise description of any alternative solutions or features you've considered. Please include the versions of OCRmyPDF and other supporting programs (Tesseract OCR, Ghostscript) - maybe an alternative already exists in a newer version.
+
+**Example file**
+If your issue concerns how OCRmyPDF processes certain files, and please provide an example file that helps illustrate how OCRmyPDF's output could be improve.
+
+Please provide an input file with no personal or confidential information. At your option you may [GPG-encrypt the file](https://github.com/jbarlow83/OCRmyPDF/wiki) for OCRmyPDF's author only.
+
+Links to files hosted elsewhere are perfectly acceptable. You could also look in ``tests/resources`` and see if any of those files reproduce your issue.
 
 **Additional context**
 Add any other context or screenshots about the feature request here.

.github/workflows/build.yml

@@ -0,0 +1,293 @@
+name: Test and deploy
+
+on:
+  push:
+    branches:
+      - master
+      - ci
+      - release/*
+    tags:
+      - v*
+    paths-ignore:
+      - README*
+  pull_request:
+
+jobs:
+  test_linux:
+    name: Test ${{ matrix.os }} with Python ${{ matrix.python }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-18.04
+            python: 3.6
+          - os: ubuntu-18.04
+            python: 3.7
+          - os: ubuntu-20.04
+            python: 3.8
+          - os: ubuntu-20.04
+            python: 3.9
+          - os: ubuntu-latest
+            python: 3.9
+          - os: ubuntu-latest
+            python: 3.9
+            tesseract5: true
+
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: ${{ matrix.python }}
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
+
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install Tesseract 5
+        if: matrix.tesseract5
+        run: |
+          sudo add-apt-repository ppa:alex-p/tesseract-ocr-devel
+
+      - name: Install common packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            curl \
+            ghostscript \
+            img2pdf \
+            libffi-dev \
+            liblept5 \
+            libsm6 libxext6 libxrender-dev \
+            pngquant \
+            poppler-utils \
+            tesseract-ocr \
+            tesseract-ocr-deu \
+            tesseract-ocr-eng \
+            unpaper \
+            zlib1g
+
+      - name: Install Ubuntu 18.04 packages
+        if: matrix.os == 'ubuntu-18.04'
+        run: |
+          sudo apt-get install -y --no-install-recommends \
+            libexempi3
+
+      - name: Install Ubuntu 20.04 packages
+        if: matrix.os == 'ubuntu-20.04'
+        run: |
+          sudo apt-get install -y --no-install-recommends \
+            libexempi8
+
+      - name: Install Python packages
+        run: |
+          python -m pip install .[test]
+
+      - name: Report versions
+        run: |
+          tesseract --version
+          gs --version
+          pngquant --version
+          unpaper --version
+          img2pdf --version
+
+      - name: Test
+        run: |
+          python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          files: ./coverage.xml
+          env_vars: OS,PYTHON
+
+  test_macos:
+    name: Test macOS
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-latest]
+        python: ["3.9"]
+
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: ${{ matrix.python }}
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
+
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install Homebrew deps
+        run: |
+          brew update
+          brew install \
+            exempi \
+            ghostscript \
+            jbig2enc \
+            leptonica \
+            openjpeg \
+            pngquant \
+            tesseract
+
+      - name: Install Python packages
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install .[test]
+
+      - name: Report versions
+        run: |
+          tesseract --version
+          gs --version
+          pngquant --version
+          img2pdf --version
+
+      - name: Test
+        run: |
+          python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          files: ./coverage.xml
+          env_vars: OS,PYTHON
+
+  test_windows:
+    name: Test Windows
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest]
+        python: ["3.9"]
+
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: ${{ matrix.python }}
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
+
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install system packages
+        run: |
+          choco install --yes --no-progress --pre tesseract
+          choco install --yes --no-progress ghostscript
+          choco install --yes --no-progress pngquant
+
+      - name: Install Python packages
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install .[test]
+
+      - name: Test
+        run: |
+          python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          files: ./coverage.xml
+          env_vars: OS,PYTHON
+
+  wheel_sdist_linux:
+    name: Build sdist and wheels
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
+
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: "3.6"
+
+      - name: Make wheels and sdist
+        run: |
+          python -m pip install --upgrade pip wheel
+          python setup.py sdist
+          python setup.py bdist_wheel
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: |
+            ./dist/*.whl
+            ./dist/*.tar.gz
+
+  upload_pypi:
+    name: Deploy artifacts to PyPI
+    needs: [wheel_sdist_linux, test_linux, test_macos, test_windows]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: artifact
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@master
+        with:
+          user: __token__
+          password: ${{ secrets.TOKEN_PYPI }}
+          # repository_url: https://test.pypi.org/legacy/
+
+  docker:
+    name: Build Docker images
+    needs: [wheel_sdist_linux, test_linux, test_macos, test_windows]
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    steps:
+      - name: Set image tag to release or branch
+        run: echo "DOCKER_IMAGE_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV
+
+      - name: If master, set to latest
+        run: echo 'DOCKER_IMAGE_TAG=latest' >> $GITHUB_ENV
+        if: env.DOCKER_IMAGE_TAG == 'master'
+
+      - name: Set Docker Hub repository to username
+        run: echo "DOCKER_REPOSITORY=jbarlow83" >> $GITHUB_ENV
+
+      - name: Set image name
+        run: echo "DOCKER_IMAGE_NAME=ocrmypdf" >> $GITHUB_ENV
+
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: jbarlow83
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Print image tag
+        run: echo "Building image ${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
+
+      - name: Build
+        run: |
+          docker buildx build \
+            --push \
+            --platform linux/arm64/v8,linux/amd64  \
+            --tag "${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" \
+            --file .docker/Dockerfile .

.gitignore

@@ -6,8 +6,9 @@
 !.gitattributes
 !.gitignore
 !.pre-commit-config.yaml
-!.readthedocs.yml
+!.readthedocs.yaml
 !.github/
+!.docker/
 
 # Dev scratch
 *.ipynb
@@ -24,6 +25,7 @@ venv*/
 /debug_tests.py
 *.traineddata
 /private
+/coverage.xml
 
 # Package building
 *.egg-info/

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.1.0
+    rev: v3.4.0
     hooks:
       - id: check-case-conflict
       - id: check-merge-conflict
@@ -12,12 +12,12 @@ repos:
     hooks:
       - id: seed-isort-config
   - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.0.5 # pick the isort version you'd like to use from https://github.com/pre-commit/mirrors-isort/releases
+    rev: v5.7.0 # pick the isort version you'd like to use from https://github.com/pre-commit/mirrors-isort/releases
     hooks:
       - id: isort
   - repo: https://github.com/psf/black
-    rev: 19.10b0
+    rev: 20.8b1
     hooks:
       - id: black
-        language_version: python3.8
+        language_version: python
         exclude: ^src/ocrmypdf/lib/_leptonica.py

.readthedocs.yaml

@@ -0,0 +1,22 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Optionally build your docs in additional formats such as PDF
+formats:
+  - pdf
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  version: 3.7
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs

.readthedocs.yml

@@ -1,10 +0,0 @@
-build:
-  image: latest
-
-python:
-  version: 3.6
-
-formats:
-  - pdf
-
-requirements_file: requirements/main.txt

README.md

@@ -1,6 +1,6 @@
 <img src="docs/images/logo.svg" width="240" alt="OCRmyPDF">
 
-[![Build Status][azure]](https://dev.azure.com/jim0585/ocrmypdf/_build/latest?definitionId=2&branchName=master) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs] ![Python versions][pyversions]
+[![Build Status](https://github.com/jbarlow83/OCRmyPDF/actions/workflows/build.yml/badge.svg)](https://github.com/jbarlow83/OCRmyPDF/actions/workflows/build.yml) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs] ![Python versions][pyversions]
 
 [azure]: https://dev.azure.com/jim0585/ocrmypdf/_apis/build/status/jbarlow83.OCRmyPDF?branchName=master
 [travis]: https://travis-ci.org/jbarlow83/OCRmyPDF.svg?branch=master "Travis build status"
@@ -57,25 +57,17 @@ I searched the web for a free command line tool to OCR PDF files: I found many,
 
 ## Installation
 
-Linux, Windows, macOS and FreeBSD are supported. Docker images are also available.
+Linux, Windows, macOS and FreeBSD are supported. Docker images are also available, for both x64 and ARM.
 
-Users of Debian 9 or later or Ubuntu 16.10 or later may simply
-
-```bash
-apt-get install ocrmypdf
-```
-
-and users of Fedora 29 or later may simply
-
-```bash
-dnf install ocrmypdf
-```
-
-and Homebrew users (macOS, Linux, Windows Subsystem for Linux) may simply
-
-```bash
-brew install ocrmypdf
-```
+| Operating system              | Install command               |
+| ----------------------------- | ------------------------------|
+| Debian, Ubuntu                | ``apt install ocrmypdf``      |
+| Windows Subsystem for Linux   | ``apt install ocrmypdf``      |
+| Fedora                        | ``dnf install ocrmypdf``      |
+| macOS                         | ``brew install ocrmypdf``     |
+| LinuxBrew                     | ``brew install ocrmypdf``     |
+| FreeBSD                       | ``pkg install py37-ocrmypdf`` |
+| Conda                         | ``conda install ocrmypdf``    |
 
 For everyone else, [see our documentation](https://ocrmypdf.readthedocs.io/en/latest/installation.html) for installation steps.
 
@@ -92,10 +84,18 @@ apt-get install tesseract-ocr-chi-sim  # Example: Install Chinese Simplified lan
 
 # Arch Linux users
 pacman -S tesseract-data-eng tesseract-data-deu # Example: Install the English and German language packs
+
+# brew macOS users
+brew install tesseract-lang
 ```
 
 You can then pass the `-l LANG` argument to OCRmyPDF to give a hint as to what languages it should search for. Multiple languages can be requested.
 
+OCRmyPDF supports Tesseract 4.0 and the beta versions of Tesseract 5.0. It will
+automatically use whichever version it finds first on the `PATH` environment
+variable. On Windows, if `PATH` does not provide a Tesseract binary, we use
+the highest version number that is installed according to the Windows Registry.
+
 ## Documentation and support
 
 Once OCRmyPDF is installed, the built-in help which explains the command syntax and options can be accessed via:
@@ -120,6 +120,7 @@ In addition to the required Python version (3.6+), OCRmyPDF requires external pr
 - [heise Open Source, 09/2014: Texterkennung mit OCRmyPDF](https://heise.de/-2356670)
 - [heise Durchsuchbare PDF-Dokumente mit OCRmyPDF erstellen](https://www.heise.de/ratgeber/Durchsuchbare-PDF-Dokumente-mit-OCRmyPDF-erstellen-4607592.html)
 - [Excellent Utilities: OCRmyPDF](https://www.linuxlinks.com/excellent-utilities-ocrmypdf-add-ocr-text-layer-scanned-pdfs/)
+- [LinuxUser Texterkennung mit OCRmyPDF und Scanbd automatisieren](https://www.linux-community.de/ausgaben/linuxuser/2021/06/texterkennung-mit-ocrmypdf-und-scanbd-automatisieren/)
 
 ## Business enquiries
 

azure-pipelines.yml

@@ -1,249 +0,0 @@
-trigger:
-  tags:
-    include:
-      - v*
-  branches:
-    include:
-      - "*"
-    exclude:
-      - "travis"
-
-stages:
-  - stage: "Test"
-    jobs:
-      - job: Windows
-        pool:
-          vmImage: "vs2017-win2016"
-        strategy:
-          matrix:
-            Python36:
-              python.version: "3.6"
-            Python37:
-              python.version: "3.7"
-            Python38:
-              python.version: "3.8"
-        steps:
-          - task: UsePythonVersion@0
-            inputs:
-              versionSpec: "$(python.version)"
-          - pwsh: |
-              choco install --yes --no-progress --pre tesseract
-              choco install --yes --no-progress python3
-              choco install --yes --no-progress ghostscript
-              choco install --yes --no-progress pngquant
-            displayName: "Install system packages"
-          - pwsh: |
-              refreshenv
-              python -m pip install --upgrade pip wheel
-              python -m pip install -r requirements/main.txt -r requirements/test.txt .
-            displayName: "Install Python packages"
-          - pwsh: |
-              refreshenv
-              $env:pathext += ';.py'
-              # -n auto helps Windows
-              python -m pytest -n auto --junitxml=test.xml --cov=ocrmypdf --cov-report=xml
-            displayName: "Test"
-          - task: PublishTestResults@2
-            inputs:
-              testResultsFiles: "test.xml"
-              testRunTitle: "$(Agent.OS) - $(Build.DefinitionName) - Python $(python.version)"
-            condition: succeededOrFailed()
-      - job: "Ubuntu_1804"
-        pool:
-          vmImage: "ubuntu-18.04"
-        strategy:
-          matrix:
-            Python36:
-              python.version: "3.6"
-            Python37:
-              python.version: "3.7"
-            Python38:
-              python.version: "3.8"
-        steps:
-          - task: UsePythonVersion@0
-            inputs:
-              versionSpec: "$(python.version)"
-          - bash: |
-              sudo apt-get update
-              sudo apt-get install -y --no-install-recommends \
-                python3-software-properties \
-                curl \
-                ghostscript \
-                img2pdf \
-                libexempi3 \
-                libffi-dev \
-                liblept5 \
-                libsm6 libxext6 libxrender-dev \
-                pngquant \
-                poppler-utils \
-                tesseract-ocr \
-                tesseract-ocr-deu \
-                tesseract-ocr-eng \
-                unpaper \
-                zlib1g
-            displayName: "Install system packages"
-          - bash: |
-              curl https://bootstrap.pypa.io/get-pip.py | python3
-              pip3 install -r requirements/main.txt -r requirements/test.txt .
-            displayName: "Install Python packages"
-          - bash: |
-              tesseract --version
-            displayName: "Record versions"
-          - bash: |
-              # -n auto is slower on Linux and breaks on Python 3.8
-              pytest -n0 --junitxml=test.xml --cov=ocrmypdf --cov-report=xml
-            displayName: "Test"
-          - task: PublishTestResults@2
-            inputs:
-              testResultsFiles: "test.xml"
-              testRunTitle: "$(Agent.OS) - $(Build.DefinitionName) - Python $(python.version)"
-            condition: succeededOrFailed()
-      - job: "Ubuntu_1604"
-        pool:
-          vmImage: "ubuntu-16.04"
-        strategy:
-          matrix:
-            Python36:
-              python.version: "3.6"
-        steps:
-          - task: UsePythonVersion@0
-            inputs:
-              versionSpec: "$(python.version)"
-          - bash: |
-              sudo apt-get update
-              sudo apt-get install -y --no-install-recommends \
-                software-properties-common
-              sudo add-apt-repository -y ppa:alex-p/tesseract-ocr
-              sudo apt-get update
-              sudo apt-get install -y --no-install-recommends \
-                ghostscript \
-                img2pdf \
-                libexempi3 \
-                libffi-dev \
-                liblept5 \
-                libsm6 libxext6 libxrender-dev \
-                pngquant \
-                poppler-utils \
-                tesseract-ocr \
-                tesseract-ocr-deu \
-                tesseract-ocr-eng \
-                unpaper \
-                zlib1g
-            displayName: "Install system packages"
-          - bash: |
-              curl https://bootstrap.pypa.io/get-pip.py | python3
-              pip3 install -r requirements/main.txt -r requirements/test.txt .
-            displayName: "Install Python packages"
-          - bash: |
-              tesseract --version
-            displayName: "Record versions"
-          - bash: |
-              # -n auto is slower on Linux and breaks on Python 3.8
-              pytest -n0 --junitxml=test.xml --cov=ocrmypdf --cov-report=xml
-            displayName: "Test"
-          - task: PublishTestResults@2
-            inputs:
-              testResultsFiles: "test.xml"
-              testRunTitle: "$(Agent.OS) - $(Build.DefinitionName) - Python $(python.version)"
-            condition: succeededOrFailed()
-      - job: "macOS_Mojave"
-        pool:
-          vmImage: "macos-10.14"
-        steps:
-          # https://github.com/actions/virtual-environments/issues/664
-          # - task: UsePythonVersion@0
-          #   inputs:
-          #     versionSpec: "$(python.version)"
-          - bash: |
-              brew update
-              brew unlink python@2
-              brew upgrade python
-              echo "Using Python `python3 --version`"
-            displayName: "Update brew and Python"
-          - bash: |
-              brew install \
-                exempi \
-                ghostscript \
-                jbig2enc \
-                leptonica \
-                openjpeg \
-                pngquant \
-                tesseract
-            displayName: "Install system packages"
-          - bash: |
-              pip3 install --upgrade pip
-              pip3 install -r requirements/main.txt -r requirements/test.txt .
-            displayName: "Install Python packages"
-          - bash: |
-              tesseract --version
-            displayName: "Record versions"
-          - bash: pytest -nauto --junitxml=test.xml --cov=ocrmypdf --cov-report=xml
-            displayName: "Test"
-          - task: PublishTestResults@2
-            inputs:
-              testResultsFiles: "test.xml"
-              testRunTitle: "$(Agent.OS) - $(Build.DefinitionName) - Python $(python.version)"
-            condition: succeededOrFailed()
-          - task: PublishCodeCoverageResults@1
-            inputs:
-              codeCoverageTool: Cobertura
-              summaryFileLocation: "$(System.DefaultWorkingDirectory)/**/coverage.xml"
-
-  - stage: "Artifacts"
-    jobs:
-      - job: "sdist_wheel"
-        pool:
-          vmImage: "ubuntu-18.04"
-        steps:
-          - task: UsePythonVersion@0
-            inputs:
-              versionSpec: "3.7"
-          - bash: |
-              python -m pip install --upgrade pip wheel
-              python setup.py sdist bdist_wheel
-          - publish: dist
-            artifact: sdist_wheel
-
-  - stage: "Deploy"
-    jobs:
-      - deployment: "PyPI"
-        pool:
-          vmImage: "ubuntu-18.04"
-        environment: "deploy"
-        strategy:
-          runOnce:
-            deploy:
-              steps:
-                - download: current
-                  artifact: sdist_wheel
-                - script: |
-                    mkdir -p dist
-                    mv $(Pipeline.Workspace)/sdist_wheel/* dist
-                  displayName: "Move dist files"
-                - task: UsePythonVersion@0
-                  inputs:
-                    versionSpec: "3.8"
-                    architecture: x64
-                - script: |
-                    pip install --upgrade twine
-                  displayName: "Generate artifacts"
-                - script: |
-                    cat <<FILE >.pypirc
-                    [distutils]
-                    index-servers =
-                        pypi
-
-                    [pypi]
-                    username: __token__
-                    password: $(TOKEN_PYPI)
-
-                    FILE
-                  displayName: "Generate PyPI auth file"
-                - script: |
-                    python -m twine upload --config-file .pypirc dist/*
-                  displayName: "Upload to PyPI"
-                  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
-                - script: |
-                    curl -X POST -d "token=$(TOKEN_RTD)" https://readthedocs.org/api/v2/webhook/pikepdf/39557/
-                  displayName: "Trigger ReadTheDocs"
-                  condition: and(succeeded(), or(startsWith(variables['Build.SourceBranch'], 'refs/tags/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/master')))

docs/advanced.rst

@@ -125,8 +125,7 @@ include:
 .. envvar:: OMP_THREAD_LIMIT
 
    Controls the number of threads Tesseract will use. OCRmyPDF will
-   manage this environment if it is not already set. (Currently, it will
-   set it to 1 because this gives the best results in testing.)
+   manage this environment variable if it is not already set.
 
 For example, if you have a development build of Tesseract don't wish to
 use the system installation, you can launch OCRmyPDF as follows:
@@ -229,8 +228,8 @@ preprocessing is specified, then the image layer is a new PDF.
 Unlike ``sandwich`` this renderer is implemented within OCRmyPDF; anyone
 looking to customize how OCR is presented should look here. A major
 disadvantage of this renderer is it not capable of correctly handling
-text outside the Latin alphabet. Pull requests to improve the situation
-are welcome.
+text outside the Latin alphabet (specifically, it supports the ISO 8859-1
+character). Pull requests to improve the situation are welcome.
 
 Currently, this renderer has the best compatibility with Mozilla's
 PDF.js viewer.
@@ -315,7 +314,7 @@ message is:
 .. code-block:: none
 
     Temporary working files retained at:
-    /tmp/com.github.ocrmypdf.u20wpz07
+    /tmp/ocrmypdf.io.u20wpz07
 
 The organization of this folder is an implementation detail and subject
 to change between releases. However the general organization is that

docs/api.rst

@@ -20,7 +20,8 @@ and largely have the same functions.
 
     import ocrmypdf
 
-    ocrmypdf.ocr('input.pdf', 'output.pdf', deskew=True)
+    if __name__ == '__main__':  # To ensure correct behavior on Windows and macOS
+        ocrmypdf.ocr('input.pdf', 'output.pdf', deskew=True)
 
 With a few exceptions, all of the command line arguments are available
 and may be passed as equivalent keywords.
@@ -35,33 +36,37 @@ The :func:`ocrmypdf.ocr` function runs OCRmyPDF similar to command line
 execution. To do this, it will:
 
 - create a monitoring thread
-- create worker processes (forking itself)
-- manage the signal flags of worker processes
+- create worker processes (on Linux, forking itself; on Windows and macOS, by
+  spawning)
+- manage the signal flags of its worker processes
 - execute other subprocesses (forking and executing other programs)
 
 The Python process that calls ``ocrmypdf.ocr()`` must be sufficiently
-privileged to perform these actions. If it is not, ``ocrmypdf()`` will
-fail.
+privileged to perform these actions.
 
-There is no currently no option to manage how jobs are scheduled other
+There currently is no option to manage how jobs are scheduled other
 than the argument ``jobs=`` which will limit the number of worker
 processes.
 
-Forking a child process to call ``ocrmypdf.ocr()`` is suggested. That
+Creating a child process to call ``ocrmypdf.ocr()`` is suggested. That
 way your application will survive and remain interactive even if
-OCRmyPDF does not.
+OCRmyPDF fails for any reason.
 
 Programs that call ``ocrmypdf.ocr()`` should also install a SIGBUS signal
 handler (except on Windows), to raise an exception if access to a memory
 mapped file fails. OCRmyPDF may use memory mapping.
 
+``ocrmypdf.ocr()`` will take a threading lock to prevent multiple runs of itself
+in the same Python interpreter process. This is not thread-safe, because of how
+OCRmyPDF's plugins and Python's library import system work. If you need to parallelize
+OCRmyPDF, use processes.
+
 .. warning::
 
-    On Windows, the script that calls ``ocrmypdf.ocr()`` must be protected
-    by an "ifmain" guard (``if __name__ == '__main__'``) or you must use
-    ``ocrmypdf.ocr(...use_threads=True)``. If you do not take at least one
-    of these steps, Windows process semantics will prevent OCRmyPDF from working
-    correctly.
+    On Windows and macOS, the script that calls ``ocrmypdf.ocr()`` must be
+    protected by an "ifmain" guard (``if __name__ == '__main__'``). If you do
+    not take at least one of these steps, process semantics will prevent
+    OCRmyPDF from working correctly.
 
 Logging
 -------

docs/apiref.rst

@@ -5,6 +5,15 @@ API Reference
 This page summarizes the rest of the public API. Generally speaking this
 should mainly of interest to plugin developers.
 
+ocrmypdf
+========
+
+.. autoclass:: ocrmypdf.PageContext
+    :members:
+
+.. autoclass:: ocrmypdf.PdfContext
+    :members:
+
 ocrmypdf.exceptions
 ===================
 
@@ -17,6 +26,9 @@ ocrmypdf.helpers
 
 .. automodule:: ocrmypdf.helpers
     :members:
+    :noindex: deprecated
+
+    .. autodecorator:: deprecated
 
 ocrmypdf.hocrtransform
 ======================

docs/batch.rst

@@ -111,7 +111,7 @@ Users may need to customize the script to meet their requirements.
 
 .. code-block:: bash
 
-    pip3 install -r requirements/watcher.txt
+    pip3 install ocrmypdf[watcher]
 
     env OCR_INPUT_DIRECTORY=/mnt/input-pdfs \
         OCR_OUTPUT_DIRECTORY=/mnt/output-pdfs \
@@ -127,7 +127,7 @@ Users may need to customize the script to meet their requirements.
     "OCR_ON_SUCCESS_DELETE", "This will delete the input file if the exit code is 0 (OK)"
     "OCR_OUTPUT_DIRECTORY_YEAR_MONTH", "This will place files in the output in ``{output}/{year}/{month}/{filename}``"
     "OCR_DESKEW", "Apply deskew to crooked input PDFs"
-    "OCR_JSON_SETTINGS", "A JSON string specifying any other arguments for ``ocrmypdf.ocr``, e.g. ``'OCR_JSON_SETTINGS={"rotate_pages": true}'``.
+    "OCR_JSON_SETTINGS", "A JSON string specifying any other arguments for ``ocrmypdf.ocr``, e.g. ``'OCR_JSON_SETTINGS={""rotate_pages"": true}'``."
     "OCR_POLL_NEW_FILE_SECONDS", "Polling interval"
     "OCR_LOGLEVEL", "Level of log messages to report"
 
@@ -202,13 +202,6 @@ Alternatives
 -  `Watchman <https://facebook.github.io/watchman/>`__ is a more
    powerful alternative to ``watchmedo``.
 
-AWS Lambda is not viable
-------------------------
-
-AWS Lambda and its equivalents have low limits on execution time and payload
-size, relative to OCRmyPDF's needs. As of this writing, the request/response
-payload for AWS Lambda was 6 MB, which means many PDFs will not fit.
-
 macOS Automator
 ===============
 

docs/conf.py

@@ -31,9 +31,11 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.napoleon']
+extensions = ['sphinx.ext.napoleon', 'sphinx_issues']
 
+# Extension settings
 napoleon_use_rtype = False
+issues_github_path = "jbarlow83/OCRmyPDF"
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

docs/cookbook.rst

@@ -58,9 +58,11 @@ portrait pages.
 You can increase (decrease) the parameter ``--rotate-pages-threshold``
 to make page rotation more (less) aggressive. The threshold number is the ratio
 of how confidence the OCR engine is that the document image should be changed,
-compared to kept the same. A value of ``15.0`` is the default, and is fairly
-conservative. A value of ``2.0`` will produce more rotations, and more false
-positives.
+compared to kept the same. The default value is quite conservative; on some files
+it may not attempt rotations at all unless it is very confident that the current
+rotation is wrong. A lower value of ``2.0`` will produce more rotations, and
+more false positives. Run with ``-v1`` to see the confidence level for each
+page to see if there may be a better value for your files.
 
 If the page is "just a little off horizontal", like a crooked picture,
 then you want ``--deskew``. ``--rotate-pages`` is for when the cardinal

docs/docker.rst

@@ -1,3 +1,5 @@
+.. _docker:
+
 =====================
 OCRmyPDF Docker image
 =====================
@@ -101,16 +103,41 @@ Adding languages to the Docker image
 By default the Docker image includes English, German, Simplified Chinese,
 French, Portuguese and Spanish, the most popular languages for OCRmyPDF
 users based on feedback. You may add other languages by creating a new
-Dockerfile based on the public one:
+Dockerfile based on the public one.
 
 .. code-block:: dockerfile
 
    FROM jbarlow83/ocrmypdf
 
-   # Add French
-   RUN apt install tesseract-ocr-fra
+   # Example: add Italian
+   RUN apt install tesseract-ocr-ita
 
-You can also copy training data to ``/usr/share/tesseract-ocr/<tesseract version>/tessdata``.
+To install language packs (training data) such as the 
+`tessdata_best <https://github.com/tesseract-ocr/tessdata_best>`_ suite or
+custom data, you first need to determine the version of Tesseract data files, which
+may differ from the Tesseract program version. Use this command to determine the data
+file version:
+
+.. code-block:: bash
+
+   docker run -i --rm --entrypoint /bin/ls jbarlow83/ocrmypdf /usr/share/tesseract-ocr
+
+As of 2021, the data file version is probably ``4.00``.
+
+You can then add new data with either a Dockerfile:
+
+.. code-block:: dockerfile
+
+   FROM jbarlow83/ocrmypdf
+
+   # Example: add a tessdata_best file
+   COPY chi_tra_vert.traineddata /usr/share/tesseract-ocr/<data version>/tessdata/
+
+Alternately, you can copy training data into a Docker container as follows:
+
+.. code-block:: bash
+
+   docker cp mycustomtraining.traineddata name_of_container:/usr/share/tesseract-ocr/<tesseract version>/tessdata/
 
 Executing the test suite
 ========================

docs/images/logo-square.svg

@@ -0,0 +1,233 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:serif="http://www.serif.com/"
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="503"
+   height="503"
+   viewBox="0 0 503 503"
+   version="1.1"
+   xml:space="preserve"
+   style="clip-rule:evenodd;fill-rule:evenodd;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:1.5"
+   id="svg270"
+   sodipodi:docname="logo-square.svg"
+   inkscape:export-filename="/home/jb/src/ocrmypdf/docs/images/logo-square.png"
+   inkscape:export-xdpi="96"
+   inkscape:export-ydpi="96"
+   inkscape:version="1.0.2 (e86c870879, 2021-01-15)"><metadata
+   id="metadata276"><rdf:RDF><cc:Work
+       rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
+         rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
+   id="defs274" /><sodipodi:namedview
+   pagecolor="#ffffff"
+   bordercolor="#666666"
+   borderopacity="1"
+   objecttolerance="10"
+   gridtolerance="10"
+   guidetolerance="10"
+   inkscape:pageopacity="0"
+   inkscape:pageshadow="2"
+   inkscape:window-width="2396"
+   inkscape:window-height="1691"
+   id="namedview272"
+   showgrid="false"
+   lock-margins="false"
+   inkscape:zoom="2.0079523"
+   inkscape:cx="251.5"
+   inkscape:cy="193.18317"
+   inkscape:window-x="26"
+   inkscape:window-y="23"
+   inkscape:window-maximized="0"
+   inkscape:current-layer="svg270" />
+    <g
+   id="svg"
+   transform="matrix(0.965977,0,0,0.807602,0,138.43572)">
+        <rect
+   x="0"
+   y="0"
+   width="520"
+   height="280"
+   style="fill:#ffffff"
+   id="rect188" />
+        <g
+   transform="matrix(1.03522,0,0,1.23823,-69.7528,-83.422)"
+   id="g267">
+            <g
+   transform="translate(243.977,20.0703)"
+   id="g218">
+                <g
+   id="Page">
+                    <g
+   transform="matrix(0.961773,0,0,1.05962,6.19811,-3.01071)"
+   id="g192">
+                        <path
+   d="m 328.5,97.682 c 0,-1.217 -0.517,-2.386 -1.444,-3.264 -7.03,-6.66 -37.614,-35.638 -44.828,-42.474 -0.977,-0.925 -2.327,-1.448 -3.738,-1.448 -13.997,0 -90.407,0 -111.151,0 -2.871,0 -5.198,2.113 -5.198,4.718 0,27.837 0,170.351 0,198.186 0,2.605 2.327,4.717 5.197,4.717 24.904,0 131.821,0 156.2,0 2.74,0 4.962,-2.016 4.962,-4.504 0,-24.345 0,-139.717 0,-155.931 z"
+   style="fill:#fdfdfd;stroke:#333333;stroke-width:3.95px"
+   id="path190" />
+                    </g>
+                    <g
+   id="Dog-ear"
+   serif:id="Dog ear"
+   transform="translate(-4,2)">
+                        <path
+   d="m 277.072,48.496 v 45.352 c 0,1.324 0.526,2.593 1.462,3.529 0.936,0.936 2.205,1.462 3.529,1.462 12.485,0 44.078,0 44.078,0"
+   style="fill:#f5f5f5;stroke:#333333;stroke-width:4px"
+   id="path194" />
+                    </g>
+                </g>
+                <g
+   transform="translate(-29.6816,-0.395178)"
+   id="g216">
+                    <g
+   transform="matrix(1.00243,0,0,1.11818,-144.72,-8.80181)"
+   id="g200">
+                        <path
+   d="m 465.73,119.654 c 0,-2.049 -1.856,-3.713 -4.142,-3.713 H 310.259 c -2.286,0 -4.142,1.664 -4.142,3.713 v 63.454 c 0,2.049 1.856,3.713 4.142,3.713 h 151.329 c 2.286,0 4.142,-1.664 4.142,-3.713 z"
+   style="fill:#f80000;stroke:#ffffff;stroke-width:3.77px"
+   id="path198" />
+                    </g>
+                    <g
+   transform="matrix(1.24571,0,0,1.35864,116.812,84.3924)"
+   id="g214">
+                        <g
+   transform="matrix(64,0,0,64,42.1437,77.6203)"
+   id="g204">
+                            <path
+   d="m 0.084,0 v -0.68 h 0.213 c 0.074,0 0.137,0.017 0.19,0.05 0.053,0.034 0.079,0.09 0.079,0.168 0,0.077 -0.028,0.134 -0.085,0.17 -0.057,0.037 -0.121,0.055 -0.193,0.055 H 0.213 V 0 Z m 0.209,-0.572 h -0.08 v 0.228 h 0.082 c 0.039,0 0.07,-0.009 0.094,-0.027 0.024,-0.017 0.037,-0.045 0.04,-0.083 0,-0.044 -0.012,-0.075 -0.036,-0.092 -0.024,-0.017 -0.057,-0.026 -0.1,-0.026 z"
+   style="fill:#ffffff;fill-rule:nonzero"
+   id="path202" />
+                        </g>
+                        <g
+   transform="matrix(64,0,0,64,79.7117,77.6203)"
+   id="g208">
+                            <path
+   d="M 0.332,0 H 0.084 v -0.68 h 0.252 c 0.105,0 0.182,0.032 0.233,0.095 0.051,0.063 0.076,0.144 0.076,0.241 0,0.105 -0.027,0.189 -0.082,0.251 C 0.508,-0.031 0.431,0 0.332,0 Z M 0.337,-0.57 H 0.213 v 0.461 H 0.33 c 0.055,0 0.099,-0.018 0.132,-0.054 C 0.495,-0.199 0.511,-0.259 0.511,-0.344 0.511,-0.415 0.497,-0.47 0.469,-0.51 0.441,-0.55 0.397,-0.57 0.337,-0.57 Z"
+   style="fill:#ffffff;fill-rule:nonzero"
+   id="path206" />
+                        </g>
+                        <g
+   transform="matrix(64,0,0,64,123.424,77.6203)"
+   id="g212">
+                            <path
+   d="M 0.405,-0.288 H 0.213 V 0 H 0.084 v -0.68 h 0.385 l 0.02,0.102 H 0.213 v 0.189 h 0.173 z"
+   style="fill:#ffffff;fill-rule:nonzero"
+   id="path210" />
+                        </g>
+                    </g>
+                </g>
+            </g>
+            <g
+   transform="matrix(1,0,0,1.52217,67.3796,10.7507)"
+   id="g222">
+                <rect
+   x="23.500999"
+   y="81.300003"
+   width="162.30499"
+   height="61.77"
+   style="fill:#b4d5ff"
+   id="rect220" />
+            </g>
+            <g
+   transform="matrix(0.967536,0,0,0.961535,5.90498,47.9703)"
+   id="g236">
+                <g
+   transform="matrix(90.4804,0,0,90.4804,82.6698,167.705)"
+   id="g226">
+                    <path
+   d="m 0.057,-0.337 c 0,-0.105 0.027,-0.19 0.082,-0.257 0.055,-0.066 0.132,-0.1 0.231,-0.102 0.107,0 0.186,0.034 0.237,0.103 0.051,0.069 0.077,0.152 0.077,0.249 0,0.105 -0.027,0.191 -0.082,0.258 -0.055,0.067 -0.133,0.1 -0.232,0.1 C 0.264,0.014 0.185,-0.02 0.134,-0.089 0.083,-0.157 0.057,-0.24 0.057,-0.337 Z m 0.135,-0.001 c 0,0.071 0.014,0.13 0.043,0.175 0.029,0.045 0.073,0.068 0.134,0.068 0.055,0 0.098,-0.02 0.131,-0.061 0.033,-0.041 0.049,-0.103 0.049,-0.188 0,-0.071 -0.014,-0.129 -0.043,-0.174 -0.029,-0.045 -0.073,-0.068 -0.134,-0.068 -0.053,0 -0.097,0.022 -0.13,0.067 -0.033,0.045 -0.05,0.105 -0.05,0.181 z"
+   style="fill:#333333;fill-rule:nonzero"
+   id="path224" />
+                </g>
+                <g
+   transform="matrix(90.4804,0,0,90.4804,147.906,167.705)"
+   id="g230">
+                    <path
+   d="M 0.505,-0.557 C 0.473,-0.567 0.448,-0.574 0.429,-0.579 0.41,-0.583 0.388,-0.585 0.361,-0.585 c -0.054,0 -0.096,0.022 -0.125,0.066 -0.029,0.044 -0.044,0.104 -0.044,0.181 0,0.066 0.012,0.123 0.037,0.171 0.025,0.048 0.066,0.072 0.124,0.072 0.029,0 0.056,-0.003 0.081,-0.009 0.025,-0.006 0.047,-0.013 0.068,-0.022 L 0.551,-0.03 C 0.525,-0.017 0.494,-0.006 0.457,0.002 0.42,0.01 0.388,0.014 0.36,0.014 0.254,0.014 0.177,-0.02 0.129,-0.088 0.081,-0.156 0.057,-0.239 0.057,-0.337 c 0,-0.105 0.027,-0.19 0.08,-0.257 0.053,-0.067 0.129,-0.1 0.228,-0.1 0.02,0 0.048,0.003 0.083,0.01 0.035,0.007 0.068,0.018 0.097,0.034 z"
+   style="fill:#333333;fill-rule:nonzero"
+   id="path228" />
+                </g>
+                <g
+   transform="matrix(90.4804,0,0,90.4804,199.751,167.705)"
+   id="g234">
+                    <path
+   d="m 0.293,-0.572 h -0.08 v 0.208 h 0.082 c 0.039,0 0.071,-0.008 0.096,-0.024 0.025,-0.015 0.038,-0.041 0.038,-0.077 0,-0.038 -0.012,-0.065 -0.036,-0.082 -0.024,-0.017 -0.057,-0.025 -0.1,-0.025 z M 0.479,0 0.335,-0.26 C 0.328,-0.259 0.32,-0.259 0.312,-0.259 0.304,-0.258 0.296,-0.258 0.288,-0.258 H 0.213 V 0 H 0.084 v -0.68 h 0.213 c 0.074,0 0.137,0.017 0.19,0.051 0.053,0.034 0.079,0.087 0.079,0.158 0,0.042 -0.011,0.078 -0.032,0.108 -0.022,0.031 -0.05,0.054 -0.084,0.071 L 0.617,0 Z"
+   style="fill:#333333;fill-rule:nonzero"
+   id="path232" />
+                </g>
+            </g>
+            <g
+   transform="matrix(0.916882,0,0,1,121.475,-32.6535)"
+   id="g246">
+                <g
+   transform="matrix(86.953,0,0,86.953,152.996,241.878)"
+   id="g240">
+                    <path
+   d="M 0.479,-0.428 C 0.5,-0.451 0.527,-0.47 0.562,-0.484 c 0.034,-0.013 0.065,-0.02 0.092,-0.02 0.066,0 0.113,0.019 0.141,0.058 0.027,0.039 0.041,0.086 0.041,0.142 V 0 H 0.705 v -0.298 c 0,-0.031 -0.007,-0.054 -0.022,-0.071 -0.015,-0.016 -0.036,-0.024 -0.064,-0.024 -0.019,0 -0.038,0.005 -0.059,0.015 -0.021,0.01 -0.039,0.021 -0.056,0.034 0.001,0.007 0.001,0.013 0.002,0.02 0.001,0.007 0.001,0.013 0.001,0.02 V 0 H 0.376 v -0.298 c 0,-0.031 -0.007,-0.054 -0.022,-0.071 -0.015,-0.016 -0.036,-0.024 -0.063,-0.024 -0.017,0 -0.033,0.003 -0.05,0.01 -0.017,0.007 -0.034,0.016 -0.049,0.027 V 0 H 0.062 V -0.485 H 0.13 l 0.032,0.044 c 0.022,-0.02 0.049,-0.035 0.08,-0.047 0.031,-0.011 0.058,-0.016 0.083,-0.016 0.038,0 0.07,0.007 0.095,0.02 0.025,0.014 0.045,0.033 0.059,0.056 z"
+   style="fill:#333333;fill-rule:nonzero"
+   id="path238" />
+                </g>
+                <g
+   transform="matrix(86.953,0,0,86.953,228.906,241.878)"
+   id="g244">
+                    <path
+   d="M 0.156,0.023 0.179,-0.034 0.006,-0.467 0.14,-0.485 0.252,-0.191 0.358,-0.485 H 0.495 L 0.278,0.064 C 0.263,0.103 0.236,0.137 0.197,0.165 0.158,0.193 0.118,0.212 0.075,0.222 L 0.029,0.115 C 0.052,0.105 0.077,0.093 0.104,0.079 0.13,0.064 0.147,0.046 0.156,0.023 Z"
+   style="fill:#333333;fill-rule:nonzero"
+   id="path242" />
+                </g>
+            </g>
+            <g
+   id="Selectors"
+   transform="matrix(0.965977,0,0,0.807602,67.3796,67.3718)">
+                <g
+   id="Right-selector"
+   serif:id="Right selector">
+                    <g
+   transform="matrix(1.03522,0,0,1.23823,2.07044,0)"
+   id="g250">
+                        <path
+   d="M 185.806,161.156 V 67.132"
+   style="fill:none;stroke:#4c9fff;stroke-width:4px;stroke-linecap:butt"
+   id="path248" />
+                    </g>
+                    <g
+   transform="matrix(1.03522,0,0,1.23823,161.788,169.469)"
+   id="g254">
+                        <circle
+   cx="31.523001"
+   cy="34.313999"
+   r="10.021"
+   style="fill:#4c9fff;stroke:#4c9fff;stroke-width:4px;stroke-linecap:butt"
+   id="circle252" />
+                    </g>
+                </g>
+                <g
+   id="Left-selector"
+   serif:id="Left selector">
+                    <g
+   transform="matrix(1.03522,0,0,1.23823,-170.092,0)"
+   id="g259">
+                        <path
+   d="M 185.806,161.156 V 67.132"
+   style="fill:none;stroke:#4c9fff;stroke-width:4px;stroke-linecap:butt"
+   id="path257" />
+                    </g>
+                    <g
+   transform="matrix(1.03522,0,0,1.23823,-10.3742,28.2274)"
+   id="g263">
+                        <circle
+   cx="31.523001"
+   cy="34.313999"
+   r="10.021"
+   style="fill:#4c9fff;stroke:#4c9fff;stroke-width:4px;stroke-linecap:butt"
+   id="circle261" />
+                    </g>
+                </g>
+            </g>
+        </g>
+    </g>
+</svg>

docs/index.rst

@@ -1,7 +1,7 @@
 OCRmyPDF documentation
 ======================
 
-OCRmyPDF adds an optical charcter recognition (OCR) text layer to scanned PDF
+OCRmyPDF adds an optical character recognition (OCR) text layer to scanned PDF
 files, allowing them to be searched.
 
 PDF is the best format for storing and exchanging scanned documents.

docs/installation.rst

@@ -12,19 +12,21 @@ system/platform. This version may be out of date, however.
 
 These platforms have one-liner installs:
 
-+-----------------------------+-------------------------------+
-| Debian, Ubuntu              | ``apt install ocrmypdf``      |
-+-----------------------------+-------------------------------+
-| Windows Subsystem for Linux | ``apt install ocrmypdf``      |
-+-----------------------------+-------------------------------+
-| Fedora                      | ``dnf install ocrmypdf``      |
-+-----------------------------+-------------------------------+
-| macOS                       | ``brew install ocrmypdf``     |
-+-----------------------------+-------------------------------+
-| LinuxBrew                   | ``brew install ocrmypdf``     |
-+-----------------------------+-------------------------------+
-| FreeBSD                     | ``pkg install py37-ocrmypdf`` |
-+-----------------------------+-------------------------------+
++-------------------------------+-------------------------------+
+| Debian, Ubuntu                | ``apt install ocrmypdf``      |
++-------------------------------+-------------------------------+
+| Windows Subsystem for Linux   | ``apt install ocrmypdf``      |
++-------------------------------+-------------------------------+
+| Fedora                        | ``dnf install ocrmypdf``      |
++-------------------------------+-------------------------------+
+| macOS                         | ``brew install ocrmypdf``     |
++-------------------------------+-------------------------------+
+| LinuxBrew                     | ``brew install ocrmypdf``     |
++-------------------------------+-------------------------------+
+| FreeBSD                       | ``pkg install py38-ocrmypdf`` |
++-------------------------------+-------------------------------+
+| Conda (WSL, macOS, Linux)     | ``conda install ocrmypdf``    |
++-------------------------------+-------------------------------+
 
 More detailed procedures are outlined below. If you want to do a manual
 install, or install a more recent version than your platform provides, read on.
@@ -54,6 +56,9 @@ Debian and Ubuntu 18.04 or newer
 .. |ubu-2004| image:: https://repology.org/badge/version-for-repo/ubuntu_20_04/ocrmypdf.svg
     :alt: Ubuntu 20.04 LTS
 
+.. |ubu-2010| image:: https://repology.org/badge/version-for-repo/ubuntu_20_10/ocrmypdf.svg
+    :alt: Ubuntu 20.10
+
 +-----------------------------------------------+
 | **OCRmyPDF versions in Debian & Ubuntu**      |
 +-----------------------------------------------+
@@ -61,7 +66,7 @@ Debian and Ubuntu 18.04 or newer
 +-----------------------------------------------+
 | |deb-stable| |deb-testing| |deb-unstable|     |
 +-----------------------------------------------+
-| |ubu-1804| |ubu-2004|                         |
+| |ubu-1804| |ubu-2004| |ubu-2010|              |
 +-----------------------------------------------+
 
 Users of Debian 9 ("stretch") or later, or Ubuntu 18.04 or later, including users
@@ -90,15 +95,15 @@ For full details on version availability for your platform, check the
    automatically detect it (specifically the ``jbig2`` binary) on the
    ``PATH``. To add JBIG2 encoding, see :ref:`jbig2`.
 
-Fedora 29 or newer
-------------------
-
-.. |fedora-31| image:: https://repology.org/badge/version-for-repo/fedora_31/ocrmypdf.svg
-    :alt: Fedora 31
+Fedora
+------
 
 .. |fedora-32| image:: https://repology.org/badge/version-for-repo/fedora_32/ocrmypdf.svg
     :alt: Fedora 32
 
+.. |fedora-33| image:: https://repology.org/badge/version-for-repo/fedora_33/ocrmypdf.svg
+    :alt: Fedora 33
+
 .. |fedora-rawhide| image:: https://repology.org/badge/version-for-repo/fedora_rawhide/ocrmypdf.svg
     :alt: Fedore Rawhide
 
@@ -107,7 +112,7 @@ Fedora 29 or newer
 +-----------------------------------------------+
 | |latest|                                      |
 +-----------------------------------------------+
-| |fedora-31| |fedora-32| |fedora-rawhide|      |
+| |fedora-32| |fedora-33| |fedora-rawhide|      |
 +-----------------------------------------------+
 
 Users of Fedora 29 or later may simply
@@ -158,7 +163,7 @@ To install ocrmypdf for the system:
 
 .. code-block:: bash
 
-    sudo pip3 install ocrmypdf
+    pip3 install ocrmypdf
 
 To install for the current user only:
 
@@ -355,7 +360,8 @@ To install OCRmyPDF for Alpine Linux:
 Mageia 7
 --------
 
-Install the following dependencies:
+There is no OS-level packaging available for Mageia, so you must install the
+dependencies:
 
 .. code-block:: bash
 
@@ -380,12 +386,16 @@ Install the following dependencies:
 
 To install ocrmypdf for the system:
 
+.. code-block:: bash
+
     # As root user
     pip3 install ocrmypdf
     ldconfig
 
 Or, to install for the current user only:
 
+.. code-block:: bash
+
     export PATH=$HOME/.local/bin:$PATH
     pip3 install --user ocrmypdf
 
@@ -454,7 +464,7 @@ Update Homebrew:
 
 Install or upgrade the required Homebrew packages, if any are missing.
 To do this, use ``brew edit ocrmypdf`` to obtain a recent list of Homebrew
-dependencies. You could also check the ``azure-pipelines.yml``.
+dependencies. You could also check the ``.workflows/build.yml``.
 
 This will include the English, French, German and Spanish language
 packs. If you need other languages you can optionally install them all:
@@ -495,10 +505,6 @@ Installing on Windows
 Native Windows
 --------------
 
-.. note::
-
-    It is easier to install OCRmyPDF on Windows Subsystem for Linux.
-
 .. note::
 
     Administrator privileges will be required for some of these steps.
@@ -509,31 +515,41 @@ You must install the following for Windows:
 * Tesseract 4.0 or later
 * Ghostscript 9.50 or later
 
-You can install these with the Chocolatey package manager:
+Using the `Chocolatey <https://chocolatey.org/>`_ package manager, install the
+following when running in an Administrator command prompt:
 
 * ``choco install python3``
 * ``choco install --pre tesseract``
 * ``choco install ghostscript``
+* ``choco install pngquant`` (optional)
 
-Also consider adding:
+The commands above will install Python 3.x (latest version), Tesseract, Ghostscript
+and pngquant. Chocolatey may also need to install the Windows Visual C++ Runtime
+DLLs or other Windows patches, and may require a reboot.
 
-* ``choco install pngquant``
-
-Windows 10 64-bit and 64-bit versions of applications are recommended. Earlier
-versions of Windows and 32-bit versions of these programs are not tested, and not
-supported at this time.
-
-OCRmyPDF will check for Tesseract-OCR and Ghostscript in your Program Files folder.
-If they are in some other location, you may need to modify the ``PATH``
-environment variable so Tesseract, Ghostscript, and other any optional executables can
-be found. You can enter it in the command line or
-`follow these directions <https://www.computerhope.com/issues/ch000549.htm#dospath>`_
-to make the change persistent and system-wide.
-
-You may then use pip to install ocrmypdf:
+You may then use ``pip`` to install ocrmypdf. (This can performed by a user or
+Administrator.):
 
 * ``pip install ocrmypdf``
 
+Chocolatey automatically selects appropriate versions of these applications. If you
+are installing them manually, please install 64-bit versions of all applications for
+64-bit Windows, or 32-bit versions of all applications for 32-bit Windows. Mixing
+the "bitness" of these programs will lead to errors.
+
+OCRmyPDF will check the Windows Registry and standard locations in your Program Files
+for third party software it needs (specifically, Tesseract and Ghostscript). To
+override the versions OCRmyPDF selects, you can modify the ``PATH`` environment
+variable. `Follow these directions <https://www.computerhope.com/issues/ch000549.htm#dospath>`_
+to change the PATH.
+
+.. warning::
+
+    As of early 2021, users have reported problems with the Microsoft Store version of
+    Python and OCRmyPDF. These issues affect many other third party Python packages.
+    Please download Python from Python.org or Chocolatey instead, and do not use the
+    Microsoft Store version.
+
 Windows Subsystem for Linux
 ---------------------------
 
@@ -604,7 +620,7 @@ However, the OCR-to-text-layer functionality is available.
 Docker
 ------
 
-You can also :ref:`Install the Docker <docker-install>` container on Windows. Ensure that
+You can also :ref:`Install the Docker <docker>` container on Windows. Ensure that
 your command prompt can run the docker "hello world" container.
 
 Installing on FreeBSD
@@ -619,7 +635,7 @@ versions likely work but have not been tested.
 
 .. code-block:: bash
 
-    pkg install py37-ocrmypdf
+    pkg install py38-ocrmypdf
 
 To install a more recent version, you could attempt to first install the system
 version with ``pkg``, then use ``pip install --user ocrmypdf``.
@@ -630,7 +646,7 @@ Installing the Docker image
 For some users, installing the Docker image will be easier than
 installing all of OCRmyPDF's dependencies.
 
-See `OCRmyPDF Docker Image <docker>`__ for more information.
+See :ref:`docker` for more information.
 
 Installing with Python pip
 ==========================
@@ -638,7 +654,22 @@ Installing with Python pip
 OCRmyPDF is delivered by PyPI because it is a convenient way to install
 the latest version. However, PyPI and ``pip`` cannot address the fact
 that ``ocrmypdf`` depends on certain non-Python system libraries and
-programs being instsalled.
+programs being installed.
+
+.. warning::
+
+    Debian and Ubuntu users: unfortunately, Debian and Ubuntu customize
+    Python in non-standard ways, and the nature of these customizations
+    varies from release to release. This can make for a frustrating
+    user experience. The instructions below work on almost all platforms that
+    have Python installed, except for Debian and Ubuntu, where you may need
+    to take additional steps. For best results on Debian and Ubuntu, use the
+    ``apt`` packages; or if these are too old, run
+    ``apt install python3-pip python3-venv``, create a virtual environment,
+    and install OCRmyPDF in that environment.
+
+    `See here for more inforation on Debian-Python issues
+    <https://gist.github.com/tiran/2dec9e03c6f901814f6d1e8dad09528e>`__.
 
 For best results, first install `your platform's
 version <https://repology.org/metapackage/ocrmypdf/versions>`__ of
@@ -781,8 +812,7 @@ To install all of the development and test requirements:
     python3 -m venv
     source venv/bin/activate
     cd OCRmyPDF
-    pip install -e .
-    pip install -r requirements/dev.txt -r requirements/test.txt
+    pip install -e .[test]
 
 To add JBIG2 encoding, see :ref:`jbig2`.
 

docs/introduction.rst

@@ -139,7 +139,7 @@ Limitations
 OCRmyPDF is limited by the Tesseract OCR engine. As such it experiences
 these limitations, as do any other programs that rely on Tesseract:
 
--  The OCR is not as accurate as commercial solutions such as Abbyy.
+-  The OCR is not as accurate as commercial OCR solutions.
 -  It is not capable of recognizing handwriting.
 -  It may find gibberish and report this as OCR output.
 -  If a document contains languages outside of those given in the
@@ -207,8 +207,9 @@ consider one of these similar open source programs:
 
 -  pdf2pdfocr
 -  pdfsandwich
--  pypdfocr
--  pdfbeads
+
+Ghostscript recently added three "pdfocr" output devices. They work by
+rasterizing all content and converting all pages to a single colour space.
 
 Web front-ends
 ==============

docs/jbig2.rst

@@ -9,11 +9,11 @@ encoding was patented for a long time. All known JBIG2 US patents have
 expired as of 2017, but it is possible that unknown patents exist.
 
 JBIG2 encoding is recommended for OCRmyPDF and is used to losslessly
-create smaller PDFs. If JBIG2 encoding not available, lower quality
+create smaller PDFs. If JBIG2 encoding is not available, lower quality
 encodings will be used.
 
 JBIG2 decoding is not patented and is performed automatically by most
-PDF viewers. It is widely supported has been part of the PDF
+PDF viewers. It is widely supported and has been part of the PDF
 specification since 2001.
 
 On macOS, Homebrew packages jbig2enc and OCRmyPDF includes it by
@@ -37,7 +37,7 @@ Lossy mode JBIG2
 
 OCRmyPDF provides lossy mode JBIG2 as an advanced feature. Users should
 `review the technical concerns with JBIG2 in lossy
-mode <https://abbyy.technology/en:kb:tip:jbig2_compression_and_ocr>`__
+mode <https://en.wikipedia.org/wiki/JBIG2#Disadvantages>`__
 and decide if this feature is acceptable for their use case.
 
 JBIG2 lossy mode does achieve higher compression ratios than any other

docs/languages.rst

@@ -12,9 +12,9 @@ languages <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.
 Languages are identified by standardized three-letter codes (called ISO 639-2 Alpha-3).
 Tesseract's documentation also lists the three-letter code for your language.
 Some are anglicized, e.g. Spanish is ``spa`` rather than ``esp``, while others
-are not, e.g. German is ``deu``.
+are not, e.g. German is ``deu`` and French is ``fra``.
 
-After you have installed a language pack, you can use it ``ocrmypdf -l <language>``,
+After you have installed a language pack, you can use it with ``ocrmypdf -l <language>``,
 for example ``ocrmypdf -l spa``. For multilingual documents, you can specify
 all languages to be expected, e.g. ``ocrmypdf -l eng+fra`` for English and French.
 English is assumed by default unless other language(s) are specified.
@@ -35,8 +35,8 @@ Debian and Ubuntu users
 
 You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as
 to what languages it should search for. Multiple languages can be
-requested using either ``-l eng+fre`` (English and French) or
-``-l eng -l fre``.
+requested using either ``-l eng+fra`` (English and French) or
+``-l eng -l fra``.
 
 Fedora users
 ============
@@ -51,8 +51,8 @@ Fedora users
 
 You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as
 to what languages it should search for. Multiple languages can be
-requested using either ``-l eng+fre`` (English and French) or
-``-l eng -l fre``.
+requested using either ``-l eng+fra`` (English and French) or
+``-l eng -l fra``.
 
 macOS users
 ===========
@@ -70,4 +70,7 @@ derived Docker image as
 Windows users
 =============
 
-The Tesseract installer provided by Chocolatey already includes 100 languages.
+The Tesseract installer provided by Chocolatey currently includes only English language. 
+To install other languages, download the respective language pack (``.traineddata`` file) 
+from https://github.com/tesseract-ocr/tessdata/ and place it in 
+``C:\\Program Files\\Tesseract-OCR\\tessdata`` (or wherever Tesseract OCR is installed).

docs/pdfsecurity.rst

@@ -19,7 +19,7 @@ PDF is a rich, complex file format. The official PDF 1.7 specification,
 ISO 32000:2008, is hundreds of pages long and references several annexes
 each of which are similar in length. PDFs can contain video, audio, XML,
 JavaScript and other programming, and forms. In some cases, they can
-open internet connections to pre-selected URLs. All of these possible
+open internet connections to pre-selected URLs. All of these are possible
 attack vectors.
 
 In short, PDFs `may contain
@@ -31,7 +31,7 @@ describes a high-paranoia method which allows potentially hostile PDFs
 to be viewed and rasterized safely in a disposable virtual machine. A
 trusted PDF created in this manner is converted to images and loses all
 information making it searchable and losing all compression. OCRmyPDF
-could be used restore searchability.
+could be used to restore searchability.
 
 How OCRmyPDF processes PDFs
 ===========================
@@ -41,8 +41,8 @@ layer. First, it runs all PDFs through
 `pikepdf <https://github.com/pikepdf/pikepdf>`__, a library based on
 `qpdf <https://github.com/qpdf/qpdf>`__, a program that repairs PDFs
 with syntax errors. This is done because, in the author's experience, a
-significant number of PDFs in the wild especially those created by
-scanners are not well-formed files. qpdf makes it more likely that
+significant number of PDFs in the wild, especially those created by
+scanners, are not well-formed files. qpdf makes it more likely that
 OCRmyPDF will succeed, but offers no security guarantees. qpdf is also
 used to split the PDF into single page PDFs.
 
@@ -66,8 +66,8 @@ service. OCRmyPDF relies on Ghostscript, and therefore, if deployed
 online one should be prepared to comply with Ghostscript's Affero GPL
 license, and any other licenses.
 
-Setting aside these concerns, a side effect of OCRmyPDF is it may
-incidentally sanitize PDFs that contain certain types of malware. It
+Setting aside these concerns, a side effect of OCRmyPDF is that it may
+incidentally sanitize PDFs containing certain types of malware. It
 repairs the PDF with pikepdf/libqpdf, which could correct malformed PDF
 structures that are part of an attack. When PDF/A output is selected
 (the default), the input PDF is partially reconstructed by Ghostscript.
@@ -83,7 +83,7 @@ Limiting CPU usage
 OCRmyPDF will attempt to use all available CPUs and storage, so
 executing ``nice ocrmypdf`` or limiting the number of jobs with the
 ``-j`` argument may ensure the server remains available. Another option
-would be run OCRmyPDF jobs inside a Docker container, a virtual machine,
+would be to run OCRmyPDF jobs inside a Docker container, a virtual machine,
 or a cloud instance, which can impose its own limits on CPU usage and be
 terminated "from orbit" if it fails to complete.
 
@@ -128,8 +128,9 @@ Commercial alternatives
 The author also provides professional services that include OCR and
 building databases around PDFs, and is happy to provide consultation.
 
-Abbyy Cloud OCR is a viable commercial alternative with a web services
-API.
+Abbyy Cloud OCR is viable commercial alternative with a web services
+API. Amazon Textract, Google Cloud Vision, and Microsoft Azure 
+Computer Vision provide advanced OCR but have less PDF rendering capability.
 
 Password protection, digital signatures and certification
 =========================================================

docs/pipeline.svg

@@ -1,392 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
- "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<!-- Generated by graphviz version 2.40.1 (20161225.0304)
- -->
-<!-- Title: Pipeline: Pages: 1 -->
-<svg width="1484pt" height="1277pt"
- viewBox="0.00 0.00 1484.00 1277.20" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 1273.1977)">
-<title>Pipeline:</title>
-<polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-1273.1977 1480,-1273.1977 1480,4 -4,4"/>
-<g id="clust1" class="cluster">
-<title>clustertasks</title>
-<polygon fill="none" stroke="#000000" points="8,-8 8,-1261.1977 1468,-1261.1977 1468,-8 8,-8"/>
-<text text-anchor="middle" x="738" y="-1233.1977" font-family="Times,serif" font-size="30.00" fill="#ff3232">Pipeline:</text>
-</g>
-<!-- t0 -->
-<g id="node1" class="node">
-<title>t0</title>
-<polygon fill="#efa03b" stroke="#000000" points="1215.7053,-1215.1977 1006.2947,-1215.1977 1002.2947,-1211.1977 1002.2947,-1179.1977 1211.7053,-1179.1977 1215.7053,-1183.1977 1215.7053,-1215.1977"/>
-<polyline fill="none" stroke="#000000" points="1211.7053,-1211.1977 1002.2947,-1211.1977 "/>
-<polyline fill="none" stroke="#000000" points="1211.7053,-1211.1977 1211.7053,-1179.1977 "/>
-<polyline fill="none" stroke="#000000" points="1211.7053,-1211.1977 1215.7053,-1215.1977 "/>
-<text text-anchor="middle" x="1109" y="-1191.1977" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.triage</text>
-</g>
-<!-- t1 -->
-<g id="node2" class="node">
-<title>t1</title>
-<polygon fill="#efa03b" stroke="#000000" points="1280.0433,-1157.1977 941.9567,-1157.1977 937.9567,-1153.1977 937.9567,-1121.1977 1276.0433,-1121.1977 1280.0433,-1125.1977 1280.0433,-1157.1977"/>
-<polyline fill="none" stroke="#000000" points="1276.0433,-1153.1977 937.9567,-1153.1977 "/>
-<polyline fill="none" stroke="#000000" points="1276.0433,-1153.1977 1276.0433,-1121.1977 "/>
-<polyline fill="none" stroke="#000000" points="1276.0433,-1153.1977 1280.0433,-1157.1977 "/>
-<text text-anchor="middle" x="1109" y="-1133.1977" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.repair_and_parse_pdf</text>
-</g>
-<!-- t0&#45;&gt;t1 -->
-<g id="edge1" class="edge">
-<title>t0&#45;&gt;t1</title>
-<path fill="none" stroke="#0044a0" d="M1109,-1179.1641C1109,-1175.4895 1109,-1171.5395 1109,-1167.604"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="1112.5001,-1167.4199 1109,-1157.4199 1105.5001,-1167.42 1112.5001,-1167.4199"/>
-</g>
-<!-- t2 -->
-<g id="node3" class="node">
-<title>t2</title>
-<polygon fill="#efa03b" stroke="#000000" points="1030.0926,-1078.2732 785,-1095.1731 539.9074,-1078.2732 540.1362,-1050.9285 1029.8638,-1050.9285 1030.0926,-1078.2732"/>
-<polygon fill="none" stroke="#000000" points="1034.1249,-1082.0021 785,-1099.1801 535.8751,-1082.0021 536.1685,-1046.931 1033.8315,-1046.931 1034.1249,-1082.0021"/>
-<text text-anchor="middle" x="785" y="-1064.7153" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.marker_pages</text>
-</g>
-<!-- t1&#45;&gt;t2 -->
-<g id="edge2" class="edge">
-<title>t1&#45;&gt;t2</title>
-<path fill="none" stroke="#0044a0" d="M1023.449,-1121.1152C984.5695,-1112.8974 938.0675,-1103.0685 896.4957,-1094.2816"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="897.1066,-1090.8335 886.5989,-1092.1898 895.6589,-1097.6822 897.1066,-1090.8335"/>
-</g>
-<!-- t16 -->
-<g id="node17" class="node">
-<title>t16</title>
-<polygon fill="#00cc66" stroke="#000000" points="1215.5059,-328.1861 942.4941,-328.1861 938.4941,-324.1861 938.4941,-292.1861 1211.5059,-292.1861 1215.5059,-296.1861 1215.5059,-328.1861"/>
-<polyline fill="none" stroke="#000000" points="1211.5059,-324.1861 938.4941,-324.1861 "/>
-<polyline fill="none" stroke="#000000" points="1211.5059,-324.1861 1211.5059,-292.1861 "/>
-<polyline fill="none" stroke="#000000" points="1211.5059,-324.1861 1215.5059,-328.1861 "/>
-<text text-anchor="middle" x="1077" y="-304.1861" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.weave_layers</text>
-</g>
-<!-- t1&#45;&gt;t16 -->
-<g id="edge24" class="edge">
-<title>t1&#45;&gt;t16</title>
-<path fill="none" stroke="#0044a0" d="M1097.2373,-1120.9576C1077.987,-1088.7184 1043.7317,-1019.7372 1063,-963.2656 1106.87,-834.6914 1482.8098,-573.3183 1409,-459.2656 1362.7595,-387.8137 1273.2613,-350.2799 1198.8305,-330.7528"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="1199.4555,-327.3005 1188.9015,-328.2301 1197.7316,-334.0849 1199.4555,-327.3005"/>
-</g>
-<!-- t17 -->
-<g id="node18" class="node">
-<title>t17</title>
-<polygon fill="#efa03b" stroke="#000000" points="1439.5156,-1009.7492 1076.4844,-1009.7492 1072.4844,-1005.7492 1072.4844,-973.7492 1435.5156,-973.7492 1439.5156,-977.7492 1439.5156,-1009.7492"/>
-<polyline fill="none" stroke="#000000" points="1435.5156,-1005.7492 1072.4844,-1005.7492 "/>
-<polyline fill="none" stroke="#000000" points="1435.5156,-1005.7492 1435.5156,-973.7492 "/>
-<polyline fill="none" stroke="#000000" points="1435.5156,-1005.7492 1439.5156,-1009.7492 "/>
-<text text-anchor="middle" x="1256" y="-985.7492" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.generate_postscript_stub</text>
-</g>
-<!-- t1&#45;&gt;t17 -->
-<g id="edge25" class="edge">
-<title>t1&#45;&gt;t17</title>
-<path fill="none" stroke="#0044a0" d="M1127.0251,-1121.1176C1153.0342,-1095.0292 1201.2007,-1046.7157 1230.7429,-1017.0834"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="1233.2458,-1019.5302 1237.8275,-1009.9772 1228.2885,-1014.5879 1233.2458,-1019.5302"/>
-</g>
-<!-- t18 -->
-<g id="node19" class="node">
-<title>t18</title>
-<polygon fill="#efa03b" stroke="#000000" points="1070.189,-214.4288 1263,-187.1099 1455.811,-214.4288 1455.631,-258.6317 1070.369,-258.6317 1070.189,-214.4288"/>
-<polygon fill="none" stroke="#000000" points="1066.1715,-210.9543 1263,-183.0662 1459.8285,-210.9543 1459.6181,-262.6353 1066.3819,-262.6353 1066.1715,-210.9543"/>
-<text text-anchor="middle" x="1263" y="-220.6461" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.metadata_fixup</text>
-</g>
-<!-- t1&#45;&gt;t18 -->
-<g id="edge28" class="edge">
-<title>t1&#45;&gt;t18</title>
-<path fill="none" stroke="#0044a0" d="M1241.1115,-1121.1192C1311.0076,-1105.2558 1393.5851,-1075.5895 1449,-1020.2329 1458.4922,-1010.7507 1459,-1005.1661 1459,-991.7492 1459,-991.7492 1459,-991.7492 1459,-393.7258 1459,-341.9727 1441.8377,-325.2216 1402,-292.1861 1390.9869,-283.0534 1378.4885,-274.871 1365.6104,-267.6415"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="1367.1079,-264.4723 1356.6476,-262.7888 1363.775,-270.6279 1367.1079,-264.4723"/>
-</g>
-<!-- t3 -->
-<g id="node4" class="node">
-<title>t3</title>
-<polygon fill="#efa03b" stroke="#000000" points="1015.5194,-999.3074 785,-1016.2081 554.4806,-999.3074 554.6958,-971.9616 1015.3042,-971.9616 1015.5194,-999.3074"/>
-<polygon fill="none" stroke="#000000" points="1019.5491,-1003.0204 785,-1020.2165 550.4509,-1003.0204 550.7267,-967.9639 1019.2733,-967.9639 1019.5491,-1003.0204"/>
-<text text-anchor="middle" x="785" y="-985.7492" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.ocr_or_skip</text>
-</g>
-<!-- t2&#45;&gt;t3 -->
-<g id="edge3" class="edge">
-<title>t2&#45;&gt;t3</title>
-<path fill="none" stroke="#0044a0" d="M785,-1046.5938C785,-1041.5635 785,-1036.139 785,-1030.7273"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="788.5001,-1030.4427 785,-1020.4427 781.5001,-1030.4428 788.5001,-1030.4427"/>
-</g>
-<!-- t4 -->
-<g id="node5" class="node">
-<title>t4</title>
-<polygon fill="#efa03b" stroke="#000000" points="889.3166,-941.2656 582.6834,-941.2656 578.6834,-937.2656 578.6834,-905.2656 885.3166,-905.2656 889.3166,-909.2656 889.3166,-941.2656"/>
-<polyline fill="none" stroke="#000000" points="885.3166,-937.2656 578.6834,-937.2656 "/>
-<polyline fill="none" stroke="#000000" points="885.3166,-937.2656 885.3166,-905.2656 "/>
-<polyline fill="none" stroke="#000000" points="885.3166,-937.2656 889.3166,-941.2656 "/>
-<text text-anchor="middle" x="734" y="-917.2656" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.rasterize_preview</text>
-</g>
-<!-- t3&#45;&gt;t4 -->
-<g id="edge4" class="edge">
-<title>t3&#45;&gt;t4</title>
-<path fill="none" stroke="#0044a0" d="M767.1627,-967.797C762.8396,-961.9919 758.1927,-955.7519 753.7858,-949.8343"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="756.3465,-947.4129 747.5666,-941.483 750.7323,-951.5939 756.3465,-947.4129"/>
-</g>
-<!-- t5 -->
-<g id="node6" class="node">
-<title>t5</title>
-<polygon fill="#efa03b" stroke="#000000" points="1028.0824,-883.2656 769.9176,-883.2656 765.9176,-879.2656 765.9176,-847.2656 1024.0824,-847.2656 1028.0824,-851.2656 1028.0824,-883.2656"/>
-<polyline fill="none" stroke="#000000" points="1024.0824,-879.2656 765.9176,-879.2656 "/>
-<polyline fill="none" stroke="#000000" points="1024.0824,-879.2656 1024.0824,-847.2656 "/>
-<polyline fill="none" stroke="#000000" points="1024.0824,-879.2656 1028.0824,-883.2656 "/>
-<text text-anchor="middle" x="897" y="-859.2656" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.orient_page</text>
-</g>
-<!-- t3&#45;&gt;t5 -->
-<g id="edge6" class="edge">
-<title>t3&#45;&gt;t5</title>
-<path fill="none" stroke="#0044a0" d="M869.1177,-967.8063C880.553,-961.0264 890.81,-952.3527 898,-941.2656 907.042,-927.3226 906.9457,-908.7053 904.5139,-893.387"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="907.914,-892.5383 902.5408,-883.4071 901.0469,-893.8961 907.914,-892.5383"/>
-</g>
-<!-- t4&#45;&gt;t5 -->
-<g id="edge5" class="edge">
-<title>t4&#45;&gt;t5</title>
-<path fill="none" stroke="#0044a0" d="M784.6807,-905.2319C801.0865,-899.3943 819.4462,-892.8614 836.4913,-886.7963"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="838.1026,-889.938 846.3506,-883.288 835.7559,-883.343 838.1026,-889.938"/>
-</g>
-<!-- t6 -->
-<g id="node7" class="node">
-<title>t6</title>
-<polygon fill="#efa03b" stroke="#000000" points="1064.7325,-825.2656 689.2675,-825.2656 685.2675,-821.2656 685.2675,-789.2656 1060.7325,-789.2656 1064.7325,-793.2656 1064.7325,-825.2656"/>
-<polyline fill="none" stroke="#000000" points="1060.7325,-821.2656 685.2675,-821.2656 "/>
-<polyline fill="none" stroke="#000000" points="1060.7325,-821.2656 1060.7325,-789.2656 "/>
-<polyline fill="none" stroke="#000000" points="1060.7325,-821.2656 1064.7325,-825.2656 "/>
-<text text-anchor="middle" x="875" y="-801.2656" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.rasterize_with_ghostscript</text>
-</g>
-<!-- t5&#45;&gt;t6 -->
-<g id="edge7" class="edge">
-<title>t5&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M890.1597,-847.2319C888.6839,-843.3412 887.0909,-839.1417 885.5115,-834.9776"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="888.731,-833.5965 881.9119,-825.4878 882.186,-836.0791 888.731,-833.5965"/>
-</g>
-<!-- t13 -->
-<g id="node16" class="node">
-<title>t13</title>
-<polygon fill="#00cc66" stroke="#000000" points="1077,-515.2656 754.4141,-487.2656 1077,-459.2656 1399.5859,-487.2656 1077,-515.2656"/>
-<text text-anchor="middle" x="1077" y="-481.2656" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.select_image_layer</text>
-</g>
-<!-- t5&#45;&gt;t13 -->
-<g id="edge20" class="edge">
-<title>t5&#45;&gt;t13</title>
-<path fill="none" stroke="#0044a0" d="M1000.2704,-847.241C1031.2359,-840.6873 1060.5415,-832.9862 1074,-825.2656 1114.0954,-802.2644 1145,-795.4899 1145,-749.2656 1145,-749.2656 1145,-749.2656 1145,-633.2656 1145,-590.368 1154.8687,-575.2844 1135,-537.2656 1131.571,-530.7041 1126.8893,-524.6314 1121.6622,-519.1451"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="1123.8989,-516.4405 1114.2626,-512.0367 1119.0494,-521.4885 1123.8989,-516.4405"/>
-</g>
-<!-- t7 -->
-<g id="node8" class="node">
-<title>t7</title>
-<polygon fill="#efa03b" stroke="#000000" points="856.0801,-767.2656 431.9199,-767.2656 427.9199,-763.2656 427.9199,-731.2656 852.0801,-731.2656 856.0801,-735.2656 856.0801,-767.2656"/>
-<polyline fill="none" stroke="#000000" points="852.0801,-763.2656 427.9199,-763.2656 "/>
-<polyline fill="none" stroke="#000000" points="852.0801,-763.2656 852.0801,-731.2656 "/>
-<polyline fill="none" stroke="#000000" points="852.0801,-763.2656 856.0801,-767.2656 "/>
-<text text-anchor="middle" x="642" y="-743.2656" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.preprocess_remove_background</text>
-</g>
-<!-- t6&#45;&gt;t7 -->
-<g id="edge8" class="edge">
-<title>t6&#45;&gt;t7</title>
-<path fill="none" stroke="#0044a0" d="M802.5546,-789.2319C777.7884,-783.067 749.9071,-776.1266 724.4138,-769.7806"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="724.95,-766.3073 714.4006,-767.288 723.259,-773.1 724.95,-766.3073"/>
-</g>
-<!-- t12 -->
-<g id="node15" class="node">
-<title>t12</title>
-<polygon fill="#efa03b" stroke="#000000" points="742,-593.2656 358.1874,-565.2656 742,-537.2656 1125.8126,-565.2656 742,-593.2656"/>
-<text text-anchor="middle" x="742" y="-559.2656" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.select_visible_page_image</text>
-</g>
-<!-- t6&#45;&gt;t12 -->
-<g id="edge18" class="edge">
-<title>t6&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M875.1443,-789.1129C874.6867,-773.2956 872.5831,-750.0189 865,-731.2656 844.2991,-680.0716 804.0126,-630.4165 775.1838,-598.9641"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="777.4937,-596.3085 768.1202,-591.3701 772.3682,-601.076 777.4937,-596.3085"/>
-</g>
-<!-- t8 -->
-<g id="node9" class="node">
-<title>t8</title>
-<polygon fill="#efa03b" stroke="#000000" points="732.6759,-709.2656 413.3241,-709.2656 409.3241,-705.2656 409.3241,-673.2656 728.6759,-673.2656 732.6759,-677.2656 732.6759,-709.2656"/>
-<polyline fill="none" stroke="#000000" points="728.6759,-705.2656 409.3241,-705.2656 "/>
-<polyline fill="none" stroke="#000000" points="728.6759,-705.2656 728.6759,-673.2656 "/>
-<polyline fill="none" stroke="#000000" points="728.6759,-705.2656 732.6759,-709.2656 "/>
-<text text-anchor="middle" x="571" y="-685.2656" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.preprocess_deskew</text>
-</g>
-<!-- t7&#45;&gt;t8 -->
-<g id="edge9" class="edge">
-<title>t7&#45;&gt;t8</title>
-<path fill="none" stroke="#0044a0" d="M619.9243,-731.2319C613.9709,-726.3685 607.4267,-721.0226 601.115,-715.8666"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="603.2653,-713.1037 593.3065,-709.4878 598.8367,-718.5248 603.2653,-713.1037"/>
-</g>
-<!-- t7&#45;&gt;t12 -->
-<g id="edge17" class="edge">
-<title>t7&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M717.0346,-731.1646C726.8757,-725.6884 735.7103,-718.5557 742,-709.2656 763.0291,-678.205 759.4083,-634.0888 752.8082,-602.8898"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="756.1481,-601.7976 750.487,-592.842 749.3278,-603.3733 756.1481,-601.7976"/>
-</g>
-<!-- t9 -->
-<g id="node10" class="node">
-<title>t9</title>
-<polygon fill="#efa03b" stroke="#000000" points="712.8828,-651.2656 411.1172,-651.2656 407.1172,-647.2656 407.1172,-615.2656 708.8828,-615.2656 712.8828,-619.2656 712.8828,-651.2656"/>
-<polyline fill="none" stroke="#000000" points="708.8828,-647.2656 407.1172,-647.2656 "/>
-<polyline fill="none" stroke="#000000" points="708.8828,-647.2656 708.8828,-615.2656 "/>
-<polyline fill="none" stroke="#000000" points="708.8828,-647.2656 712.8828,-651.2656 "/>
-<text text-anchor="middle" x="560" y="-627.2656" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.preprocess_clean</text>
-</g>
-<!-- t8&#45;&gt;t9 -->
-<g id="edge10" class="edge">
-<title>t8&#45;&gt;t9</title>
-<path fill="none" stroke="#0044a0" d="M567.5798,-673.2319C566.8624,-669.4493 566.0897,-665.3747 565.3216,-661.3247"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="568.7581,-660.6605 563.4559,-651.4878 561.8807,-661.9649 568.7581,-660.6605"/>
-</g>
-<!-- t8&#45;&gt;t12 -->
-<g id="edge16" class="edge">
-<title>t8&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M670.0045,-673.2618C693.2528,-667.2338 713.6919,-659.8307 722,-651.2656 734.1971,-638.6912 739.4901,-620.2051 741.6123,-603.5914"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="745.1041,-603.8427 742.5574,-593.5585 738.1349,-603.1862 745.1041,-603.8427"/>
-</g>
-<!-- t10 -->
-<g id="node11" class="node">
-<title>t10</title>
-<polygon fill="#efa03b" stroke="#000000" points="340.1094,-583.2656 35.8906,-583.2656 31.8906,-579.2656 31.8906,-547.2656 336.1094,-547.2656 340.1094,-551.2656 340.1094,-583.2656"/>
-<polyline fill="none" stroke="#000000" points="336.1094,-579.2656 31.8906,-579.2656 "/>
-<polyline fill="none" stroke="#000000" points="336.1094,-579.2656 336.1094,-547.2656 "/>
-<polyline fill="none" stroke="#000000" points="336.1094,-579.2656 340.1094,-583.2656 "/>
-<text text-anchor="middle" x="186" y="-559.2656" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.select_ocr_image</text>
-</g>
-<!-- t9&#45;&gt;t10 -->
-<g id="edge11" class="edge">
-<title>t9&#45;&gt;t10</title>
-<path fill="none" stroke="#0044a0" d="M460.7568,-615.2213C409.7279,-605.9434 347.4237,-594.6153 295.0537,-585.0935"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="295.5223,-581.6214 285.0575,-583.276 294.2701,-588.5085 295.5223,-581.6214"/>
-</g>
-<!-- t9&#45;&gt;t12 -->
-<g id="edge15" class="edge">
-<title>t9&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M608.2948,-615.2213C627.0226,-608.2242 648.8712,-600.0609 669.4134,-592.3859"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="670.9148,-595.5613 679.0573,-588.7826 668.4648,-589.004 670.9148,-595.5613"/>
-</g>
-<!-- t11 -->
-<g id="node12" class="node">
-<title>t11</title>
-<polygon fill="#00cc66" stroke="#000000" points="334.0996,-505.2656 19.9004,-505.2656 15.9004,-501.2656 15.9004,-469.2656 330.0996,-469.2656 334.0996,-473.2656 334.0996,-505.2656"/>
-<polyline fill="none" stroke="#000000" points="330.0996,-501.2656 15.9004,-501.2656 "/>
-<polyline fill="none" stroke="#000000" points="330.0996,-501.2656 330.0996,-469.2656 "/>
-<polyline fill="none" stroke="#000000" points="330.0996,-501.2656 334.0996,-505.2656 "/>
-<text text-anchor="middle" x="175" y="-481.2656" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.ocr_tesseract_hocr</text>
-</g>
-<!-- t10&#45;&gt;t11 -->
-<g id="edge12" class="edge">
-<title>t10&#45;&gt;t11</title>
-<path fill="none" stroke="#0044a0" d="M183.4487,-547.1748C182.1254,-537.7914 180.4788,-526.1154 178.9917,-515.5701"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="182.4378,-514.9417 177.5755,-505.5285 175.5064,-515.9193 182.4378,-514.9417"/>
-</g>
-<!-- t15 -->
-<g id="node14" class="node">
-<title>t15</title>
-<polygon fill="#ff69b4" stroke="#000000" points="736.2696,-505.2656 355.7304,-505.2656 351.7304,-501.2656 351.7304,-469.2656 732.2696,-469.2656 736.2696,-473.2656 736.2696,-505.2656"/>
-<polyline fill="none" stroke="#000000" points="732.2696,-501.2656 351.7304,-501.2656 "/>
-<polyline fill="none" stroke="#000000" points="732.2696,-501.2656 732.2696,-469.2656 "/>
-<polyline fill="none" stroke="#000000" points="732.2696,-501.2656 736.2696,-505.2656 "/>
-<text text-anchor="middle" x="544" y="-481.2656" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.ocr_tesseract_textonly_pdf</text>
-</g>
-<!-- t10&#45;&gt;t15 -->
-<g id="edge14" class="edge">
-<title>t10&#45;&gt;t15</title>
-<path fill="none" stroke="#0044a0" d="M269.0322,-547.1748C323.4947,-535.3086 394.7854,-519.776 451.1284,-507.5002"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="452.0829,-510.8744 461.1086,-505.3257 450.5927,-504.0348 452.0829,-510.8744"/>
-</g>
-<!-- t14 -->
-<g id="node13" class="node">
-<title>t14</title>
-<polygon fill="#00cc66" stroke="#000000" points="330.0433,-411.7258 21.9567,-411.7258 17.9567,-407.7258 17.9567,-375.7258 326.0433,-375.7258 330.0433,-379.7258 330.0433,-411.7258"/>
-<polyline fill="none" stroke="#000000" points="326.0433,-407.7258 17.9567,-407.7258 "/>
-<polyline fill="none" stroke="#000000" points="326.0433,-407.7258 326.0433,-375.7258 "/>
-<polyline fill="none" stroke="#000000" points="326.0433,-407.7258 330.0433,-411.7258 "/>
-<text text-anchor="middle" x="174" y="-387.7258" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.render_hocr_page</text>
-</g>
-<!-- t11&#45;&gt;t14 -->
-<g id="edge13" class="edge">
-<title>t11&#45;&gt;t14</title>
-<path fill="none" stroke="#0044a0" d="M174.8072,-469.2302C174.6642,-455.8584 174.4666,-437.3702 174.3027,-422.0357"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="177.8024,-421.987 174.1956,-412.025 170.8028,-422.0619 177.8024,-421.987"/>
-</g>
-<!-- t19 -->
-<g id="node22" class="node">
-<title>t19</title>
-<polygon fill="#efa03b" stroke="#000000" points="351.7205,-381.5085 545,-354.1897 738.2795,-381.5085 738.0991,-425.7112 351.9009,-425.7112 351.7205,-381.5085"/>
-<polygon fill="none" stroke="#000000" points="347.7033,-378.0329 545,-350.1463 742.2967,-378.0329 742.0858,-429.7148 347.9142,-429.7148 347.7033,-378.0329"/>
-<text text-anchor="middle" x="545" y="-387.7258" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.merge_sidecars</text>
-</g>
-<!-- t11&#45;&gt;t19 -->
-<g id="edge32" class="edge">
-<title>t11&#45;&gt;t19</title>
-<path fill="none" stroke="#0044a0" d="M246.3397,-469.2302C287.966,-458.7066 342.1269,-445.0142 392.8772,-432.184"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="393.8058,-435.5594 402.6429,-429.7151 392.0901,-428.773 393.8058,-435.5594"/>
-</g>
-<!-- t14&#45;&gt;t16 -->
-<g id="edge22" class="edge">
-<title>t14&#45;&gt;t16</title>
-<path fill="none" stroke="#0044a0" d="M230.6615,-375.7128C262.2207,-366.4581 302.4172,-355.9425 339,-350.1861 448.3957,-332.9725 745.7869,-320.7866 927.9215,-314.6743"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="928.248,-318.1655 938.1258,-314.3342 928.0148,-311.1693 928.248,-318.1655"/>
-</g>
-<!-- t15&#45;&gt;t16 -->
-<g id="edge21" class="edge">
-<title>t15&#45;&gt;t16</title>
-<path fill="none" stroke="#0044a0" d="M630.0636,-469.2083C667.5329,-460.6467 711.76,-449.5912 751,-437.2656 850.4985,-406.0121 963.1653,-359.5222 1026.586,-332.2894"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="1028.1566,-335.4238 1035.9553,-328.2523 1025.3866,-328.9951 1028.1566,-335.4238"/>
-</g>
-<!-- t15&#45;&gt;t19 -->
-<g id="edge31" class="edge">
-<title>t15&#45;&gt;t19</title>
-<path fill="none" stroke="#0044a0" d="M544.1928,-469.2302C544.2822,-460.8687 544.393,-450.5066 544.5035,-440.166"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="548.006,-439.9443 544.6132,-429.9075 541.0064,-439.8694 548.006,-439.9443"/>
-</g>
-<!-- t12&#45;&gt;t13 -->
-<g id="edge19" class="edge">
-<title>t12&#45;&gt;t13</title>
-<path fill="none" stroke="#0044a0" d="M833.9833,-543.8486C879.0975,-533.3444 933.4556,-520.6879 979.3116,-510.0109"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="980.3156,-513.3709 989.2614,-507.6943 978.7282,-506.5532 980.3156,-513.3709"/>
-</g>
-<!-- t13&#45;&gt;t16 -->
-<g id="edge23" class="edge">
-<title>t13&#45;&gt;t16</title>
-<path fill="none" stroke="#0044a0" d="M1077,-459.2068C1077,-426.2624 1077,-371.9187 1077,-338.7552"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="1080.5001,-338.3818 1077,-328.3819 1073.5001,-338.3819 1080.5001,-338.3818"/>
-</g>
-<!-- t16&#45;&gt;t18 -->
-<g id="edge27" class="edge">
-<title>t16&#45;&gt;t18</title>
-<path fill="none" stroke="#0044a0" d="M1117.3592,-292.0592C1133.8695,-284.6437 1153.6803,-275.7459 1173.2953,-266.9361"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="1174.8979,-270.0531 1182.586,-262.7632 1172.0299,-263.6676 1174.8979,-270.0531"/>
-</g>
-<!-- t17&#45;&gt;t18 -->
-<g id="edge26" class="edge">
-<title>t17&#45;&gt;t18</title>
-<path fill="none" stroke="#0044a0" d="M1326.2292,-973.7283C1377.8362,-955.768 1439,-922.4247 1439,-865.2656 1439,-865.2656 1439,-865.2656 1439,-393.7258 1439,-338.7092 1393.2056,-296.3464 1348.3356,-268.0433"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="1350.0403,-264.9831 1339.6842,-262.7465 1346.3852,-270.9531 1350.0403,-264.9831"/>
-</g>
-<!-- t20 -->
-<g id="node20" class="node">
-<title>t20</title>
-<polygon fill="#efa03b" stroke="#000000" points="1400.3184,-161.1062 1129.6816,-161.1062 1125.6816,-157.1062 1125.6816,-125.1062 1396.3184,-125.1062 1400.3184,-129.1062 1400.3184,-161.1062"/>
-<polyline fill="none" stroke="#000000" points="1396.3184,-157.1062 1125.6816,-157.1062 "/>
-<polyline fill="none" stroke="#000000" points="1396.3184,-157.1062 1396.3184,-125.1062 "/>
-<polyline fill="none" stroke="#000000" points="1396.3184,-157.1062 1400.3184,-161.1062 "/>
-<text text-anchor="middle" x="1263" y="-137.1062" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.optimize_pdf</text>
-</g>
-<!-- t18&#45;&gt;t20 -->
-<g id="edge29" class="edge">
-<title>t18&#45;&gt;t20</title>
-<path fill="none" stroke="#0044a0" d="M1263,-182.9188C1263,-179.129 1263,-175.3607 1263,-171.739"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="1266.5001,-171.4394 1263,-161.4394 1259.5001,-171.4394 1266.5001,-171.4394"/>
-</g>
-<!-- t21 -->
-<g id="node21" class="node">
-<title>t21</title>
-<polygon fill="#efa03b" stroke="#000000" points="1096.0652,-47.332 1263,-20.0049 1429.9348,-47.332 1429.779,-91.5483 1096.221,-91.5483 1096.0652,-47.332"/>
-<polygon fill="none" stroke="#000000" points="1092.0538,-43.9306 1263,-15.9468 1433.9462,-43.9306 1433.7643,-95.5532 1092.2357,-95.5532 1092.0538,-43.9306"/>
-<text text-anchor="middle" x="1263" y="-53.5531" font-family="Times,serif" font-size="20.00" fill="#000000">ocrmypdf.pipeline.copy_final</text>
-</g>
-<!-- t20&#45;&gt;t21 -->
-<g id="edge30" class="edge">
-<title>t20&#45;&gt;t21</title>
-<path fill="none" stroke="#0044a0" d="M1263,-124.9764C1263,-119.37 1263,-112.9163 1263,-106.2938"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="1266.5001,-105.8456 1263,-95.8457 1259.5001,-105.8457 1266.5001,-105.8456"/>
-</g>
-</g>
-</svg>

docs/plugins.rst

@@ -61,9 +61,38 @@ similar to ``pytest`` packages such as ``pytest-cov`` (the package) and
 
 .. note::
 
-    We strongly recommend plugin authors name their plugins with the prefix
+    We recommend plugin authors name their plugins with the prefix
     ``ocrmypdf-`` (for the package name on PyPI) and ``ocrmypdf_`` (for the
-    module), just like pytest plugins.
+    module), just like pytest plugins. At the same time, please make it clear
+    that your package is not official.
+
+Setuptools plugins
+==================
+
+You can also create a plugin that OCRmyPDF will always automatically load if both are
+installed in the same virtual environment, using a setuptools entrypoint.
+
+Your package's ``setup.py`` would need to contain the following, for a plugin
+named ``ocrmypdf-exampleplugin``:
+
+.. code-block:: python
+
+    # sample ./setup.py file
+    from setuptools import setup
+
+    setup(
+        name="ocrmypdf-exampleplugin",
+        packages=["exampleplugin"],
+        # the following makes a plugin available to pytest
+        entry_points={"ocrmypdf": ["exampleplugin = exampleplugin.pluginmodule"]},
+    )
+
+.. code-block:: ini
+
+    # equivalent setup.cfg
+    [options.entry_points]
+    ocrmypdf =
+        exampleplugin = exampleplugin.pluginmodule
 
 Plugin requirements
 ===================
@@ -130,6 +159,18 @@ Custom command line arguments
 
 .. autofunction:: ocrmypdf.pluginspec.check_options
 
+Execution and progress reporting
+--------------------------------
+
+.. autoclass: ocrmypdf.pluginspec.Executor
+    :members:
+
+.. autofunction:: ocrmypdf.pluginspec.get_logging_console
+
+.. autofunction:: ocrmypdf.pluginspec.get_executor
+
+.. autofunction:: ocrmypdf.pluginspec.get_progressbar_class
+
 Applying special behavior before processing
 -------------------------------------------
 
@@ -147,6 +188,8 @@ Modifying intermediate images
 
 .. autofunction:: ocrmypdf.pluginspec.filter_page_image
 
+.. autofunction:: ocrmypdf.pluginspec.filter_pdf_page
+
 OCR engine
 ----------
 

docs/release_notes.rst

@@ -12,6 +12,306 @@ may be unreliable. Use the API to depend on precise behavior.
 The public API may be useful in scripts that launch OCRmyPDF processes or that
 wish to use some of its features for working with PDFs.
 
+v12.3.3
+=======
+
+-  watcher.py: fixed interpretation of boolean env vars (:issue:`821`).
+-  Adjust CI scripts to test Tesseract 5 betas.
+-  Document our support for the Tesseract 5 betas.
+
+v12.3.2
+=======
+
+-  Indicate support for flask 2.x, watcher 2.x (:issue:`815, 816`).
+
+v12.3.1
+=======
+
+-  Fixed issue with selection of text when using the hOCR renderer (:issue:`813`).
+-  Fixed build errors with the Docker image by upgrading to a newer Ubuntu.
+   Also set the timezone of this image to UTC.
+
+v12.3.0
+=======
+
+-  Fixed a regression introduced in Pillow 8.3.0. Pillow no longer rounds DPI
+   for image resolutions. We now account for this (:issue:`802`).
+-  We no longer use some API calls that are deprecated in the latest versions of
+   pikepdf.
+-  Improved error message when a language is requested that doesn't look like a
+   typical ISO 639-2 code.
+-  Fixed some tests that attempted to symlink on Windows, breaking tests on a
+   Windows desktop but not usually on CI.
+-  Documentation fixes (thanks to @mara004)
+
+v12.2.0
+=======
+
+-  Fixed invalid Tesseract version number on Windows (:issue:`795`).
+-  Documentation tweaks. Documentation build now depends on sphinx-issues package.
+
+v12.1.0
+=======
+
+-  For security reasons we now require Pillow >= 8.2.x. (Older versions will continue
+   to work if upgrading is not an option.)
+-  The build system was reorganized to rely on ``setup.cfg`` instead of ``setup.py``.
+   All changes should work with previously supported versions of setuptools.
+-  The files in ``requirements/*`` are now considered deprecated but will be retained for v12.
+   Instead use ``pip install ocrmypdf[test]`` instead of ``requirements/test.txt``, etc.
+   These files will be removed in v13.
+
+v12.0.3
+=======
+
+-  Expand the list of languages supported by the hocr PDF renderer.
+   Several languages were previously considered not supported, particularly those
+   non-European languages that use the Latin alphabet.
+-  Fixed a case where the exception stack trace was suppressed in verbose mode.
+-  Improved documentation around commercial OCR.
+
+v12.0.2
+=======
+
+-  Fixed exception thrown when using ``--remove-background`` on files containing small
+   images (:issue:`769`).
+-  Improve documentation for description of adding language packs to the Docker image
+   and corrected name of French language pack.
+
+v12.0.1
+=======
+
+-  Fixed "invalid version number" for untagged tesseract versions (:issue:`770`).
+
+v12.0.0
+=======
+
+**Breaking changes**
+
+-  Due to recent security issues in pikepdf, Pillow and reportlab, we now require
+   newer versions of these libraries and some of their dependencies. (If necessary,
+   package maintainers may override these versions at their discretion; lower
+   versions will often work.)
+-  We now use the "LeaveColorUnchanged" color conversion strategy when directing
+   Ghostscript to create a PDF/A. Generally this is faster than performing a
+   color conversion, which is not always necessary.
+-  OCR text is now packaged in a Form XObject. This makes it easier to isolate
+   OCR from other document content. However, some poorly implemented PDF text
+   extraction algorithms may fail to detect the text.
+-  Many API functions have stricter parameter checking or expect keyword arguments
+   were they previously did not.
+-  Some deprecated functions in ``ocrmypdf.optimize`` were removed.
+-  The ``ocrmypdf.leptonica`` module is now deprecated, due to difficulties with
+   the current strategy of ABI binding on newer platforms like Apple Silicon.
+   It will be removed and replaced, either by repackaging Leptonica as an
+   independent library using or using a different image processing library.
+-  Continuous integration moved to GitHub Actions.
+-  We no longer depend on ``pytest_helpers_namespace`` for testing.
+
+**New features**
+
+-  New plugin hook: ``get_progressbar_class``, for progress reporting,
+   allowing developers to replace the standard console progress bar with some
+   other mechanism, such as updating a GUI progress bar.
+-  New plugin hook: ``get_executor``, for replacing the concurrency model.
+   This is primarily to support execution on AWS Lambda, which does not support
+   standard Python ``multiprocessing`` due to its lack of shared memory.
+-  New plugin hook: ``get_logging_console``, for replacing the standard
+   way OCRmyPDF outputs its messages.
+-  New plugin hook: ``filter_pdf_page``, for modifying individual PDF
+   pages produced by OCRmyPDF.
+-  OCRmyPDF now runs on nonstandard execution environments that do not have
+   interprocess semaphores, such as AWS Lambda and Android Termux. If the environment
+   does not have semaphores, OCRmyPDF will automatically select an alternate
+   process executor that does not use semaphores.
+-  Continuous integration moved to GitHub Actions.
+-  We now generate an ARM64-compatible Docker image alongside the x64 image.
+   Thanks to @andkrause for doing most of the work in a pull request several months
+   ago, which we were finally able to integrate now. Also thanks to @0x326 for
+   review comments.
+
+**Fixes**
+
+-  Fixed a possible deadlock on attempting to flush ``sys.stderr`` when older
+   versions of Leptonica are in use.
+-  Some worker processes inherited resources from their parents such as log
+   handlers that may have also lead to deadlocks. These resources are now released.
+-  Improvements to test coverage.
+-  Removed vestiges of support for Tesseract versions older than 4.0.0-beta1 (
+   which ships with Ubuntu 18.04).
+-  OCRmyPDF can now parse all of Tesseract version numbers, since several
+   schemes have been in use.
+-  Fixed an issue with parsing PDFs that contain images drawn at a scale of 0. (:issue:`761`)
+-  Removed a frequently repeated message about disabling mmap.
+
+v11.7.3
+=======
+
+-  Exclude CCITT Group 3 images from being optimized. Some libraries
+   OCRmyPDF uses do not seem to handle this obscure compression format properly.
+   You may get errors or possible corrupted output images without this fix.
+
+v11.7.2
+=======
+
+-  Updated pinned versions in main.txt, primarily to upgrade Pillow to 8.1.2, due
+   to recently disclosed security vulnerabilities in that software.
+-  The ``--sidecar`` parameter now causes an exception if set to the same file as
+   the input or output PDF.
+
+v11.7.1
+=======
+
+-  Some exceptions while attempting image optimization were only logged at the debug
+   level, causing them to be suppressed. These errors are now logged appropriately.
+-  Improved the error message related to ``--unpaper-args``.
+-  Updated documentation to mention the new conda distribution.
+
+v11.7.0
+=======
+
+-  We now support using ``--sidecar`` in conjunction with ``--pages``; these arguments
+   used to be mutually exclusive. (:issue:`735`)
+-  Fixed a possible issue with PDF/A-1b generation. Acrobat complained that our PDFs use
+   object streams. More robust PDF/A validators like veraPDF don't consider this a
+   problem, but we'll honor Acrobat's objection from here on. This may increase file
+   size of PDF/A-1b files. PDF/A-2b files will not be affected.
+
+v11.6.2
+=======
+
+-  Fixed a regression where the wrong page orientation would be produced when using
+   arguments such as ``--deskew --rotate-pages`` (:issue:`730`).
+
+v11.6.1
+=======
+
+-  Fixed an issue with attempting optimize unusually narrow-width images by excluding
+   these images from optimization (:issue:`732`).
+-  Remove an obsolete compatibility shim for a version of pikepdf that is no longer
+   supported.
+
+v11.6.0
+=======
+
+-  OCRmyPDF will now automatically register plugins from the same virtual environment
+   with an appropriate setuptools entrypoint.
+-  Refactor the plugin manager to remove unnecessary complications and make plugin
+   registration more automatic.
+-  ``PageContext`` and ``PdfContext`` are now formally part of the API, as they
+   should have been, since they were part of ``ocrmypdf.pluginspec``.
+
+v11.5.0
+=======
+
+-  Fixed an issue where the output page size might differ by a fractional amount
+   due to rounding, when ``--force-ocr`` was used and the page contained objects
+   with multiple resolutions.
+-  When determining the resolution at which to rasterize a page, we now consider
+   printed text on the page as requiring a higher resolution. This fixes issues
+   with certain pages being rendered with unacceptably low resolution text, but
+   may increase output file sizes in some workflows where low resolution text
+   is acceptable.
+-  Added a workaround to fix an exception that occurs when trying to
+   ``import ocrmypdf.leptonica`` on Apple ARM silicon (or potentially, other
+   platforms that do not permit write+executable memory).
+
+v11.4.5
+=======
+
+-  Fixed an issue where files may not be closed when the API is used.
+-  Improved ``setup.cfg`` with better settings for test coverage.
+
+v11.4.4
+=======
+
+-  Fixed ``AttributeError: 'NoneType' object has no attribute 'userunit'`` (:issue:`700`),
+   related to OCRmyPDF not properly forwarded an error message from pdfminer.six.
+-  Adjusted typing of some arguments.
+-  ``ocrmypdf.ocr`` now takes a ``threading.Lock`` for reasons outlined in the
+   documentation.
+
+v11.4.3
+=======
+
+-  Removed a redundant debug message.
+-  Test suite now asserts that most patched functions are called when they should be.
+-  Test suite now skips a test that fails on two particular versions of piekpdf.
+
+v11.4.2
+=======
+
+-  Fixed support for Cygwin, hopefully.
+-  watcher.py: Fixed an issue with the OCR_LOGLEVEL not being interpreted.
+
+v11.4.1
+=======
+
+-  Fixed an issue where invalid pages ranges passed using the ``pages`` argument,
+   such as "1-0" would cause unhandled exceptions.
+-  Accepted a user-contributed to the Synology demo script in misc/synology.py.
+-  Clarified documentation about change of temporary file location ``ocrmypdf.io``.
+-  Fixed Python wheel tag which was incorrectly set to py35 even though we long
+   since dropped support for Python 3.5.
+
+v11.4.0
+=======
+
+-  When looking for Tesseract and Ghostscript, we now check the Windows Registry to
+   see if their installers registered the location of their executables. This should
+   help Windows users who have installed these programs to non-standard
+   locations.
+-  We now report on the progress of PDF/A conversion, since this operation is
+   sometimes slow.
+-  Improved command line completions.
+-  The prefix of the temporary folder OCRmyPDF creates has been changed from
+   ``com.github.ocrmypdf`` to ``ocrmypdf.io``. Scripts that chose to depend on this
+   prefix may need to be adjusted. (This has always been an implementation detail so is
+   not considered part of the semantic versioning "contract".)
+-  Fixed :issue:`692`, where a particular file with malformed fonts would flood an
+   internal message cue by generating so many debug messages.
+-  Fixed an exception on processing hOCR files with no page record. Tesseract
+   is not known to generate such files.
+
+v11.3.4
+=======
+
+-  Fixed an error message 'called readLinearizationData for file that is not
+   linearized' that may occur when pikepdf 2.1.0 is used. (Upgrading to pikepdf
+   2.1.1 also fixes the issue.)
+-  File watcher now automatically includes ``.PDF`` in addition to ``.pdf`` to
+   better support case sensitive file systems.
+-  Some documentation and comment improvements.
+
+v11.3.3
+=======
+
+-  If unpaper outputs non-UTF-8 data, quietly fix this rather than choke on the
+   conversion. (Possibly addresses :issue:`671`.)
+
+v11.3.2
+=======
+
+-  Explicitly require pikepdf 2.0.0 or newer when running on Python 3.9. (There are
+   concerns about the stability of pybind11 2.5.x with Python 3.9, which is used in
+   pikepdf 1.x.)
+-  Fixed another issue related to page rotation.
+-  Fixed an issue where image marked as image masks were not properly considered
+   as optimization candidates.
+-  On some systems, unpaper seems to be unable to process the PNGs we offer it
+   as input. We now convert the input to PNM format, which unpaper always accepts.
+   Fixes :issue:`665` and :issue:`667`.
+-  DPI sent to unpaper is now rounded to a more reasonable number of decimal digits.
+-  Debug and error messages from unpaper were being suppressed.
+-  Some documentation tweaks.
+
+v11.3.1
+=======
+
+-  Declare support for new versions: pdfminer.six 20201018 and pikepdf 2.x
+-  Fixed warning related to ``--pdfa-image-compression`` that appears at the wrong
+   time.
+
 v11.3.0
 =======
 
@@ -29,7 +329,7 @@ v11.3.0
    macOS and Windows only where the parent process is not forked.
 -  Fixed the hookspec of rasterize_pdf_page to remove default parameters that
    were not handled in an expected way by pluggy.
--  Fixed another issue with automatic page rotation (#658) due to the issue above.
+-  Fixed another issue with automatic page rotation (:issue:`658`) due to the issue above.
 
 v11.2.1
 =======
@@ -51,7 +351,7 @@ v11.1.2
 -  Fixed hOCR renderer writing the text in roughly reverse order. This should not
    affect reasonably smart PDF readers that properly locate the position of all
    text, but may confuse those that rely on the order of objects in the content
-   stream. (#642)
+   stream. (:issue:`642`)
 
 v11.1.1
 =======
@@ -64,9 +364,9 @@ v11.1.1
 v11.1.0
 =======
 
--  Fixed page rotation issues: #634, #589.
+-  Fixed page rotation issues: :issue:`634,589`.
 -  Fixed some cases where optimization created an invalid image such as a
-   1-bit "RGB" image: #629, #620.
+   1-bit "RGB" image: :issue:`629,620`.
 -  Page numbers are now displayed in debug logs when pages are being grafted.
 -  ocrmypdf.optimize.rewrite_png and ocrmypdf.optimize.rewrite_png_as_g4 were
    marked deprecated. Strictly speaking these should have been internal APIs,
@@ -79,7 +379,7 @@ v11.1.0
 v11.0.2
 =======
 
--  Fixed issue #612, TypeError exception. Fixed by eliminating unnecessary repair of
+-  Fixed :issue:`612`, TypeError exception. Fixed by eliminating unnecessary repair of
    input PDF metadata in memory.
 
 v11.0.1
@@ -96,7 +396,7 @@ v11.0.0
 -  Project license changed to Mozilla Public License 2.0. Some miscellaneous
    code is now under MIT license and non-code content/media remains under
    CC-BY-SA 4.0. License changed with approval of all people who were found
-   to have contributed to GPLv3 licensed sections of the project. (#600)
+   to have contributed to GPLv3 licensed sections of the project. (:issue:`600`)
 -  Because the license changed, this is being treated as a major version number
    change; however, there are no known breaking changes in functional behavior
    or API compared to v10.x.
@@ -105,7 +405,7 @@ v10.3.3
 =======
 
 -  Fixed a "KeyError: 'dpi'" error message when using ``--threshold`` on an image.
-   (#607)
+   (:issue:`607`)
 
 v10.3.2
 =======
@@ -148,16 +448,16 @@ v10.2.0
 =======
 
 -  Update Docker image to use Ubuntu 20.04.
--  Fixed issue PDF/A acquires title "Untitled" after conversion. (#582)
+-  Fixed issue PDF/A acquires title "Untitled" after conversion. (:issue:`582`)
 -  Fixed a problem where, when using ``--pdf-renderer hocr``, some text would
    be missing from the output when using a more recent version of Tesseract.
    Tesseract began adding more detailed markup about the semantics of text
    that our HOCR transform did not recognize, so it ignored them. This option is
    not the default. If necessary ``--redo-ocr`` also redoing OCR to fix such issues.
 -  Fixed an error in Python 3.9 beta, due to removal of deprecated
-   ``Element.getchildren()``. (#584)
+   ``Element.getchildren()``. (:issue:`584`)
 -  Implemented support using the API with ``BytesIO`` and other file stream objects.
-   (#545)
+   (:issue:`545`)
 
 v10.1.1
 =======
@@ -250,7 +550,7 @@ v9.8.0
 
 -  Fixed issue where only the first PNG (FlateDecode) image in a file would be
    considered for optimization. File sizes should be improved from here on.
--  Fixed a startup crash when the chosen language was Japanese (#543).
+-  Fixed a startup crash when the chosen language was Japanese (:issue:`543`).
 -  Added options to configure polling and log level to watcher.py.
 
 v9.7.2
@@ -297,11 +597,11 @@ v9.6.1
         they can be copied out as whole files, and to ensure syntax checking
         is maintained.
 
--  Fixed an error that caused bash completions to fail on macOS. (#502, #504;
+-  Fixed an error that caused bash completions to fail on macOS. (:issue:`502,504`;
    @AlexanderWillner)
 -  Fixed a rare case where OCRmyPDF threw an exception while processing a PDF
    with the wrong object type in its ``/Trailer /Info``. The error is now logged
-   and incorrect object is ignored. (#497)
+   and incorrect object is ignored. (:issue:`497`)
 -  Removed potentially non-free file ``enron1.pdf`` and simplified the test that
    used it.
 -  Removed potentially non-free file ``misc/media/logo.afdesign``.
@@ -503,7 +803,7 @@ v8.3.1
 ======
 
 -  Fixed an issue where PDFs with malformed metadata would be rendered as
-   blank pages. `#398 <https://github.com/jbarlow83/OCRmyPDF/issues/398>`_.
+   blank pages. :issue:`398`.
 
 v8.3.0
 ======
@@ -586,7 +886,7 @@ v8.2.0
    designed. However, quality would not be impacted. Lossless JBIG2 was
    entirely unaffected.
 -  Updated dependencies, including pikepdf to 1.1.0. This fixes
-   `#358 <https://github.com/jbarlow83/OCRmyPDF/issues/358>`__.
+   :issue:`358`.
 -  The install-time version checks for certain external programs have
    been removed from setup.py. These tests are now performed at
    run-time.
@@ -607,7 +907,7 @@ v8.1.0
    (specifically, those with invalid destination objects)
 -  Fixed an issue when using ``--tesseract-timeout`` and image
    processing features on a file with more than 100 pages.
-   `#347 <https://github.com/jbarlow83/OCRmyPDF/issues/347>`__
+   :issue:`347`
 -  OCRmyPDF now always calls ``os.nice(5)`` to signal to operating
    systems that it is a background process.
 
@@ -615,7 +915,7 @@ v8.0.1
 ======
 
 -  Fixed an exception when parsing PDFs that are missing a required
-   field. `#325 <https://github.com/jbarlow83/OCRmyPDF/issues/325>`__
+   field. :issue:`325`
 -  pikepdf 1.0.5 is now required, to address some other PDF parsing
    issues.
 
@@ -636,7 +936,7 @@ older versions of certain dependencies.
 **Other changes**
 
 -  Fixed an unhandled exception when attempting to mask barcodes.
-   `#322 <https://github.com/jbarlow83/OCRmyPDF/issues/322>`__
+   :issue:`322`
 -  It is now possible to use ocrmypdf without pdfminer.six, to support
    distributions that do not have it or cannot currently use it (e.g.
    Homebrew). Downstream maintainers should include pdfminer.six if
@@ -663,13 +963,13 @@ v7.4.0
 -  chardet >= 3.0.4 is temporarily listed as required. pdfminer.six
    depends on it, but the most recent release does not specify this
    requirement.
-   (`#326 <https://github.com/jbarlow83/OCRmyPDF/issues/326>`__)
+   (:issue:`326`)
 -  python-xmp-toolkit and libexempi are no longer required.
 -  A new Docker image is now being provided for users who wish to access
    OCRmyPDF over a simple HTTP interface, instead of the command line.
 -  Increase tolerance of PDFs that overflow or underflow the PDF
    graphics stack.
-   (`#325 <https://github.com/jbarlow83/OCRmyPDF/issues/325>`__)
+   (:issue:`325`)
 
 v7.3.1
 ======
@@ -743,7 +1043,7 @@ v7.3.0
 v7.2.1
 ======
 
--  Fix compatibility with an API change in pikepdf 0.3.5.
+-  Fixed compatibility with an API change in pikepdf 0.3.5.
 -  A kludge to support Leptonica versions older than 1.72 in the test
    suite was dropped. Older versions of Leptonica are likely still
    compatible. The only impact is that a portion of the test suite will
@@ -780,7 +1080,7 @@ Users who did not install an optional JBIG2 encoder are unaffected.
    will now attempt to further optimize that image as CCITT or JBIG2,
    instead of keeping it in the "flate" encoding which is not efficient
    for 1 bpp images.
-   (`#297 <https://github.com/jbarlow83/OCRmyPDF/issues/297>`__)
+   (:issue:`297`)
 -  Images in PDFs that are used as soft masks (i.e. transparency masks
    or alpha channels) are now excluded from optimization.
 -  Fixed handling of Tesseract 4.0-rc1 which now accepts invalid
@@ -792,15 +1092,14 @@ v7.1.0
 -  Improve the performance of initial text extraction, which is done to
    determine if a file contains existing text of some kind or not. On
    large files, this initial processing is now about 20x times faster.
-   (`#299 <https://github.com/jbarlow83/OCRmyPDF/issues/299>`__)
+   (:issue:`299`)
 -  pikepdf 0.3.3 is now required.
--  Fixed issue
-   `#231 <https://github.com/jbarlow83/OCRmyPDF/issues/231>`__, a
+-  Fixed :issue:`231`, a
    problem with JPEG2000 images where image metadata was only available
    inside the JPEG2000 file.
 -  Fixed some additional Ghostscript 9.25 compatibility issues.
 -  Improved handling of KeyboardInterrupt error messages.
-   (`#301 <https://github.com/jbarlow83/OCRmyPDF/issues/301>`__)
+   (:issue:`301`)
 -  README.md is now served in GitHub markdown instead of
    reStructuredText.
 
@@ -832,35 +1131,34 @@ v7.0.5
 v7.0.4
 ======
 
--  Fix exception thrown when trying to optimize a certain type of PNG
+-  Fixed exception thrown when trying to optimize a certain type of PNG
    embedded in a PDF with the ``-O2``
 -  Update to pikepdf 0.3.2, to gain support for optimizing some
    additional image types that were previously excluded from
    optimization (CMYK and grayscale). Fixes
-   `#285 <https://github.com/jbarlow83/OCRmyPDF/issues/285>`__.
+   :issue:`285`.
 
 v7.0.3
 ======
 
--  Fix issue
-   `#284 <https://github.com/jbarlow83/OCRmyPDF/issues/284>`__, an error
+-  Fixed :issue:`284`, an error
    when parsing inline images that have are also image masks, by
    upgrading pikepdf to 0.3.1
 
 v7.0.2
 ======
 
--  Fix a regression with ``--rotate-pages`` on pages that already had
+-  Fixed a regression with ``--rotate-pages`` on pages that already had
    rotations applied.
-   (`#279 <https://github.com/jbarlow83/OCRmyPDF/issues/279>`__)
+   (:issue:`279`)
 -  Improve quality of page rotation in some cases by rasterizing a
    higher quality preview image.
-   (`#281 <https://github.com/jbarlow83/OCRmyPDF/issues/281>`__)
+   (:issue:`281`)
 
 v7.0.1
 ======
 
--  Fix compatibility with img2pdf >= 0.3.0 by rejecting input images
+-  Fixed compatibility with img2pdf >= 0.3.0 by rejecting input images
    that have an alpha channel
 -  Add forward compatibility for pikepdf 0.3.0 (unrelated to img2pdf)
 -  Various documentation updates for v7.0.0 changes
@@ -963,7 +1261,7 @@ v6.2.4
 v6.2.3
 ======
 
--  Fix compatibility with img2pdf >= 0.3.0 by rejecting input images
+-  Fixed compatibility with img2pdf >= 0.3.0 by rejecting input images
    that have an alpha channel
 -  This version will be included in Ubuntu 18.10
 
@@ -980,9 +1278,8 @@ v6.2.2
 v6.2.1
 ======
 
--  Fix recent versions of Tesseract (after 4.0.0-beta1) not being
-   detected as supporting the ``sandwich`` renderer
-   (`#271 <https://github.com/ppjbarlow83/OCRmyPDF/issues/271>`__).
+-  Fixed recent versions of Tesseract (after 4.0.0-beta1) not being
+   detected as supporting the ``sandwich`` renderer (:issue:`271`).
 
 v6.2.0
 ======
@@ -995,21 +1292,19 @@ v6.2.0
 -  Creation of PDF/A-3 is now supported. However, there is no ability to
    attach files to PDF/A-3.
 -  Lists more reasons why the file size might grow.
--  Fix issue
-   `#262 <https://github.com/ppjbarlow83/OCRmyPDF/issues/262>`__,
+-  Fixed :issue:`262`,
    ``--remove-background`` error on PDFs contained colormapped
    (paletted) images.
--  Fix another XMP metadata validation issue, in cases where the input
+-  Fixed another XMP metadata validation issue, in cases where the input
    file's creation date has no timezone and the creation date is not
    overridden.
 
 v6.1.5
 ======
 
--  Fix issue
-   `#253 <https://github.com/jbarlow83/OCRmyPDF/issues/253>`__, a
+-  Fixed :issue:`253`, a
    possible division by zero when using the ``hocr`` renderer.
--  Fix incorrectly formatted ``<xmp:ModifyDate>`` field inside XMP
+-  Fixed incorrectly formatted ``<xmp:ModifyDate>`` field inside XMP
    metadata for PDF/As. veraPDF flags this as a PDF/A validation
    failure. The error is caused the timezone and final digit of the
    seconds of modified time to be omitted, so at worst the modification
@@ -1018,7 +1313,7 @@ v6.1.5
 v6.1.4
 ======
 
--  Fix issue `#248 <https://github.com/jbarlow83/OCRmyPDF/issues/248>`__
+-  Fixed :issue:`248`
    ``--clean`` argument may remove OCR from left column of text on
    certain documents. We now set ``--layout none`` to suppress this.
 -  The test cache was updated to reflect the change above.
@@ -1043,8 +1338,7 @@ Notes
 v6.1.3
 ======
 
--  Fix issue
-   `#247 <https://github.com/jbarlow83/OCRmyPDF/issues/247>`__,
+-  Fixed :issue:`247`,
    ``/CreationDate`` metadata not copied from input to output.
 -  A warning is now issued when Python 3.5 is used on files with a large
    page count, as this case is known to regress to single core
@@ -1054,13 +1348,13 @@ v6.1.2
 ======
 
 -  Upgrade to PyMuPDF v1.12.5 which includes a more complete fix to
-   `#239 <https://github.com/jbarlow83/OCRmyPDF/issues/239>`__.
+   :issue:`239`.
 -  Add ``defusedxml`` dependency.
 
 v6.1.1
 ======
 
--  Fix text being reported as found on all pages if PyMuPDF is not
+-  Fixed text being reported as found on all pages if PyMuPDF is not
    installed.
 
 v6.1.0
@@ -1071,15 +1365,15 @@ v6.1.0
    PyMuPDF than the author anticipated. (For version 6.x only) install
    OCRmyPDF with ``pip install ocrmypdf[fitz]`` to use it to its full
    potential.
--  Fix ``FileExistsError`` that could occur if OCR timed out while it
+-  Fixed ``FileExistsError`` that could occur if OCR timed out while it
    was generating the output file.
-   (`#218 <https://github.com/jbarlow83/OCRmyPDF/issues/218>`__)
--  Fix table of contents/bookmarks all being redirected to page 1 when
+   (:issue:`218`)
+-  Fixed table of contents/bookmarks all being redirected to page 1 when
    generating a PDF/A (with PyMuPDF). (Without PyMuPDF the table of
    contents is removed in PDF/A mode.)
--  Fix "RuntimeError: invalid key in dict" when table of
+-  Fixed "RuntimeError: invalid key in dict" when table of
    contents/bookmarks titles contained the character ``)``.
-   (`#239 <https://github.com/jbarlow83/OCRmyPDF/issues/239>`__)
+   (:issue:`239`)
 -  Added a new argument ``--skip-repair`` to skip the initial PDF repair
    step if the PDF is already well-formed (because another program
    repaired it).
@@ -1106,35 +1400,29 @@ v6.0.0
 
 -  Fixed an issue where OCRmyPDF failed to detect existing text on
    pages, depending on how the text and fonts were encoded within the
-   PDF. (`#233 <https://github.com/jbarlow83/OCRmyPDF/issues/233>`__,
-   `#232 <https://github.com/jbarlow83/OCRmyPDF/issues/232>`__)
+   PDF. (:issue:`233,232`)
 -  Fixed an issue that caused dramatic inflation of file sizes when
    ``--skip-text --output-type pdf`` was used. OCRmyPDF now removes
    duplicate resources such as fonts, images and other objects that it
-   generates.
-   (`#237 <https://github.com/jbarlow83/OCRmyPDF/issues/237>`__)
+   generates. (:issue:`237`)
 -  Improved performance of the initial page splitting step. Originally
    this step was not believed to be expensive and ran in a process.
    Large file testing revealed it to be a bottleneck, so it is now
    parallelized. On a 700 page file with quad core machine, this change
-   saves about 2 minutes.
-   (`#234 <https://github.com/jbarlow83/OCRmyPDF/issues/234>`__)
+   saves about 2 minutes. (:issue:`234`)
 -  The test suite now includes a cache that can be used to speed up test
    runs across platforms. This also does not require computing
-   checksums, so it's faster.
-   (`#217 <https://github.com/jbarlow83/OCRmyPDF/issues/217>`__)
+   checksums, so it's faster. (:issue:`217`)
 
 v5.7.0
 ======
 
 -  Fixed an issue that caused poor CPU utilization on machines with more
-   than 4 cores when running Tesseract 4. (Related to issue
-   `#217 <https://github.com/jbarlow83/OCRmyPDF/issues/217>`__.)
+   than 4 cores when running Tesseract 4. (Related to :issue:`217`.)
 -  The 'hocr' renderer has been improved. The 'sandwich' and 'tesseract'
    renderers are still better for most use cases, but 'hocr' may be
    useful for people who work with the PDF.js renderer in English/ASCII
-   languages.
-   (`#225 <https://github.com/jbarlow83/OCRmyPDF/issues/225>`__)
+   languages. (:issue:`225`)
 
    -  It now formats text in a matter that is easier for certain PDF
       viewers to select and extract copy and paste text. This should
@@ -1162,11 +1450,10 @@ v5.6.2
 v5.6.1
 ======
 
--  Fix issue
-   `#219 <https://github.com/jbarlow83/OCRmyPDF/issues/219>`__: change
+-  Fixed :issue:`219`: change
    how the final output file is created to avoid triggering permission
    errors when the output is a special file such as ``/dev/null``
--  Fix test suite failures due to a qpdf 8.0.0 regression and Python
+-  Fixed test suite failures due to a qpdf 8.0.0 regression and Python
    3.5's handling of symlink
 -  The "encrypted PDF" error message was different depending on the type
    of PDF encryption. Now a single clear message appears for all types
@@ -1179,8 +1466,7 @@ v5.6.1
 v5.6.0
 ======
 
--  Fix issue
-   `#216 <https://github.com/jbarlow83/OCRmyPDF/issues/216>`__: preserve
+-  Fixed :issue:`216`: preserve
    "text as curves" PDFs without rasterizing file
 -  Related to the above, messages about rasterizing are more consistent
 -  For consistency versions minor releases will now get the trailing .0
@@ -1192,34 +1478,32 @@ v5.5
 -  Add new argument ``--max-image-mpixels``. Pillow 5.0 now raises an
    exception when images may be decompression bombs. This argument can
    be used to override the limit Pillow sets.
--  Fix output page cropped when using the sandwich renderer and OCR is
+-  Fixed output page cropped when using the sandwich renderer and OCR is
    skipped on a rotated and image-processed page
 -  A warning is now issued when old versions of Ghostscript are used in
    cases known to cause issues with non-Latin characters
--  Fix a few parameter validation checks for ``-output-type pdfa-1`` and
+-  Fixed a few parameter validation checks for ``-output-type pdfa-1`` and
    ``pdfa-2``
 
 v5.4.4
 ======
 
--  Fix issue
-   `#181 <https://github.com/jbarlow83/OCRmyPDF/issues/181>`__: fix
+-  Fixed :issue:`181`: fix
    final merge failure for PDFs with more pages than the system file
    handle limit (``ulimit -n``)
--  Fix issue
-   `#200 <https://github.com/jbarlow83/OCRmyPDF/issues/200>`__: an
+-  Fixed :issue:`200`: an
    uncommon syntax for formatting decimal numbers in a PDF would cause
    qpdf to issue a warning, which ocrmypdf treated as an error. Now this
    the warning is relayed.
--  Fix an issue where intermediate PDFs would be created at version 1.3
+-  Fixed an issue where intermediate PDFs would be created at version 1.3
    instead of the version of the original file. It's possible but
    unlikely this had side effects.
 -  A warning is now issued when older versions of qpdf are used since
    issues like
-   `#200 <https://github.com/jbarlow83/OCRmyPDF/issues/200>`__ cause
+   :issue:`200` cause
    qpdf to infinite-loop
 -  Address issue
-   `#140 <https://github.com/jbarlow83/OCRmyPDF/issues/140>`__: if
+   :issue:`140`: if
    Tesseract outputs invalid UTF-8, escape it and print its message
    instead of aborting with a Unicode error
 -  Adding previously unlisted setup requirement, pytest-runner
@@ -1285,13 +1569,13 @@ v5.3
    forwarded to Tesseract OCR as words and regular expressions
    respective to use to guide OCR. Supplying a list of subject-domain
    words should assist Tesseract with resolving words.
-   (`#165 <https://github.com/jbarlow83/OCRmyPDF/issues/165>`__)
+   (:issue:`165`)
 -  Using a non Latin-1 language with the "hocr" renderer now warns about
    possible OCR quality and recommends workarounds
-   (`#176 <https://github.com/jbarlow83/OCRmyPDF/issues/176>`__)
+   (:issue:`176`)
 -  Output file path added to error message when that location is not
    writable
-   (`#175 <https://github.com/jbarlow83/OCRmyPDF/issues/175>`__)
+   (:issue:`175`)
 -  Otherwise valid PDFs with leading whitespace at the beginning of the
    file are now accepted
 
@@ -1319,8 +1603,7 @@ v5.1
 v5.0.1
 ======
 
--  Fixed issue
-   `#169 <https://github.com/jbarlow83/OCRmyPDF/issues/169>`__,
+-  Fixed :issue:`169`,
    exception due to failure to create sidecar text files on some
    versions of Tesseract 3.04, including the jbarlow83/ocrmypdf Docker
    image
@@ -1338,19 +1621,17 @@ v5.0
 -  Add a new feature, ``--sidecar``, which allows creating "sidecar"
    text files which contain the OCR results in plain text. These OCR
    text is more reliable than extracting text from PDFs. Closes
-   `#126 <https://github.com/jbarlow83/OCRmyPDF/issues/126>`__.
+   :issue:`126`.
 
 -  New feature: ``--pdfa-image-compression``, which allows overriding
    Ghostscript's lossy-or-lossless image encoding heuristic and making
    all images JPEG encoded or lossless encoded as desired. Fixes
-   `#163 <https://github.com/jbarlow83/OCRmyPDF/issues/163>`__.
+   :issue:`163`.
 
--  Fixed issue
-   `#143 <https://github.com/jbarlow83/OCRmyPDF/issues/143>`__, added
+-  Fixed :issue:`143`, added
    ``--quiet`` to suppress "INFO" messages
 
--  Fixed issue
-   `#164 <https://github.com/jbarlow83/OCRmyPDF/issues/164>`__, a typo
+-  Fixed :issue:`164`, a typo
 
 -  Removed the command line parameters ``-n`` and ``--just-print`` since
    they have not worked for some time (reported as Ubuntu bug
@@ -1359,17 +1640,14 @@ v5.0
 v4.5.6
 ======
 
--  Fixed issue
-   `#156 <https://github.com/jbarlow83/OCRmyPDF/issues/156>`__,
+-  Fixed :issue:`156`,
    'NoneType' object has no attribute 'getObject' on pages with no
    optional /Contents record. This should resolve all issues related to
    pages with no /Contents record.
--  Fixed issue
-   `#158 <https://github.com/jbarlow83/OCRmyPDF/issues/158>`__, ocrmypdf
+-  Fixed :issue:`158`, ocrmypdf
    now stops and terminates if Ghostscript fails on an intermediate
    step, as it is not possible to proceed.
--  Fixed issue
-   `#160 <https://github.com/jbarlow83/OCRmyPDF/issues/160>`__,
+-  Fixed :issue:`160`,
    exception thrown on certain invalid arguments instead of error
    message
 
@@ -1377,20 +1655,19 @@ v4.5.5
 ======
 
 -  Automated update of macOS homebrew tap
--  Fixed issue
-   `#154 <https://github.com/jbarlow83/OCRmyPDF/issues/154>`__, KeyError
+-  Fixed :issue:`154`, KeyError
    '/Contents' when searching for text on blank pages that have no
    /Contents record. Note: incomplete fix for this issue.
 
 v4.5.4
 ======
 
--  Fix ``--skip-big`` raising an exception if a page contains no images
-   (`#152 <https://github.com/jbarlow83/OCRmyPDF/issues/152>`__) (thanks
+-  Fixed ``--skip-big`` raising an exception if a page contains no images
+   (:issue:`152`) (thanks
    to @TomRaz)
--  Fix an issue where pages with no images might trigger "cannot write
+-  Fixed an issue where pages with no images might trigger "cannot write
    mode P as JPEG"
-   (`#151 <https://github.com/jbarlow83/OCRmyPDF/issues/151>`__)
+   (:issue:`151`)
 
 v4.5.3
 ======
@@ -1409,8 +1686,7 @@ v4.5.3
 v4.5.2
 ======
 
--  Fix issue
-   `#147 <https://github.com/jbarlow83/OCRmyPDF/issues/147>`__.
+-  Fixed :issue:`147`,
    ``--pdf-renderer tess4 --clean`` will produce an oversized page
    containing the original image in the bottom left corner, due to loss
    DPI information.
@@ -1420,8 +1696,7 @@ v4.5.2
 v4.5.1
 ======
 
--  Fix issue
-   `#137 <https://github.com/jbarlow83/OCRmyPDF/issues/137>`__,
+-  Fixed :issue:`137`,
    proportions of images with a non-square pixel aspect ratio would be
    distorted in output for ``--force-ocr`` and some other combinations
    of flags
@@ -1430,7 +1705,7 @@ v4.5
 ====
 
 -  PDFs containing "Form XObjects" are now supported (issue
-   `#134 <https://github.com/jbarlow83/OCRmyPDF/issues/134>`__; PDF
+   :issue:`134`; PDF
    reference manual 8.10), and images they contain are taken into
    account when determining the resolution for rasterizing
 -  The Tesseract 4 Docker image no longer includes all languages,
@@ -1559,7 +1834,7 @@ v4.2.5
 ======
 
 -  Fixed an issue
-   (`#100 <https://github.com/jbarlow83/OCRmyPDF/issues/100>`__) with
+   (:issue:`100`) with
    PDFs that omit the optional /BitsPerComponent parameter on images
 -  Removed non-free file milk.pdf
 
@@ -1567,7 +1842,7 @@ v4.2.4
 ======
 
 -  Fixed an error
-   (`#90 <https://github.com/jbarlow83/OCRmyPDF/issues/90>`__) caused by
+   (:issue:`90`) caused by
    PDFs that use stencil masks properly
 -  Fixed handling of PDFs that try to draw images or stencil masks
    without properly setting up the graphics state (such images are now
@@ -1605,7 +1880,7 @@ v4.2
 
 -  ocrmypdf will now try to convert single image files to PDFs if they
    are provided as input
-   (`#15 <https://github.com/jbarlow83/OCRmyPDF/issues/15>`__)
+   (:issue:`15`)
 
    -  This is a basic convenience feature. It only supports a single
       image and always makes the image fill the whole page.
@@ -1633,11 +1908,11 @@ v4.2
    -  supports reinterpreting PDFs where text was rendered as curves for
       printing, and text needs to be recovered
    -  fixes issue
-      `#82 <https://github.com/jbarlow83/OCRmyPDF/issues/82>`__
+      :issue:`82`
 
 -  Fixes an issue where, with certain settings, monochrome images in
    PDFs would be converted to 8-bit grayscale, increasing file size
-   (`#79 <https://github.com/jbarlow83/OCRmyPDF/issues/79>`__)
+   (:issue:`79`)
 -  Support for Ubuntu 12.04 LTS "precise" has been dropped in favor of
    (roughly) Ubuntu 14.04 LTS "trusty"
 
@@ -1665,7 +1940,7 @@ v4.1.3
 
 -  More helpful error message for PDFs with version 4 security handler
 -  Update usage instructions for Windows/Docker users
--  Fix order of operations for matrix multiplication (no effect on most
+-  Fixed order of operations for matrix multiplication (no effect on most
    users)
 -  Add a few leptonica wrapper functions (no effect on most users)
 
@@ -1758,7 +2033,7 @@ New features
    dominant orientation of detectable text. This feature is fairly
    reliable but some false positives occur especially if there is not
    much text to work with.
-   (`#4 <https://github.com/jbarlow83/OCRmyPDF/issues/4>`__)
+   (:issue:`4`)
 -  Deskewing is now performed using Leptonica instead of unpaper.
    Leptonica is faster and more reliable at image deskewing than
    unpaper.
@@ -1771,13 +2046,13 @@ Fixes
 -  Fixed an issue where lossless reconstruction could misalign the
    graphics layer with respect to text layer if the page had been
    cropped such that its origin is not (0, 0)
-   (`#49 <https://github.com/jbarlow83/OCRmyPDF/issues/49>`__)
+   (:issue:`49`)
 
 Changes
 
 -  Logging output is now much easier to read
 -  ``--deskew`` is now performed by Leptonica instead of unpaper
-   (`#25 <https://github.com/jbarlow83/OCRmyPDF/issues/25>`__)
+   (:issue:`25`)
 -  libffi is now required
 -  Some changes were made to the Docker and Travis build environments to
    support libffi
@@ -1792,7 +2067,7 @@ v3.2.1
 
 Changes
 
--  Fixed issue `#47 <https://github.com/jbarlow83/OCRmyPDF/issues/47>`__
+-  Fixed :issue:`47`
    "convert() got and unexpected keyword argument 'dpi'" by upgrading to
    img2pdf 0.2
 -  Tweaked the Dockerfiles
@@ -1837,8 +2112,7 @@ Changes
 -  Python 3.5 and macOS El Capitan are now supported platforms - no
    changes were needed to implement support
 -  Improved some error messages related to missing input files
--  Fixed issue `#20 <https://github.com/jbarlow83/OCRmyPDF/issues/20>`__
-   - uppercase .PDF extension not accepted
+-  Fixed :issue:`20`: uppercase .PDF extension not accepted
 -  Fixed an issue where OCRmyPDF failed to text that certain pages
    contained previously OCR'ed text, such as OCR text produced by
    Tesseract 3.04
@@ -1915,19 +2189,19 @@ Release candidates^
 
 -  rc9:
 
-   -  fix issue
-      `#118 <https://github.com/jbarlow83/OCRmyPDF/issues/118>`__:
+   -  Fix
+      :issue:`118`:
       report error if ghostscript iccprofiles are missing
    -  fixed another issue related to
-      `#111 <https://github.com/jbarlow83/OCRmyPDF/issues/111>`__: PDF
+      :issue:`111`: PDF
       rasterized to palette file
    -  add support image files with a palette
    -  don't try to validate PDF file after an exception occurs
 
 -  rc8:
 
-   -  fix issue
-      `#111 <https://github.com/jbarlow83/OCRmyPDF/issues/111>`__:
+   -  Fix
+      :issue:`111`:
       exception thrown if PDF is missing DocumentInfo dictionary
 
 -  rc7:

misc/completion/ocrmypdf.fish

@@ -59,7 +59,8 @@ complete -c ocrmypdf -x -l output-type -a '(__fish_ocrmypdf_output_type)' -d "se
 
 function __fish_ocrmypdf_pdf_renderer
     echo -e "auto\t"(_ "auto select PDF renderer")
-    echo -e "hocr\t"(_ "use hocr renderer")
+    echo -e "hocr\t"(_ "use hOCR renderer")
+    echo -e "hocrdebug\t"(_ "uses hOCR renderer in debug mode, showing recognized text")
     echo -e "sandwich\t"(_ "use sandwich renderer")
 end
 complete -c ocrmypdf -x -l pdf-renderer -a '(__fish_ocrmypdf_pdf_renderer)' -d "select PDF renderer options"
@@ -135,4 +136,4 @@ complete -c ocrmypdf -r -l user-words -d "specify location of user words file"
 complete -c ocrmypdf -r -l user-patterns -d "specify location of user patterns file"
 complete -c ocrmypdf -x -l fast-web-view -d "if file size if above this amount in MB, linearize PDF"
 
-complete -c ocrmypdf -x -a "(__fish_complete_suffix .pdf)"
+complete -c ocrmypdf -x -a "(__fish_complete_suffix .pdf; __fish_complete_suffix .PDF; __fish_complete_suffix .jpg; __fish_complete_suffix .png)"

misc/docker-compose.example.yml

@@ -9,7 +9,7 @@ services:
       - "/media/scan:/input"
       - "/mnt/scan:/output"
     environment:
-      - OCR_OUTPUT_DIRECTORY_YEAR_MONT=0
+      - OCR_OUTPUT_DIRECTORY_YEAR_MONTH=0
     user: "<SET TO YOUR USER ID>:<SET TO YOUR GROUP ID>"
     entrypoint: python3
     command: watcher.py

misc/synology.py

@@ -79,8 +79,10 @@ for dir_name, subdirs, file_list in os.walk(start_dir):
                 stdout=output_file,
                 stderr=subprocess.PIPE,
                 check=False,
+                text=True,
+                errors='ignore',
             )
-        logging.info(proc.stderr.read())
+        logging.info(proc.stderr)
         os.chmod(full_path_ocr, 0o664)
         os.chmod(full_path, 0o664)
         full_path_ocr_archive = sys.argv[2]

misc/watcher.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # Copyright (C) 2019 Ian Alexander: https://github.com/ianalexander
 # Copyright (C) 2020 James R Barlow: https://github.com/jbarlow83
 #
@@ -36,16 +37,21 @@ import ocrmypdf
 
 # pylint: disable=logging-format-interpolation
 
+
+def getenv_bool(name: str, default: str = 'False'):
+    return os.getenv(name, default).lower() in ('true', 'yes', 'y', '1')
+
+
 INPUT_DIRECTORY = os.getenv('OCR_INPUT_DIRECTORY', '/input')
 OUTPUT_DIRECTORY = os.getenv('OCR_OUTPUT_DIRECTORY', '/output')
-OUTPUT_DIRECTORY_YEAR_MONTH = bool(os.getenv('OCR_OUTPUT_DIRECTORY_YEAR_MONTH', ''))
-ON_SUCCESS_DELETE = bool(os.getenv('OCR_ON_SUCCESS_DELETE', ''))
-DESKEW = bool(os.getenv('OCR_DESKEW', ''))
+OUTPUT_DIRECTORY_YEAR_MONTH = getenv_bool('OCR_OUTPUT_DIRECTORY_YEAR_MONTH')
+ON_SUCCESS_DELETE = getenv_bool('OCR_ON_SUCCESS_DELETE')
+DESKEW = getenv_bool('OCR_DESKEW')
 OCR_JSON_SETTINGS = json.loads(os.getenv('OCR_JSON_SETTINGS', '{}'))
 POLL_NEW_FILE_SECONDS = int(os.getenv('OCR_POLL_NEW_FILE_SECONDS', '1'))
-USE_POLLING = bool(os.getenv('OCR_USE_POLLING', ''))
-LOGLEVEL = os.getenv('OCR_LOGLEVEL', 'INFO').upper()
-PATTERNS = ['*.pdf']
+USE_POLLING = getenv_bool('OCR_USE_POLLING')
+LOGLEVEL = os.getenv('OCR_LOGLEVEL', 'INFO')
+PATTERNS = ['*.pdf', '*.PDF']
 
 log = logging.getLogger('ocrmypdf-watcher')
 
@@ -117,7 +123,12 @@ class HandleObserverEvent(PatternMatchingEventHandler):
 
 def main():
     ocrmypdf.configure_logging(
-        verbosity=ocrmypdf.Verbosity.default, manage_root_logger=True
+        verbosity=(
+            ocrmypdf.Verbosity.default
+            if LOGLEVEL != 'DEBUG'
+            else ocrmypdf.Verbosity.debug
+        ),
+        manage_root_logger=True,
     )
     log.setLevel(LOGLEVEL)
     log.info(
@@ -135,7 +146,7 @@ def main():
         f"ARGS: {OCR_JSON_SETTINGS}\n"
         f"POLL_NEW_FILE_SECONDS: {POLL_NEW_FILE_SECONDS}\n"
         f"USE_POLLING: {USE_POLLING}\n"
-        f"LOGLEVEL: {LOGLEVEL}\n"
+        f"LOGLEVEL: {LOGLEVEL}"
     )
 
     if 'input_file' in OCR_JSON_SETTINGS or 'output_file' in OCR_JSON_SETTINGS:

misc/webservice.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # webservice.py wrapper for OCRmyPDF
 # Copyright (C) 2019 James R. Barlow: github.com/jbarlow83
 #
@@ -28,16 +29,7 @@ import shlex
 from subprocess import PIPE, run
 from tempfile import TemporaryDirectory
 
-from flask import (
-    Flask,
-    Response,
-    abort,
-    flash,
-    redirect,
-    request,
-    send_from_directory,
-    url_for,
-)
+from flask import Flask, Response, request, send_from_directory
 from werkzeug.utils import secure_filename
 
 app = Flask(__name__)

pyproject.toml

@@ -3,11 +3,14 @@ requires = [
   "setuptools >= 30.3.0",
   "wheel",
   "cffi",
-  "setuptools_scm",
+  "setuptools_scm[toml] >= 3.4",
   "setuptools_scm_git_archive"
 ]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools_scm]
+version_scheme = "post-release"
+
 [tool.black]
 line-length = 88
 target-version = ["py36", "py37", "py38"]
@@ -31,3 +34,41 @@ exclude = '''
   | src/ocrmypdf/lib/_leptonica.py
 )/
 '''
+
+[tool.coverage.run]
+branch = true
+parallel = true
+concurrency = ["multiprocessing"]
+
+[tool.coverage.paths]
+source = ["src/ocrmypdf"]
+
+[tool.coverage.report]
+# Regexes for lines to exclude from consideration
+exclude_lines = [
+    # Have to re-enable the standard pragma
+    "pragma: no cover",
+
+    # Don't complain if tests don't hit defensive assertion code:
+    "raise AssertionError",
+    "raise NotImplementedError",
+
+    # Don't complain if non-runnable code isn't run:
+    "if 0:",
+	  "if False:",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:"
+]
+
+[tool.isort]
+profile = "black"
+known_first_party = "ocrmypdf"
+known_third_party = ["PIL","_cffi_backend","cffi","flask","img2pdf","pdfminer","pikepdf","pkg_resources","pluggy","pytest","reportlab","setuptools","sphinx_rtd_theme","tqdm","watchdog","werkzeug"]
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+norecursedirs = ["lib", ".pc", ".git", "venv", "output", "cache", "resources"]
+testpaths = ["tests"]
+addopts = "-n auto"
+markers = ["slow"]
+filterwarnings = ["ignore:.*XMLParser.*:DeprecationWarning"]

requirements/main.txt

@@ -1,12 +1,10 @@
-# requirements.txt can be used to replicate the developer's build environment
-# setup.py lists a separate set of requirements that are looser to simplify
-# installation
-cffi == 1.14.0
-coloredlogs == 14.0  # technically optional
-img2pdf == 0.3.6
-pdfminer.six == 20200517
-pikepdf == 1.16.1
+# Deprecated and not maintained; use "pip install ocrmypdf" instead
+cffi == 1.14.5
+coloredlogs == 15.0  # technically optional
+img2pdf == 0.4.0
+pdfminer.six == 20201018
+pikepdf == 2.10.0
 pluggy == 0.13.1
-Pillow == 7.1.2
-reportlab == 3.5.42
-tqdm == 4.46.1
+Pillow == 8.2.0
+reportlab == 3.5.66
+tqdm == 4.59.0

requirements/test.txt

@@ -1,7 +1,7 @@
-pytest >= 5.0.0
-pytest-helpers-namespace >= 2019.1.8
-pytest-xdist >= 1.31.0
-pytest-cov >= 2.10.0
+# Deprecated and not maintained; use "pip install ocrmypdf[test]" instead
+pytest >= 6.0.0
+pytest-xdist >= 2.2.0
+pytest-cov >= 2.11.1
 python-xmp-toolkit == 2.0.1  # requires apt-get install libexempi3
                      # or brew install exempi
 #PyMuPDF == 1.13.4    # optional

requirements/watcher.txt

@@ -1 +1,2 @@
-watchdog == 0.10.2
+# Deprecated and not maintained; use "pip install ocrmypdf[watcher]" instead
+watchdog == 1.0.2

requirements/webservice.txt

@@ -1 +1,2 @@
+# Deprecated and not maintained; use "pip install ocrmypdf[webservice]" instead
 Flask >= 1, < 2

setup.cfg

@@ -1,29 +1,104 @@
+[metadata]
+name = ocrmypdf
+description = OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched
+long_description = file: README.md
+long_description_content_type = text/markdown; charset=UTF-8
+url = https://github.com/jbarlow83/OCRmyPDF
+author = James R. Barlow
+author_email = james@purplerock.ca
+license_files =
+    LICENSE
+keywords =
+    PDF
+    OCR
+    optical character recognition
+    PDF/A
+    scanning
+classifiers =
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Development Status :: 5 - Production/Stable
+    Environment :: Console
+    Intended Audience :: End Users/Desktop
+    Intended Audience :: Science/Research
+    Intended Audience :: System Administrators
+    License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
+    Operating System :: MacOS :: MacOS X
+    Operating System :: Microsoft :: Windows :: Windows 10
+    Operating System :: POSIX
+    Operating System :: POSIX :: BSD
+    Operating System :: POSIX :: Linux
+    Topic :: Scientific/Engineering :: Image Recognition
+    Topic :: Text Processing :: Indexing
+    Topic :: Text Processing :: Linguistic
+project_urls =
+    Documentation = https://ocrmypdf.readthedocs.io/
+    Source = https://github.com/jbarlow83/ocrmypdf
+    Tracker = https://github.com/jbarlow83/ocrmypdf/issues
+
+[options]
+zip_safe = False
+packages = find:
+package_dir =
+    =src
+platforms = any
+include_package_data=True
+install_requires =
+    cffi >= 1.9.1  # must be a setup and install requirement
+    coloredlogs >= 14.0  # strictly optional
+    img2pdf >= 0.3.0, < 0.5  # pure Python, so track HEAD closely
+    pdfminer.six >= 20191110, != 20200720, <= 20201018
+    pikepdf >= 2.10.0
+    Pillow >= 8.2.0
+    pluggy >= 0.13.0, < 1.0
+    reportlab >= 3.5.66
+    setuptools
+    tqdm >= 4
+python_requires = >= 3.6
+setup_requires =  # can be removed whenever we can drop pip 9 support
+    cffi >= 1.9.1  # to build the leptonica module
+    setuptools_scm  # so that version will work
+    setuptools_scm_git_archive  # enable version from github tarballs
+
+[options.package_data]
+ocrmypdf =
+    data/sRGB.icc
+    py.typed
+
+[options.packages.find]
+where = src
+
+[options.extras_require]
+test =
+    coverage[toml] >= 5
+    pytest >= 6.0.0
+    pytest-xdist >= 2.2.0
+    pytest-cov >= 2.11.1
+    python-xmp-toolkit == 2.0.1  # also requires apt-get install libexempi3
+                                 # or brew install exempi
+docs =
+    sphinx
+    sphinx-rtd-theme
+    sphinx-issues
+extended_test =
+    PyMuPDF == 1.13.4
+watcher =
+    watchdog >= 1.0.2, < 3
+webservice =
+    Flask >= 1, < 3
+
+[options.entry_points]
+console_scripts =
+    ocrmypdf = ocrmypdf.__main__:run
+
 [bdist_wheel]
-python-tag = py35
+python-tag = py36
 
 [aliases]
-test=pytest
+test = pytest
 
 [check-manifest]
 ignore =
 	.github
-
-[tool:pytest]
-norecursedirs = lib .pc .git output cache resources
-testpaths = tests
-filterwarnings =
-	ignore:.*XMLParser.*:DeprecationWarning
-markers =
-	slow
-
-[isort]
-multi_line_output=3
-include_trailing_comma=True
-force_grid_wrap=0
-use_parentheses=True
-line_length=88
-known_first_party = ocrmypdf
-known_third_party = PIL,_cffi_backend,cffi,flask,img2pdf,pdfminer,pikepdf,pkg_resources,pluggy,pytest,reportlab,setuptools,sphinx_rtd_theme,tqdm,watchdog,werkzeug
-
-[metadata]
-license_file = LICENSE

setup.py

@@ -1,93 +1,19 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2021 James R. Barlow: github.com/jbarlow83
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
-from __future__ import print_function, unicode_literals
-
-import sys
-
-from setuptools import find_packages, setup
-
-if sys.version_info < (3, 6):
-    print("Python 3.6 or newer is required", file=sys.stderr)
-    sys.exit(1)
-
-if 'upload' in sys.argv[1:]:
-    print('Use twine to upload the package - setup.py upload is insecure')
-    sys.exit(1)
-
-tests_require = open('requirements/test.txt', encoding='utf-8').read().splitlines()
-
-
-def readme():
-    with open('README.md', encoding='utf-8') as f:
-        return f.read()
-
+from setuptools import setup
 
+# Minimal setup to support older setuptools/setuptools_scm
 setup(
-    name='ocrmypdf',
-    description='OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched',
-    long_description=readme(),
-    long_description_content_type='text/markdown',
-    url='https://github.com/jbarlow83/OCRmyPDF',
-    author='James R. Barlow',
-    author_email='james@purplerock.ca',
-    packages=find_packages('src', exclude=["tests", "tests.*"]),
-    package_dir={'': 'src'},
-    keywords=['PDF', 'OCR', 'optical character recognition', 'PDF/A', 'scanning'],
-    classifiers=[
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Development Status :: 5 - Production/Stable",
-        "Environment :: Console",
-        "Intended Audience :: End Users/Desktop",
-        "Intended Audience :: Science/Research",
-        "Intended Audience :: System Administrators",
-        "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
-        "Operating System :: MacOS :: MacOS X",
-        "Operating System :: Microsoft :: Windows :: Windows 10",
-        "Operating System :: POSIX",
-        "Operating System :: POSIX :: BSD",
-        "Operating System :: POSIX :: Linux",
-        "Topic :: Scientific/Engineering :: Image Recognition",
-        "Topic :: Text Processing :: Indexing",
-        "Topic :: Text Processing :: Linguistic",
-    ],
-    python_requires=' >= 3.6',
     setup_requires=[  # can be removed whenever we can drop pip 9 support
         'cffi >= 1.9.1',  # to build the leptonica module
-        'pytest-runner',  # to enable python setup.py test
         'setuptools_scm',  # so that version will work
         'setuptools_scm_git_archive',  # enable version from github tarballs
     ],
     use_scm_version={'version_scheme': 'post-release'},
     cffi_modules=['src/ocrmypdf/lib/compile_leptonica.py:ffibuilder'],
-    install_requires=[
-        'cffi >= 1.9.1',  # must be a setup and install requirement
-        'coloredlogs >= 14.0',  # strictly optional
-        'img2pdf >= 0.3.0, < 0.5',  # pure Python, so track HEAD closely
-        'pdfminer.six >= 20191110, != 20200720, <= 20200726',
-        'pikepdf >= 1.14.0, < 2',
-        'Pillow >= 7.0.0',
-        'pluggy >= 0.13.0, < 1.0',
-        'reportlab >= 3.3.0',  # oldest released version with sane image handling
-        'tqdm >= 4',
-    ],
-    tests_require=tests_require,
-    entry_points={'console_scripts': ['ocrmypdf = ocrmypdf.__main__:run']},
-    package_data={'ocrmypdf': ['data/sRGB.icc']},
-    include_package_data=True,
-    zip_safe=False,
-    project_urls={
-        'Documentation': 'https://ocrmypdf.readthedocs.io/',
-        'Source': 'https://github.com/jbarlow83/ocrmypdf',
-        'Tracker': 'https://github.com/jbarlow83/ocrmypdf/issues',
-    },
 )

src/ocrmypdf/RELEASE.md

@@ -0,0 +1,35 @@
+# Release checklist
+
+## Patch release
+
+- Check `pytest`
+
+- Update release notes
+
+## Minor release
+
+## Major release
+
+- Run `pre-commit autoupdate`
+
+- Check README.md
+
+- Check setup.py
+
+    - Are classifiers up to date?
+    - Is `python_requires` correct?
+    - Python 3.6 is EOL on December 2021-12. Could drop support then.
+    - Can we tighten any `install_requires` dependencies?
+
+- Search for old version shims we can remove
+
+    - "shim"
+    - ` pikepdf.__version__`
+
+- Search for deprecation: search all files for deprec*, etc.
+
+- Check requirements in setup.cfg
+
+- Delete `tests/cache`, do `pytest --runslow`, and update cache.
+
+- Do `pytest --cov-report html`

src/ocrmypdf/__init__.py

@@ -8,6 +8,8 @@
 from pluggy import HookimplMarker as _HookimplMarker
 
 from ocrmypdf import helpers, hocrtransform, leptonica, pdfa, pdfinfo
+from ocrmypdf._concurrent import Executor
+from ocrmypdf._jobcontext import PageContext, PdfContext
 from ocrmypdf._version import PROGRAM_NAME, __version__
 from ocrmypdf.api import Verbosity, configure_logging, ocr
 from ocrmypdf.exceptions import (

src/ocrmypdf/__main__.py

@@ -47,7 +47,10 @@ def run(args=None):
         verbosity = Verbosity.quiet
         options.progress_bar = False
     configure_logging(
-        verbosity, progress_bar_friendly=options.progress_bar, manage_root_logger=True
+        verbosity,
+        progress_bar_friendly=options.progress_bar,
+        manage_root_logger=True,
+        plugin_manager=plugin_manager,
     )
     log.debug('ocrmypdf %s', __version__)
     try:

src/ocrmypdf/_concurrent.py

@@ -4,136 +4,130 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-
-import logging
-import logging.handlers
-import multiprocessing
-import os
-import signal
-import sys
 import threading
-from multiprocessing import Pool as ProcessPool
-from multiprocessing.dummy import Pool as ThreadPool
+from abc import ABC, abstractmethod
 from typing import Callable, Iterable, Optional
 
-from tqdm import tqdm
 
-from ocrmypdf.exceptions import InputFileError
+def _task_noop(*_args, **_kwargs):
+    return
 
 
-def log_listener(queue):
-    """Listen to the worker processes and forward the messages to logging
+class NullProgressBar:
+    def __init__(self, **kwargs):
+        pass
 
-    For simplicity this is a thread rather than a process. Only one process
-    should actually write to sys.stderr or whatever we're using, so if this is
-    made into a process the main application needs to be directed to it.
+    def __enter__(self):
+        return self
 
-    See https://docs.python.org/3/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes
+    def __exit__(self, exc_type, exc_value, traceback):
+        return False
+
+    def update(self, _arg=None):
+        return
+
+
+class Executor(ABC):
+    pool_lock = threading.Lock()
+    pbar_class = NullProgressBar
+
+    def __init__(self, *, pbar_class=None):
+        if pbar_class:
+            self.pbar_class = pbar_class
+
+    def __call__(
+        self,
+        *,
+        use_threads: bool,
+        max_workers: int,
+        tqdm_kwargs: dict,
+        worker_initializer: Optional[Callable] = None,
+        task: Optional[Callable] = None,
+        task_arguments: Optional[Iterable] = None,
+        task_finished: Optional[Callable] = None,
+    ) -> None:
+        """
+        Set up parallel execution and progress reporting.
+
+        Args:
+            use_threads: If ``False``, the workload is the sort that will benefit from
+                running in a multiprocessing context (for example, it uses Python
+                heavily, and parallelizing it with threads is not expected to be
+                performant).
+            max_workers: The maximum number of workers that should be run.
+            tdqm_kwargs: Arguments to set up the progress bar.
+            worker_initializer: Called when a worker is initialized, in the worker's
+                execution context. If the child workers are processes, it must be
+                possible to marshall/pickle the worker initializer.
+                ``functools.partial`` can be used to bind parameters.
+            task: Called when the worker starts a new task, in the worker's execution
+                context. Must be possible to marshall to the worker.
+            task_finished: Called when a worker finishes a task, in the parent's
+                context.
+            task_arguments: An iterable that generates a group of parameters for each
+                task. This runs in the parent's context, but the parameters must be
+                marshallable to the worker.
+        """
+
+        if not task_arguments:
+            return  # Nothing to do!
+        if not worker_initializer:
+            worker_initializer = _task_noop
+        if not task_finished:
+            task_finished = _task_noop
+        if not task:
+            task = _task_noop
+
+        with self.pool_lock:
+            self._execute(
+                use_threads=use_threads,
+                max_workers=max_workers,
+                tqdm_kwargs=tqdm_kwargs,
+                worker_initializer=worker_initializer,
+                task=task,
+                task_arguments=task_arguments,
+                task_finished=task_finished,
+            )
+
+    @abstractmethod
+    def _execute(
+        self,
+        *,
+        use_threads: bool,
+        max_workers: int,
+        tqdm_kwargs: dict,
+        worker_initializer: Callable,
+        task: Callable,
+        task_arguments: Iterable,
+        task_finished: Callable,
+    ):
+        """Custom executors should override this method."""
+
+
+def setup_executor(plugin_manager) -> Executor:
+    pbar_class = plugin_manager.hook.get_progressbar_class()
+    return plugin_manager.hook.get_executor(progressbar_class=pbar_class)
+
+
+class SerialExecutor(Executor):
+    """Implements a purely sequential executor using the parallel protocol.
+
+    The current process/thread will be the worker that executes all tasks
+    in order. As such, ``worker_initializer`` will never be called.
     """
 
-    while True:
-        try:
-            record = queue.get()
-            if record is None:
-                break
-            logger = logging.getLogger(record.name)
-            logger.handle(record)
-        except Exception:  # pylint: disable=broad-except
-            import traceback  # pylint: disable=import-outside-toplevel
-
-            print("Logging problem", file=sys.stderr)
-            traceback.print_exc(file=sys.stderr)
-
-
-def process_sigbus(*args):
-    raise InputFileError("A worker process lost access to an input file")
-
-
-def process_init(queue, user_init, loglevel):
-    """Initialize a process pool worker"""
-
-    # Ignore SIGINT (our parent process will kill us gracefully)
-    signal.signal(signal.SIGINT, signal.SIG_IGN)
-
-    # Install SIGBUS handler (so our parent process can abort somewhat gracefully)
-    if hasattr(signal, 'SIGBUS'):
-        signal.signal(signal.SIGBUS, process_sigbus)
-
-    # Reconfigure the root logger for this process to send all messages to a queue
-    h = logging.handlers.QueueHandler(queue)
-    root = logging.getLogger()
-    root.setLevel(loglevel)
-    root.handlers = []
-    root.addHandler(h)
-
-    if user_init:
-        user_init()
-
-
-def thread_init(_queue, user_init, _loglevel):
-    # As a thread, block SIGBUS so the main thread deals with it...
-    if hasattr(signal, 'SIGBUS'):
-        signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGBUS})
-    if user_init:
-        user_init()
-
-
-def exec_progress_pool(
-    *,
-    use_threads: bool,
-    max_workers: int,
-    tqdm_kwargs: dict,
-    task_initializer: Optional[Callable] = None,
-    task: Optional[Callable] = None,
-    task_arguments: Optional[Iterable] = None,
-    task_finished: Optional[Callable] = None,
-):
-    log_queue: multiprocessing.Queue = multiprocessing.Queue(-1)
-    listener = threading.Thread(target=log_listener, args=(log_queue,))
-
-    if use_threads:
-        pool_class = ThreadPool
-        initializer = thread_init
-    else:
-        pool_class = ProcessPool
-        initializer = process_init
-    listener.start()
-
-    with tqdm(**tqdm_kwargs) as pbar:
-        pool = pool_class(
-            processes=max_workers,
-            initializer=initializer,
-            initargs=(log_queue, task_initializer, logging.getLogger("").level),
-        )
-        try:
-            results = pool.imap_unordered(task, task_arguments)
-            while True:
-                try:
-                    result = results.next()
-                    if task_finished:
-                        task_finished(result, pbar)
-                    else:
-                        pbar.update()
-                except StopIteration:
-                    break
-        except KeyboardInterrupt:
-            # Terminate pool so we exit instantly
-            pool.terminate()
-            # Don't try listener.join() here, will deadlock
-            raise
-        except Exception:
-            if not os.environ.get("PYTEST_CURRENT_TEST", ""):
-                # Unless inside pytest, exit immediately because no one wants
-                # to wait for child processes to finalize results that will be
-                # thrown away. Inside pytest, we want child processes to exit
-                # cleanly so that they output an error messages or coverage data
-                # we need from them.
-                pool.terminate()
-            raise
-        finally:
-            # Terminate log listener
-            log_queue.put_nowait(None)
-            pool.close()
-            pool.join()
-
-    listener.join()
+    def _execute(
+        self,
+        *,
+        use_threads: bool,
+        max_workers: int,
+        tqdm_kwargs: dict,
+        worker_initializer: Callable,
+        task: Callable,
+        task_arguments: Iterable,
+        task_finished: Callable,
+    ):  # pylint: disable=unused-argument
+        with self.pbar_class(**tqdm_kwargs) as pbar:
+            for args in task_arguments:
+                result = task(args)
+                task_finished(result, pbar)

src/ocrmypdf/_exec/ghostscript.py

@@ -15,34 +15,34 @@ from os import fspath
 from pathlib import Path
 from shutil import which
 from subprocess import PIPE, CalledProcessError
-from typing import Optional, cast
+from typing import Optional
 
 from PIL import Image
 
 from ocrmypdf.exceptions import MissingDependencyError, SubprocessOutputError
 from ocrmypdf.helpers import Resolution
-from ocrmypdf.subprocess import get_version, run
+from ocrmypdf.subprocess import get_version, run, run_polling_stderr
 
 log = logging.getLogger(__name__)
 
+missing_gs_error = """
+---------------------------------------------------------------------
+This error normally occurs when ocrmypdf find can't Ghostscript.
+Please ensure Ghostscript is installed and its location is added to
+the system PATH environment variable.
+
+For details see:
+    https://ocrmypdf.readthedocs.io/en/latest/installation.html
+---------------------------------------------------------------------
+"""
+
 _gswin = None
 if os.name == 'nt':
     _gswin = which('gswin64c')
     if not _gswin:
         _gswin = which('gswin32c')
         if not _gswin:
-            raise MissingDependencyError(
-                """
-                ---------------------------------------------------------------------
-                This error normally occurs when ocrmypdf can't Ghostscript.  Please
-                ensure Ghostscript is installed and its location is added to the
-                system PATH environment variable.
-
-                For details see:
-                    https://ocrmypdf.readthedocs.io/en/latest/installation.html
-                ---------------------------------------------------------------------
-                """
-            )
+            raise MissingDependencyError(missing_gs_error)
     _gswin = Path(_gswin).stem
 
 GS = _gswin if _gswin else 'gs'
@@ -56,16 +56,14 @@ def version():
 def jpeg_passthrough_available() -> bool:
     """Returns True if the installed version of Ghostscript supports JPEG passthru
 
-    Prior to 9.23, Ghostscript decode and re-encoded JPEGs internally. In 9.23
+    Prior to 9.23, Ghostscript decoded and re-encoded JPEGs internally. In 9.23
     it gained the ability to keep JPEGs unmodified. However, the 9.23
     implementation was buggy and would deletes the last two bytes of images in
     some cases, as reported here.
     https://bugs.ghostscript.com/show_bug.cgi?id=699216
 
     The issue was fixed for 9.24, hence that is the first version we consider
-    the feature available. (However, we don't use 9.24 at all, so the first
-    version that allows JPEG passthrough is 9.25.
-
+    the feature available. (Ghostscript 9.24 has its own problems is blacklisted.)
     """
     return version() >= '9.24'
 
@@ -122,8 +120,6 @@ def rasterize_pdf(
         stderr = p.stderr.decode(errors='replace')
         if _gs_error_reported(stderr):
             log.error(stderr)
-        elif stderr:
-            log.debug(stderr)
 
     with Image.open(BytesIO(p.stdout)) as im:
         if rotation is not None:
@@ -141,13 +137,44 @@ def rasterize_pdf(
         im.save(fspath(output_file), dpi=page_dpi)
 
 
+class GhostscriptFollower:
+    re_process = re.compile(r"Processing pages \d+ through (\d+).")
+    re_page = re.compile(r"Page (\d+)")
+
+    def __init__(self, progressbar_class):
+        self.count = 0
+        self.progressbar_class = progressbar_class
+        self.progressbar = None
+
+    def __call__(self, line):
+        if not self.progressbar_class:
+            return
+        if not self.progressbar:
+            m = self.re_process.match(line.strip())
+            if m:
+                self.count = int(m.group(1))
+                self.progressbar = self.progressbar_class(
+                    total=self.count, desc="PDF/A conversion", unit='page'
+                )
+                return
+        else:
+            m = self.re_page.match(line.strip())
+            if m:
+                self.progressbar.update()
+
+
 def generate_pdfa(
     pdf_pages,
     output_file: os.PathLike,
+    *,
     compression: str,
     pdf_version: str = '1.5',
     pdfa_part: str = '2',
+    progressbar_class=None,
 ):
+    # Ghostscript's compression is all or nothing. We can either force all images
+    # to JPEG, force all to Flate/PNG, or let it decide how to encode the images.
+    # In most case it's best to let it decide.
     compression_args = []
     if compression == 'jpeg':
         compression_args = [
@@ -169,14 +196,16 @@ def generate_pdfa(
             "-dAutoFilterGrayImages=true",
         ]
 
+    strategy = 'LeaveColorUnchanged'
     # Older versions of Ghostscript expect a leading slash in
     # sColorConversionStrategy, newer ones should not have it. See Ghostscript
     # git commit fe1c025d.
-    strategy = 'RGB' if version() >= '9.19' else '/RGB'
+    strategy = ('/' + strategy) if version() < '9.19' else strategy
 
     if version() == '9.23':
-        # 9.23: new feature JPEG passthrough is broken in some cases, best to
-        # disable it always
+        # 9.23: added JPEG passthrough as a new feature, but with a bug that
+        # incorrectly formats some images. Fixed as of 9.24. So we disable this
+        # feature for 9.23.
         # https://bugs.ghostscript.com/show_bug.cgi?id=699216
         compression_args.append('-dPassThroughJPEGImages=false')
 
@@ -186,7 +215,6 @@ def generate_pdfa(
     args_gs = (
         [
             GS,
-            "-dQUIET",
             "-dBATCH",
             "-dNOPAUSE",
             "-dSAFER",
@@ -206,16 +234,28 @@ def generate_pdfa(
         ]
     )
     args_gs.extend(fspath(s) for s in pdf_pages)  # Stringify Path objs
+
     try:
         with Path(output_file).open('wb') as output:
-            p = run(args_gs, stdout=output, stderr=PIPE, check=True)
+            p = run_polling_stderr(
+                args_gs,
+                stdout=output,
+                stderr=PIPE,
+                check=True,
+                text=True,
+                encoding='utf-8',
+                errors='replace',
+                callback=GhostscriptFollower(progressbar_class),
+            )
     except CalledProcessError as e:
         # Ghostscript does not change return code when it fails to create
         # PDF/A - check PDF/A status elsewhere
-        log.error(e.stderr.decode(errors='replace'))
-        raise SubprocessOutputError('Ghostscript PDF/A rendering failed')
+        log.error(e.stderr)
+        raise SubprocessOutputError('Ghostscript PDF/A rendering failed') from e
     else:
-        stderr = p.stderr.decode('utf-8', errors='replace')
+        stderr = p.stderr
+        # If there is an error we log the whole stderr, except for filtering
+        # duplicates.
         if _gs_error_reported(stderr):
             last_part = None
             repcount = 0
@@ -228,11 +268,3 @@ def generate_pdfa(
                 else:
                     repcount += 1
                 last_part = part
-        elif 'overprint mode not set' in stderr:
-            # Unless someone is going to print PDF/A documents on a
-            # magical sRGB printer I can't see the removal of overprinting
-            # being a problem....
-            log.debug(
-                "Ghostscript had to remove PDF 'overprinting' from the "
-                "input file to complete PDF/A conversion. "
-            )

src/ocrmypdf/_exec/jbig2enc.py

@@ -41,9 +41,17 @@ def convert_group(*, cwd, infiles, out_prefix):
     return proc
 
 
+def convert_group_mp(args):
+    return convert_group(cwd=args[0], infiles=args[1], out_prefix=args[2])
+
+
 def convert_single(*, cwd, infile, outfile):
     args = ['jbig2', '-p', infile]
     with open(outfile, 'wb') as fstdout:
         proc = run(args, cwd=cwd, stdout=fstdout, stderr=PIPE)
     proc.check_returncode()
     return proc
+
+
+def convert_single_mp(args):
+    return convert_single(cwd=args[0], infile=args[1], outfile=args[2])

src/ocrmypdf/_exec/pngquant.py

@@ -9,10 +9,8 @@
 
 from contextlib import contextmanager
 from io import BytesIO
-from os import fspath
 from pathlib import Path
 from subprocess import PIPE
-from tempfile import NamedTemporaryFile
 
 from PIL import Image
 
@@ -61,3 +59,7 @@ def quantize(input_file: Path, output_file: Path, quality_min: int, quality_max:
     if result.returncode == 0:
         # input_file could be the same as output_file, so we defer the write
         output_file.write_bytes(result.stdout)
+
+
+def quantize_mp(args):
+    return quantize(*args)

src/ocrmypdf/_exec/tesseract.py

@@ -9,12 +9,14 @@
 
 import logging
 import os
+import re
 import shutil
 from collections import namedtuple
+from distutils.version import StrictVersion
 from os import fspath
 from pathlib import Path
 from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired
-from typing import List
+from typing import List, Optional
 
 from PIL import Image
 
@@ -53,31 +55,31 @@ class TesseractLoggerAdapter(logging.LoggerAdapter):
         return '[tesseract] %s' % (msg), kwargs
 
 
+class TesseractVersion(StrictVersion):
+
+    version_re = re.compile(
+        r'''
+        ^(\d+) \. (\d+) (\. (\d+))?  # groups: 1/major, 2/minor, 3/[skip], 4/patch
+        [-]?  # optional hyphen separator
+        (?:(alpha|beta|rc|dev)?[.\-\ ]?(\d+)?)?  # 5/prerelease, 6/prerelease_num
+        (?:-(\d+)-g[0-9a-f]+)?  # untagged git version
+        $
+        ''',
+        re.VERBOSE | re.ASCII,
+    )
+
+    def parse(self, vstring):
+        try:
+            super().parse(vstring)
+        except TypeError as e:
+            if 'int() argument must be a string' in str(e):
+                super().parse(vstring + '0')
+
+
 def version():
     return get_version('tesseract', regex=r'tesseract\s(.+)')
 
 
-def has_textonly_pdf(langs=None):
-    """Does Tesseract have textonly_pdf capability?
-
-    Available in v4.00.00alpha since January 2017. Best to
-    parse the parameter list.
-    """
-    args_tess = tess_base_args(langs, engine_mode=None) + ['--print-parameters', 'pdf']
-    params = ''
-    try:
-        proc = run(args_tess, check=True, stdout=PIPE, stderr=STDOUT)
-        params = proc.stdout
-    except CalledProcessError as e:
-        raise MissingDependencyError(
-            "Could not --print-parameters from tesseract. This can happen if the "
-            "TESSDATA_PREFIX environment is not set to a valid tessdata folder. "
-        ) from e
-    if b'textonly_pdf' in params:
-        return True
-    return False
-
-
 def has_user_words():
     """Does Tesseract have --user-words capability?
 
@@ -100,7 +102,12 @@ def get_languages():
     args_tess = ['tesseract', '--list-langs']
     try:
         proc = run(
-            args_tess, universal_newlines=True, stdout=PIPE, stderr=STDOUT, check=True
+            args_tess,
+            text=True,
+            stdout=PIPE,
+            stderr=STDOUT,
+            logs_errors_to_stdout=True,
+            check=True,
         )
         output = proc.stdout
     except CalledProcessError as e:
@@ -113,7 +120,7 @@ def get_languages():
     return set(lang.strip() for lang in rest)
 
 
-def tess_base_args(langs: List[str], engine_mode: int) -> List[str]:
+def tess_base_args(langs: List[str], engine_mode: Optional[int]) -> List[str]:
     args = ['tesseract']
     if langs:
         args.extend(['-l', '+'.join(langs)])
@@ -122,7 +129,7 @@ def tess_base_args(langs: List[str], engine_mode: int) -> List[str]:
     return args
 
 
-def get_orientation(input_file: Path, engine_mode: int, timeout: float):
+def get_orientation(input_file: Path, engine_mode: Optional[int], timeout: float):
     args_tesseract = tess_base_args(['osd'], engine_mode) + [
         '--psm',
         '0',
@@ -216,6 +223,7 @@ def _generate_null_hocr(output_hocr, output_text, image):
 
 
 def generate_hocr(
+    *,
     input_file: Path,
     output_hocr: Path,
     output_text: Path,

src/ocrmypdf/_exec/unpaper.py

@@ -13,10 +13,11 @@
 import logging
 import os
 import shlex
+from decimal import Decimal
 from pathlib import Path
-from subprocess import PIPE, STDOUT, CalledProcessError
+from subprocess import PIPE, STDOUT
 from tempfile import TemporaryDirectory
-from typing import Tuple
+from typing import List, Optional, Tuple, Union
 
 from PIL import Image
 
@@ -24,10 +25,12 @@ from ocrmypdf.exceptions import MissingDependencyError, SubprocessOutputError
 from ocrmypdf.subprocess import get_version
 from ocrmypdf.subprocess import run as external_run
 
+DecFloat = Union[Decimal, float]
+
 log = logging.getLogger(__name__)
 
 
-def version():
+def version() -> str:
     return get_version('unpaper')
 
 
@@ -53,23 +56,25 @@ def _setup_unpaper_io(tmpdir: Path, input_file: Path) -> Tuple[Path, Path]:
         except KeyError:
             raise MissingDependencyError(
                 "Failed to convert image to a supported format."
-            ) from e
+            ) from None
 
-        if im_modified or input_file.suffix != '.png':
-            input_png = tmpdir / 'input.png'
-            im.save(input_png, format='PNG', compress_level=1)
+        if im_modified or input_file.suffix != '.pnm':
+            input_pnm = tmpdir / 'input.pnm'
+            im.save(input_pnm, format='PPM')
         else:
             # No changes, PNG input, just use the file we already have
-            input_png = input_file
+            input_pnm = input_file
         output_pnm = tmpdir / f'output{suffix}'
-    return input_png, output_pnm
+    return input_pnm, output_pnm
 
 
-def run(input_file, output_file, dpi, mode_args):
-    args_unpaper = ['unpaper', '-v', '--dpi', str(dpi)] + mode_args
+def run(
+    input_file: Path, output_file: Path, *, dpi: DecFloat, mode_args: List[str]
+) -> None:
+    args_unpaper = ['unpaper', '-v', '--dpi', str(round(dpi, 6))] + mode_args
 
     with TemporaryDirectory() as tmpdir:
-        input_png, output_pnm = _setup_unpaper_io(Path(tmpdir), input_file)
+        input_pnm, output_pnm = _setup_unpaper_io(Path(tmpdir), input_file)
 
         # To prevent any shenanigans from accepting arbitrary parameters in
         # --unpaper-args, we:
@@ -78,41 +83,41 @@ def run(input_file, output_file, dpi, mode_args):
         # 3) append absolute paths for the input and output file
         # This should ensure that a user cannot clobber some other file with
         # their unpaper arguments (whether intentionally or otherwise)
-        args_unpaper.extend([os.fspath(input_png), os.fspath(output_pnm)])
+        args_unpaper.extend([os.fspath(input_pnm), os.fspath(output_pnm)])
+        external_run(
+            args_unpaper,
+            close_fds=True,
+            check=True,
+            stderr=STDOUT,  # unpaper writes logging output to stdout and stderr
+            stdout=PIPE,  # and cannot send file output to stdout
+            cwd=tmpdir,
+            logs_errors_to_stdout=True,
+        )
         try:
-            proc = external_run(
-                args_unpaper,
-                check=True,
-                close_fds=True,
-                universal_newlines=True,
-                stderr=STDOUT,  # unpaper writes logging output to stdout and stderr
-                cwd=tmpdir,  # and cannot send file output to stdout
-                stdout=PIPE,
-            )
-        except CalledProcessError as e:
-            log.debug(e.stderr)
-            raise e from e
-        else:
-            log.debug(proc.stderr)
-            try:
-                with Image.open(output_pnm) as imout:
-                    imout.save(output_file, dpi=(dpi, dpi))
-            except (FileNotFoundError, OSError):
-                raise SubprocessOutputError(
-                    "unpaper: failed to produce the expected output file. "
-                    + " Called with: "
-                    + str(args_unpaper)
-                ) from None
+            with Image.open(output_pnm) as imout:
+                imout.save(output_file, dpi=(dpi, dpi))
+        except (FileNotFoundError, OSError):
+            raise SubprocessOutputError(
+                "unpaper: failed to produce the expected output file. "
+                + " Called with: "
+                + str(args_unpaper)
+            ) from None
 
 
-def validate_custom_args(args: str):
+def validate_custom_args(args: str) -> List[str]:
     unpaper_args = shlex.split(args)
     if any(('/' in arg or arg == '.' or arg == '..') for arg in unpaper_args):
         raise ValueError('No filenames allowed in --unpaper-args')
     return unpaper_args
 
 
-def clean(input_file, output_file, dpi, unpaper_args=None):
+def clean(
+    input_file: Path,
+    output_file: Path,
+    *,
+    dpi: DecFloat,
+    unpaper_args: Optional[List[str]] = None,
+):
     default_args = [
         '--layout',
         'none',
@@ -126,4 +131,4 @@ def clean(input_file, output_file, dpi, unpaper_args=None):
     ]
     if not unpaper_args:
         unpaper_args = default_args
-    run(input_file, output_file, dpi, unpaper_args)
+    run(input_file, output_file, dpi=dpi, mode_args=unpaper_args)

src/ocrmypdf/_graft.py

@@ -6,33 +6,39 @@
 
 
 import logging
+import uuid
 from contextlib import suppress
 from pathlib import Path
 from typing import Optional
 
 import pikepdf
+from pikepdf.objects import Dictionary, Name
 
 log = logging.getLogger(__name__)
 MAX_REPLACE_PAGES = 100
 
 
-def _update_page_resources(*, page, font, font_key, procset):
-    """Update this page's fonts with a reference to the Glyphless font"""
+def _ensure_dictionary(obj, name):
+    if name not in obj:
+        obj[name] = Dictionary({})
+    return obj[name]
 
-    if '/Resources' not in page:
-        page['/Resources'] = pikepdf.Dictionary({})
-    resources = page['/Resources']
-    try:
-        fonts = resources['/Font']
-    except KeyError:
-        fonts = pikepdf.Dictionary({})
+
+def _update_resources(*, obj, font, font_key, procset):
+    """Update this obj's fonts with a reference to the Glyphless font.
+
+    obj can be a page or Form XObject.
+    """
+
+    resources = _ensure_dictionary(obj, Name.Resources)
+    fonts = _ensure_dictionary(resources, Name.Font)
     if font_key is not None and font_key not in fonts:
         fonts[font_key] = font
-    resources['/Font'] = fonts
 
     # Reassign /ProcSet to one that just lists everything - ProcSet is
     # obsolete and doesn't matter but recommended for old viewer support
-    resources['/ProcSet'] = procset
+    if procset:
+        resources['/ProcSet'] = procset
 
 
 def strip_invisible_text(pdf, page):
@@ -41,7 +47,8 @@ def strip_invisible_text(pdf, page):
     render_mode = 0
     text_objects = []
 
-    page.page_contents_coalesce()
+    rich_page = pikepdf.Page(page)
+    rich_page.contents_coalesce()
     for operands, operator in pikepdf.parse_content_stream(page, ''):
         if not in_text_obj:
             if operator == pikepdf.Operator('BT'):
@@ -169,13 +176,13 @@ class OcrGrafter:
         """
 
         page0 = self.pdf_base.pages[0]
-        _update_page_resources(
-            page=page0, font=self.font, font_key=self.font_key, procset=self.procset
+        _update_resources(
+            obj=page0, font=self.font, font_key=self.font_key, procset=self.procset
         )
 
         # We cannot read and write the same file, that will corrupt it
         # but we don't to keep more copies than we need to. Delete intermediates.
-        # {interim_count} is the opened file we were updateing
+        # {interim_count} is the opened file we were updating
         # {interim_count - 1} can be deleted
         # {interim_count + 1} is the new file will produce and open
         old_file = self.output_file.with_suffix(f'.working{self.interim_count - 1}.pdf')
@@ -210,6 +217,7 @@ class OcrGrafter:
                     pdf_text_fonts = pdf_text.pages[0].Resources.get('/Font', {})
                 except (AttributeError, IndexError, KeyError):
                     return None, None
+                pdf_text_font = None
                 for f in possible_font_names:
                     pdf_text_font = pdf_text_fonts.get(f, None)
                     if pdf_text_font is not None:
@@ -279,17 +287,36 @@ class OcrGrafter:
             # finally move the lower left corner to match the mediabox
             ctm = translate @ rotate @ scale @ untranslate @ corner
 
-            pdf_text_contents = (
-                b'q %s cm\n' % ctm.encode() + pdf_text_contents + b'\nQ\n'
+            base_resources = _ensure_dictionary(base_page, Name.Resources)
+            base_xobjs = _ensure_dictionary(base_resources, Name.XObject)
+            text_xobj_name = Name('/' + str(uuid.uuid4()))
+            xobj = self.pdf_base.make_stream(pdf_text_contents)
+            base_xobjs[text_xobj_name] = xobj
+            xobj.Type = Name.XObject
+            xobj.Subtype = Name.Form
+            xobj.FormType = 1
+            xobj.BBox = mediabox
+            _update_resources(
+                obj=xobj, font=font, font_key=font_key, procset=[Name.PDF]
             )
 
-            new_text_layer = pikepdf.Stream(self.pdf_base, pdf_text_contents)
+            pdf_draw_xobj = (
+                (b'q %s cm\n' % ctm.encode()) + (b'%s Do\n' % text_xobj_name) + b'\nQ\n'
+            )
+            new_text_layer = pikepdf.Stream(self.pdf_base, pdf_draw_xobj)
 
             if strip_old_text:
                 strip_invisible_text(self.pdf_base, base_page)
 
-            base_page.page_contents_add(new_text_layer, prepend=True)
+            if hasattr(pikepdf.Page, 'contents_add'):
+                # pikepdf >= 2.14 adds this method and deprecates the one below
+                pikepdf.Page(base_page).contents_add(new_text_layer, prepend=True)
+            else:
+                # pikepdf < 2.14
+                base_page.page_contents_add(
+                    new_text_layer, prepend=True
+                )  # pragma: no cover
 
-            _update_page_resources(
-                page=base_page, font=font, font_key=font_key, procset=procset
+            _update_resources(
+                obj=base_page, font=font, font_key=font_key, procset=procset
             )

src/ocrmypdf/_jobcontext.py

@@ -10,15 +10,22 @@ import shutil
 import sys
 from argparse import Namespace
 from copy import copy
-from io import IOBase
 from pathlib import Path
 from typing import Iterator
 
+from pluggy import PluginManager
+
 from ocrmypdf.pdfinfo import PdfInfo
+from ocrmypdf.pdfinfo.info import PageInfo
 
 
 class PdfContext:
-    """Holds our context for a particular run of the pipeline"""
+    """Holds the context for a particular run of the pipeline."""
+
+    options: Namespace  #: The specified options for processing this PDF.
+    origin: Path  #: The filename of the original input file.
+    pdfinfo: PdfInfo  #: Detailed data for this PDF.
+    plugin_manager: PluginManager  #: PluginManager for processing the current PDF.
 
     def __init__(
         self,
@@ -35,21 +42,33 @@ class PdfContext:
         self.plugin_manager = plugin_manager
 
     def get_path(self, name: str) -> Path:
+        """Generate a ``Path`` for an intermediate file involved in processing.
+
+        The path will be in a temporary folder that is common for all processing
+        of this particular PDF.
+        """
         return self.work_folder / name
 
     def get_page_contexts(self) -> Iterator['PageContext']:
+        """Get all ``PageContext`` for this PDF."""
         npages = len(self.pdfinfo)
         for n in range(npages):
             yield PageContext(self, n)
 
 
 class PageContext:
-    """Holds our context for a page
+    """Holds our context for a page.
 
     Must be pickable, so stores only intrinsic/simple data elements or those
-    capable of their serializing themselves via __getstate__.
+    capable of their serializing themselves via ``__getstate__``.
     """
 
+    options: Namespace  #: The specified options for processing this PDF.
+    origin: Path  #: The filename of the original input file.
+    pageno: int  #: This page number (zero-based).
+    pageinfo: PageInfo  #: Information on this page.
+    plugin_manager: PluginManager  #: PluginManager for processing the current PDF.
+
     def __init__(self, pdf_context: PdfContext, pageno):
         self.work_folder = pdf_context.work_folder
         self.origin = pdf_context.origin
@@ -59,6 +78,11 @@ class PageContext:
         self.plugin_manager = pdf_context.plugin_manager
 
     def get_path(self, name: str) -> Path:
+        """Generate a ``Path`` for a file that is part of processing this page.
+
+        The path will be based in a common temporary folder and have a prefix based
+        on the page number.
+        """
         return self.work_folder / ("%06d_%s" % (self.pageno + 1, name))
 
     def __getstate__(self):

src/ocrmypdf/_pipeline.py

@@ -13,7 +13,7 @@ from contextlib import suppress
 from datetime import datetime, timezone
 from pathlib import Path
 from shutil import copyfileobj
-from typing import BinaryIO, Dict, Iterable, Optional, Union, cast
+from typing import Dict, Iterable, Optional
 
 import img2pdf
 import pikepdf
@@ -21,6 +21,7 @@ from pikepdf.models.metadata import encode_pdf_date
 from PIL import Image, ImageColor, ImageDraw
 
 from ocrmypdf import leptonica
+from ocrmypdf._concurrent import Executor
 from ocrmypdf._exec import unpaper
 from ocrmypdf._jobcontext import PageContext, PdfContext
 from ocrmypdf._version import PROGRAM_NAME
@@ -145,11 +146,13 @@ def triage(original_filename, input_file, output_file, options):
 
 def get_pdfinfo(
     input_file,
+    *,
+    executor: Executor,
     detailed_analysis=False,
     progbar=False,
     max_workers=None,
     check_pages=None,
-):
+) -> PdfInfo:
     try:
         return PdfInfo(
             input_file,
@@ -157,11 +160,12 @@ def get_pdfinfo(
             progbar=progbar,
             max_workers=max_workers,
             check_pages=check_pages,
+            executor=executor,
         )
-    except pikepdf.PasswordError:
-        raise EncryptedPdfError()
-    except pikepdf.PdfError:
-        raise InputFileError()
+    except pikepdf.PasswordError as e:
+        raise EncryptedPdfError() from e
+    except pikepdf.PdfError as e:
+        raise InputFileError() from e
 
 
 def validate_pdfinfo_options(context: PdfContext):
@@ -205,17 +209,21 @@ def validate_pdfinfo_options(context: PdfContext):
     context.plugin_manager.hook.validate(pdfinfo=pdfinfo, options=options)
 
 
+def _vector_page_dpi(pageinfo):
+    return VECTOR_PAGE_DPI if pageinfo.has_vector or pageinfo.has_text else 0.0
+
+
 def get_page_dpi(pageinfo, options):
     "Get the DPI when nonsquare DPI is tolerable"
     xres = max(
         pageinfo.dpi.x or VECTOR_PAGE_DPI,
         options.oversample or 0.0,
-        VECTOR_PAGE_DPI if pageinfo.has_vector else 0.0,
+        _vector_page_dpi(pageinfo),
     )
     yres = max(
         pageinfo.dpi.y or VECTOR_PAGE_DPI,
         options.oversample or 0,
-        VECTOR_PAGE_DPI if pageinfo.has_vector else 0.0,
+        _vector_page_dpi(pageinfo),
     )
     return Resolution(float(xres), float(yres))
 
@@ -229,7 +237,7 @@ def get_page_square_dpi(pageinfo, options) -> Resolution:
         max(
             (xres * userunit) or VECTOR_PAGE_DPI,
             (yres * userunit) or VECTOR_PAGE_DPI,
-            VECTOR_PAGE_DPI if pageinfo.has_vector else 0.0,
+            _vector_page_dpi(pageinfo),
             options.oversample or 0.0,
         )
     )
@@ -242,7 +250,7 @@ def get_canvas_square_dpi(pageinfo, options) -> Resolution:
         max(
             (pageinfo.dpi.x) or VECTOR_PAGE_DPI,
             (pageinfo.dpi.y) or VECTOR_PAGE_DPI,
-            VECTOR_PAGE_DPI if pageinfo.has_vector else 0.0,
+            _vector_page_dpi(pageinfo),
             options.oversample or 0.0,
         )
     )
@@ -475,7 +483,12 @@ def preprocess_deskew(input_file: Path, page_context: PageContext):
 def preprocess_clean(input_file: Path, page_context: PageContext):
     output_file = page_context.get_path('pp_clean.png')
     dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
-    unpaper.clean(input_file, output_file, dpi.x, page_context.options.unpaper_args)
+    unpaper.clean(
+        input_file,
+        output_file,
+        dpi=dpi.x,
+        unpaper_args=page_context.options.unpaper_args,
+    )
     return output_file
 
 
@@ -536,9 +549,6 @@ def create_ocr_image(image: Path, page_context: PageContext):
 
         # Pillow requires integer DPI
         dpi = tuple(round(coord) for coord in im.info['dpi'])
-        if page_context.pageinfo.rotation != 0:
-            log.info(f"Rotating {page_context.pageinfo.rotation}")
-            im = im.rotate(page_context.pageinfo.rotation)
         im.save(output_file, dpi=dpi)
     return output_file
 
@@ -582,7 +592,9 @@ def create_visible_page_jpg(image: Path, page_context: PageContext) -> Path:
     return output_file
 
 
-def create_pdf_page_from_image(image: Path, page_context: PageContext):
+def create_pdf_page_from_image(
+    image: Path, page_context: PageContext, orientation_correction
+):
     # We rasterize a square DPI version of each page because most image
     # processing tools don't support rectangular DPI. Use the square DPI as it
     # accurately describes the image. It would be possible to resample the image
@@ -590,28 +602,42 @@ def create_pdf_page_from_image(image: Path, page_context: PageContext):
     # except that the hocr renderer does not understand non-square DPI. The
     # sandwich renderer would be fine.
     output_file = page_context.get_path('visible.pdf')
-    dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
-    layout_fun = img2pdf.get_fixed_dpi_layout_fun(dpi)
+
+    pageinfo = page_context.pageinfo
+    pagesize = 72.0 * float(pageinfo.width_inches), 72.0 * float(pageinfo.height_inches)
+    effective_rotation = (pageinfo.rotation - orientation_correction) % 360
+    if effective_rotation % 180 == 90:
+        pagesize = pagesize[1], pagesize[0]
 
     # This create a single page PDF
     with open(image, 'rb') as imfile, open(output_file, 'wb') as pdf:
         log.debug('convert')
+
+        layout_fun = img2pdf.get_layout_fun(pagesize)
         img2pdf.convert(
             imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf
         )
         log.debug('convert done')
+
+    output_file = page_context.plugin_manager.hook.filter_pdf_page(
+        page=page_context, image_filename=image, output_pdf=output_file
+    )
+
     return output_file
 
 
 def render_hocr_page(hocr: Path, page_context: PageContext):
+    options = page_context.options
     output_file = page_context.get_path('ocr_hocr.pdf')
-    dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
-    hocrtransform = HocrTransform(hocr, dpi.x)  # square
+    dpi = get_page_square_dpi(page_context.pageinfo, options)
+    debug_mode = options.pdf_renderer == 'hocrdebug'
+
+    hocrtransform = HocrTransform(hocr_filename=hocr, dpi=dpi.x)  # square
     hocrtransform.to_pdf(
-        output_file,
+        out_filename=output_file,
         image_filename=None,
-        show_bounding_boxes=False,
-        invisible_text=True,
+        show_bounding_boxes=False if not debug_mode else True,
+        invisible_text=True if not debug_mode else False,
         interword_spaces=True,
     )
     return output_file
@@ -660,11 +686,6 @@ def get_docinfo(base_pdf: pikepdf.Pdf, context: PdfContext) -> Dict[str, str]:
 
     pdfmark['/Creator'] = f'{PROGRAM_NAME} {VERSION} / {creator_tag}'
     pdfmark['/Producer'] = f'pikepdf {pikepdf.__version__}'
-    if 'OCRMYPDF_CREATOR' in os.environ:
-        pdfmark['/Creator'] = os.environ['OCRMYPDF_CREATOR']
-    if 'OCRMYPDF_PRODUCER' in os.environ:
-        pdfmark['/Producer'] = os.environ['OCRMYPDF_PRODUCER']
-
     pdfmark['/ModDate'] = encode_pdf_date(datetime.now(timezone.utc))
     return pdfmark
 
@@ -712,6 +733,11 @@ def convert_to_pdfa(input_pdf: Path, input_ps_stub: Path, context: PdfContext):
         output_file=output_file,
         compression=options.pdfa_image_compression,
         pdfa_part=options.output_type[-1],  # is pdfa-1, pdfa-2, or pdfa-3
+        progressbar_class=(
+            context.plugin_manager.hook.get_progressbar_class()
+            if options.progress_bar
+            else None
+        ),
     )
 
     return output_file
@@ -724,6 +750,24 @@ def should_linearize(working_file: Path, context: PdfContext):
     return False
 
 
+def get_pdf_save_settings(output_type: str):
+    if output_type == 'pdfa-1':
+        # Trigger recompression to ensure object streams are removed, because
+        # Acrobat complains about them in PDF/A-1b validation.
+        return dict(
+            preserve_pdfa=True,
+            compress_streams=True,
+            stream_decode_level=pikepdf.StreamDecodeLevel.generalized,
+            object_stream_mode=pikepdf.ObjectStreamMode.disable,
+        )
+    else:
+        return dict(
+            preserve_pdfa=True,
+            compress_streams=True,
+            object_stream_mode=(pikepdf.ObjectStreamMode.generate),
+        )
+
+
 def metadata_fixup(working_file: Path, context: PdfContext):
     output_file = context.get_path('metafix.pdf')
     options = context.options
@@ -754,25 +798,21 @@ def metadata_fixup(working_file: Path, context: PdfContext):
             if 'xmp:CreateDate' not in meta:
                 meta['xmp:CreateDate'] = meta.get('xmp:ModifyDate', '')
 
-            # Ghostscript likes to set title to Untitled if omitted from input.
-            # Reverse this, because PDF/A TechNote 0003:Metadata in PDF/A-1
-            # and the XMP Spec do not make this recommendation.
-            if meta.get('dc:title') == 'Untitled':
-                with original.open_metadata(
-                    set_pikepdf_as_editor=False, update_docinfo=False
-                ) as original_meta:
-                    if 'dc:title' not in original_meta:
+            with original.open_metadata(
+                set_pikepdf_as_editor=False, update_docinfo=False, strict=False
+            ) as meta_original:
+                if meta.get('dc:title') == 'Untitled':
+                    # Ghostscript likes to set title to Untitled if omitted from input.
+                    # Reverse this, because PDF/A TechNote 0003:Metadata in PDF/A-1
+                    # and the XMP Spec do not make this recommendation.
+                    if 'dc:title' not in meta_original:
                         del meta['dc:title']
-
-            meta_original = original.open_metadata()
-            missing = set(meta_original.keys()) - set(meta.keys())
-            report_on_metadata(missing)
+                missing = set(meta_original.keys()) - set(meta.keys())
+                report_on_metadata(missing)
 
         pdf.save(
             output_file,
-            compress_streams=True,
-            preserve_pdfa=True,
-            object_stream_mode=pikepdf.ObjectStreamMode.generate,
+            **get_pdf_save_settings(options.output_type),
             linearize=(  # Don't linearize if optimize() will be linearizing too
                 should_linearize(working_file, context)
                 if options.optimize == 0
@@ -783,23 +823,37 @@ def metadata_fixup(working_file: Path, context: PdfContext):
     return output_file
 
 
-def optimize_pdf(input_file: Path, context: PdfContext):
+def optimize_pdf(input_file: Path, context: PdfContext, executor: Executor):
     output_file = context.get_path('optimize.pdf')
     save_settings = dict(
-        compress_streams=True,
-        preserve_pdfa=True,
-        object_stream_mode=pikepdf.ObjectStreamMode.generate,
         linearize=should_linearize(input_file, context),
+        **get_pdf_save_settings(context.options.output_type),
     )
-    optimize(input_file, output_file, context, save_settings)
+    optimize(input_file, output_file, context, save_settings, executor)
     return output_file
 
 
+def enumerate_compress_ranges(iterable):
+    skipped_from, index = None, None
+    for index, txt_file in enumerate(iterable):
+        index += 1
+        if txt_file:
+            if skipped_from is not None:
+                yield (skipped_from, index - 1), None
+                skipped_from = None
+            yield (index, index), txt_file
+        else:
+            if skipped_from is None:
+                skipped_from = index
+    if skipped_from is not None:
+        yield (skipped_from, index), None
+
+
 def merge_sidecars(txt_files: Iterable[Optional[Path]], context: PdfContext):
     output_file = context.get_path('sidecar.txt')
     with open(output_file, 'w', encoding="utf-8") as stream:
-        for page_num, txt_file in enumerate(txt_files):
-            if page_num != 0:
+        for (frm, to), txt_file in enumerate_compress_ranges(txt_files):
+            if frm != 1:
                 stream.write('\f')  # Form feed between pages
             if txt_file:
                 with open(txt_file, 'r', encoding="utf-8") as in_:
@@ -812,7 +866,11 @@ def merge_sidecars(txt_files: Iterable[Optional[Path]], context: PdfContext):
                     else:
                         stream.write(txt)
             else:
-                stream.write(f'[OCR skipped on page {(page_num + 1)}]')
+                if frm != to:
+                    pages = f'{frm}-{to}'
+                else:
+                    pages = f'{frm}'
+                stream.write(f'[OCR skipped on page(s) {pages}]')
     return output_file
 
 

src/ocrmypdf/_plugin_manager.py

@@ -8,13 +8,14 @@
 import argparse
 import importlib
 import importlib.util
+import pkgutil
 import sys
-from functools import partial
 from pathlib import Path
-from typing import Callable, List, Tuple, Union
+from typing import List, Tuple, Union
 
 import pluggy
 
+import ocrmypdf.builtin_plugins
 from ocrmypdf import pluginspec
 from ocrmypdf.cli import get_parser, plugins_only_parser
 
@@ -30,62 +31,80 @@ class OcrmypdfPluginManager(pluggy.PluginManager):
     """
 
     def __init__(
-        self, *args, setup_func: Callable[[pluggy.PluginManager], None], **kwargs
+        self,
+        *args,
+        plugins: List[Union[str, Path]],
+        builtins: bool = True,
+        **kwargs,
     ):
-        self._init_args = args
-        self._setup_func = setup_func
-        self._init_kwargs = kwargs
+        self.__init_args = args
+        self.__init_kwargs = kwargs
+        self.__plugins = plugins
+        self.__builtins = builtins
         super().__init__(*args, **kwargs)
-        setup_func(self)
+        self.setup_plugins()
 
     def __getstate__(self):
         state = dict(
-            _init_args=self._init_args,
-            _setup_func=self._setup_func,
-            _init_kwargs=self._init_kwargs,
+            init_args=self.__init_args,
+            plugins=self.__plugins,
+            builtins=self.__builtins,
+            init_kwargs=self.__init_kwargs,
         )
         return state
 
     def __setstate__(self, state):
         self.__init__(
-            *state['_init_args'],
-            setup_func=state['_setup_func'],
-            **state['_init_kwargs'],
+            *state['init_args'],
+            plugins=state['plugins'],
+            builtins=state['builtins'],
+            **state['init_kwargs'],
         )
 
+    def setup_plugins(self):
+        self.add_hookspecs(pluginspec)
 
-def _setup_plugins(
-    pm: pluggy.PluginManager, plugins: List[Union[str, Path]], builtins: bool = True
-):
-    pm.add_hookspecs(pluginspec)
+        # 1. Register builtins
+        if self.__builtins:
+            for module in sorted(
+                pkgutil.iter_modules(ocrmypdf.builtin_plugins.__path__)
+            ):
+                name = f'ocrmypdf.builtin_plugins.{module.name}'
+                module = importlib.import_module(name)
+                self.register(module)
 
-    all_plugins: List[Union[str, Path]] = []
-    if builtins:
-        all_plugins.extend(
-            [
-                'ocrmypdf.builtin_plugins.ghostscript',
-                'ocrmypdf.builtin_plugins.tesseract_ocr',
-            ]
-        )
-    all_plugins.extend(plugins)
-    for name in all_plugins:
-        if isinstance(name, Path) or name.endswith('.py'):
-            # Import by filename
-            module_name = Path(name).stem
-            spec = importlib.util.spec_from_file_location(module_name, name)
-            module = importlib.util.module_from_spec(spec)
-            sys.modules[module_name] = module
-            spec.loader.exec_module(module)
-        else:
-            # Import by dotted module name
-            module = importlib.import_module(name)
-        pm.register(module)
+        # 2. Install semfree if needed
+        try:
+            # pylint: disable=import-outside-toplevel
+            from multiprocessing.synchronize import SemLock
+
+            del SemLock
+        except ImportError:
+            self.register(importlib.import_module('ocrmypdf.extra_plugins.semfree'))
+
+        # 3. Register setuptools plugins
+        self.load_setuptools_entrypoints('ocrmypdf')
+
+        # 4. Register plugins specified on command line
+        for name in self.__plugins:
+            if isinstance(name, Path) or name.endswith('.py'):
+                # Import by filename
+                module_name = Path(name).stem
+                spec = importlib.util.spec_from_file_location(module_name, name)
+                module = importlib.util.module_from_spec(spec)
+                sys.modules[module_name] = module
+                spec.loader.exec_module(module)
+            else:
+                # Import by dotted module name
+                module = importlib.import_module(name)
+            self.register(module)
 
 
-def get_plugin_manager(plugins: List[str], builtins=True):
+def get_plugin_manager(plugins: List[Union[str, Path]], builtins=True):
     pm = OcrmypdfPluginManager(
         project_name='ocrmypdf',
-        setup_func=partial(_setup_plugins, plugins=plugins, builtins=builtins),
+        plugins=plugins,
+        builtins=builtins,
     )
     return pm
 

src/ocrmypdf/_sync.py

@@ -15,10 +15,9 @@ from pathlib import Path
 from tempfile import mkdtemp
 from typing import List, NamedTuple, Optional, Tuple
 
-import pikepdf
 import PIL
 
-from ocrmypdf._concurrent import exec_progress_pool
+from ocrmypdf._concurrent import Executor, setup_executor
 from ocrmypdf._graft import OcrGrafter
 from ocrmypdf._jobcontext import PageContext, PdfContext, cleanup_working_files
 from ocrmypdf._logging import PageNumberFilter
@@ -55,6 +54,7 @@ from ocrmypdf._validation import (
 )
 from ocrmypdf.exceptions import ExitCode, ExitCodeException
 from ocrmypdf.helpers import (
+    NeverRaise,
     available_cpu_count,
     check_pdf,
     pikepdf_enable_mmap,
@@ -65,7 +65,7 @@ from ocrmypdf.pdfa import file_claims_pdfa
 log = logging.getLogger(__name__)
 
 
-class PageResult(NamedTuple):
+class PageResult(NamedTuple):  # pylint: disable=inherit-non-class
     pageno: int
     pdf_page_from_image: Optional[Path]
     ocr: Optional[Path]
@@ -203,10 +203,10 @@ def exec_page_sync(page_context: PageContext):
         if filtered_image:
             visible_image_out = filtered_image
         pdf_page_from_image_out = create_pdf_page_from_image(
-            visible_image_out, page_context
+            visible_image_out, page_context, orientation_correction
         )
 
-    if options.pdf_renderer == 'hocr':
+    if options.pdf_renderer.startswith('hocr'):
         (hocr_out, text_out) = ocr_engine_hocr(ocr_image_out, page_context)
         ocr_out = render_hocr_page(hocr_out, page_context)
     elif options.pdf_renderer == 'sandwich':
@@ -223,14 +223,14 @@ def exec_page_sync(page_context: PageContext):
     )
 
 
-def post_process(pdf_file, context: PdfContext):
+def post_process(pdf_file, context: PdfContext, executor: Executor):
     pdf_out = pdf_file
     if context.options.output_type.startswith('pdfa'):
         ps_stub_out = generate_postscript_stub(context)
         pdf_out = convert_to_pdfa(pdf_out, ps_stub_out, context)
 
     pdf_out = metadata_fixup(pdf_out, context)
-    return optimize_pdf(pdf_out, context)
+    return optimize_pdf(pdf_out, context, executor)
 
 
 def worker_init(max_pixels: int):
@@ -241,7 +241,7 @@ def worker_init(max_pixels: int):
     pikepdf_enable_mmap()
 
 
-def exec_concurrent(context: PdfContext):
+def exec_concurrent(context: PdfContext, executor: Executor):
     """Execute the pipeline concurrently"""
 
     # Run exec_page_sync on every page context
@@ -268,7 +268,7 @@ def exec_concurrent(context: PdfContext):
         finally:
             tls.pageno = None
 
-    exec_progress_pool(
+    executor(
         use_threads=options.use_threads,
         max_workers=max_workers,
         tqdm_kwargs=dict(
@@ -278,7 +278,7 @@ def exec_concurrent(context: PdfContext):
             unit_scale=0.5,
             disable=not options.progress_bar,
         ),
-        task_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
+        worker_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
         task=exec_page_sync,
         task_arguments=context.get_page_contexts(),
         task_finished=update_page,
@@ -295,19 +295,20 @@ def exec_concurrent(context: PdfContext):
 
     # PDF/A and metadata
     log.info("Postprocessing...")
-    pdf = post_process(pdf, context)
+    pdf = post_process(pdf, context, executor)
 
     # Copy PDF file to destination
     copy_final(pdf, options.output_file, context)
 
 
-class NeverRaise(Exception):
-    """An exception that is never raised"""
+def configure_debug_logging(log_filename: Path, prefix: str = ''):
+    """
+    Create a debug log file at a specified location.
 
-    pass  # pylint: disable=unnecessary-pass
-
-
-def configure_debug_logging(log_filename, prefix=''):
+    Arguments:
+        log_filename: Where to the put the log file.
+        prefix: The logging domain prefix that should be sent to the log.
+    """
     log_file_handler = logging.FileHandler(log_filename, delay=True)
     log_file_handler.setLevel(logging.DEBUG)
     formatter = logging.Formatter(
@@ -328,7 +329,7 @@ def run_pipeline(options, *, plugin_manager, api=False):
     if not plugin_manager:
         plugin_manager = get_plugin_manager(options.plugins)
 
-    work_folder = Path(mkdtemp(prefix="com.github.ocrmypdf."))
+    work_folder = Path(mkdtemp(prefix="ocrmypdf.io."))
     debug_log_handler = None
     if (
         (options.keep_temporary_files or options.verbose >= 1)
@@ -336,10 +337,15 @@ def run_pipeline(options, *, plugin_manager, api=False):
         and not api
     ):
         # Debug log for command line interface only with verbose output
-        debug_log_handler = configure_debug_logging(Path(work_folder) / "debug.log")
+        # See https://github.com/pytest-dev/pytest/issues/5502 for why we skip this
+        # when pytest is running
+        debug_log_handler = configure_debug_logging(
+            Path(work_folder) / "debug.log"
+        )  # pragma: no cover
 
     pikepdf_enable_mmap()
 
+    executor = setup_executor(plugin_manager)
     try:
         check_requested_output_file(options)
         start_input_file, original_filename = create_input_file(options, work_folder)
@@ -352,6 +358,7 @@ def run_pipeline(options, *, plugin_manager, api=False):
         # Gather pdfinfo and create context
         pdfinfo = get_pdfinfo(
             origin_pdf,
+            executor=executor,
             detailed_analysis=options.redo_ocr,
             progbar=options.progress_bar,
             max_workers=options.jobs if not options.use_threads else 1,  # To help debug
@@ -364,7 +371,7 @@ def run_pipeline(options, *, plugin_manager, api=False):
         validate_pdfinfo_options(context)
 
         # Execute the pipeline
-        exec_concurrent(context)
+        exec_concurrent(context, executor)
 
         if options.output_file == '-':
             log.info("Output sent to stdout")
@@ -399,7 +406,9 @@ def run_pipeline(options, *, plugin_manager, api=False):
             log.error("KeyboardInterrupt")
         return ExitCode.ctrl_c
     except (ExitCodeException if not api else NeverRaise) as e:
-        if str(e):
+        if options.verbose >= 1:
+            log.exception("ExitCodeException")
+        elif str(e):
             log.error("%s: %s", type(e).__name__, str(e))
         else:
             log.error(type(e).__name__)

src/ocrmypdf/_validation.py

@@ -13,7 +13,7 @@ import sys
 import unicodedata
 from pathlib import Path
 from shutil import copyfileobj
-from typing import Tuple
+from typing import List, Set, Tuple, Union
 
 import pikepdf
 import PIL
@@ -32,12 +32,12 @@ from ocrmypdf.helpers import (
     monotonic,
     safe_symlink,
 )
+from ocrmypdf.hocrtransform import HOCR_OK_LANGS
 from ocrmypdf.subprocess import check_external_program
 
 # -------------
 # External dependencies
 
-HOCR_OK_LANGS = frozenset(['eng', 'deu', 'spa', 'ita', 'por'])
 DEFAULT_LANGUAGE = 'eng'  # Enforce English hegemony
 
 log = logging.getLogger(__name__)
@@ -65,20 +65,21 @@ def check_options_languages(options, ocr_engine_languages):
             log.debug("No language specified; assuming --language %s", DEFAULT_LANGUAGE)
     if not ocr_engine_languages:
         return
-    if not options.languages.issubset(ocr_engine_languages):
+    missing_languages = options.languages - ocr_engine_languages
+    if missing_languages:
         msg = (
             f"OCR engine does not have language data for the following "
             "requested languages: \n"
         )
-        for lang in options.languages - ocr_engine_languages:
-            msg += lang + '\n'
+        msg += '\n'.join(lang for lang in missing_languages)
+        msg += '\nNote: most languages are identified by a 3-digit ISO 639-2 Code'
         raise MissingDependencyError(msg)
 
 
 def check_options_output(options):
     is_latin = options.languages.issubset(HOCR_OK_LANGS)
 
-    if options.pdf_renderer == 'hocr' and not is_latin:
+    if options.pdf_renderer.startswith('hocr') and not is_latin:
         msg = (
             "The 'hocr' PDF renderer is known to cause problems with one "
             "or more of the languages in your document.  Use "
@@ -112,6 +113,10 @@ def check_options_sidecar(options):
                 "--sidecar filename must be specified when output file is stdout."
             )
         options.sidecar = options.output_file + '.txt'
+    if options.sidecar == options.input_file or options.sidecar == options.output_file:
+        raise BadArgsError(
+            "--sidecar file must be different from the input and output files"
+        )
 
 
 def check_options_preprocessing(options):
@@ -133,13 +138,13 @@ def check_options_preprocessing(options):
                     options.unpaper_args
                 )
         except Exception as e:
-            raise BadArgsError(str(e))
+            raise BadArgsError("--unpaper-args: " + str(e)) from e
 
 
-def _pages_from_ranges(ranges):
+def _pages_from_ranges(ranges: str) -> Set[int]:
     if is_iterable_notstr(ranges):
         return set(ranges)
-    pages = []
+    pages: List[int] = []
     page_groups = ranges.replace(' ', '').split(',')
     for g in page_groups:
         if not g:
@@ -150,9 +155,18 @@ def _pages_from_ranges(ranges):
             pages.append(int(g) - 1)
         else:
             try:
-                pages.extend(range(int(start) - 1, int(end)))
+                new_pages = list(range(int(start) - 1, int(end)))
+                if not new_pages:
+                    raise BadArgsError(f"invalid page subrange '{start}-{end}'")
+                pages.extend(new_pages)
             except ValueError:
-                raise BadArgsError("invalid page range")
+                raise BadArgsError("invalid page range") from None
+
+    if not pages:
+        raise BadArgsError(
+            f"The string of page ranges '{ranges}' did not contain any recognizable "
+            f"page ranges."
+        )
 
     if not monotonic(pages):
         log.warning(
@@ -175,8 +189,6 @@ def check_options_ocr_behavior(options):
     )
     if exclusive_options >= 2:
         raise BadArgsError("Choose only one of --force-ocr, --skip-text, --redo-ocr.")
-    if options.pages and options.sidecar:
-        raise BadArgsError("--pages and --sidecar are mutually exclusive")
     if options.pages:
         options.pages = _pages_from_ranges(options.pages)
 
@@ -213,12 +225,12 @@ def check_options_optimizing(options):
 
 
 def check_options_advanced(options):
-    if options.pdfa_image_compression != 'auto' and options.output_type.startswith(
+    if options.pdfa_image_compression != 'auto' and not options.output_type.startswith(
         'pdfa'
     ):
         log.warning(
-            "--pdfa-image-compression argument has no effect when "
-            "--output-type is not 'pdfa', 'pdfa-1', or 'pdfa-2'"
+            "--pdfa-image-compression argument only applies when "
+            "--output-type is one of 'pdfa', 'pdfa-1', or 'pdfa-2'"
         )
 
 
@@ -268,7 +280,7 @@ def check_closed_streams(options):  # pragma: no cover
     Attempting to a fork/exec a new Python process when any of std{in,out,err}
     are closed or not flushable for some reason may raise an exception.
     Fix this by opening devnull if the handle seems to be closed.  Do this
-    globally to avoid tracking places all places that fork.
+    globally to avoid tracking all places that fork.
 
     Seems to be specific to multiprocessing.Process not all Python process
     forkers.
@@ -330,7 +342,17 @@ def create_input_file(options, work_folder: Path) -> Tuple[Path, str]:
             safe_symlink(options.input_file, target)
             return target, os.fspath(options.input_file)
         except FileNotFoundError:
-            raise InputFileError(f"File not found - {options.input_file}")
+            msg = f"File not found - {options.input_file}"
+            if Path('/.dockerenv').exists():  # pragma: no cover
+                msg += (
+                    "\nDocker cannot your working directory unless you "
+                    "explicitly share it with the Docker container and set up"
+                    "permissions correctly.\n"
+                    "You may find it easier to use stdin/stdout:"
+                    "\n"
+                    "\tdocker run -i --rm jbarlow83/ocrmypdf - - <input.pdf >output.pdf\n"
+                )
+            raise InputFileError(msg)
 
 
 def check_requested_output_file(options):

src/ocrmypdf/api.py

@@ -8,12 +8,17 @@
 import logging
 import os
 import sys
+import threading
 from enum import IntEnum
+from io import IOBase
 from pathlib import Path
-from typing import BinaryIO, Iterable, Union
+from typing import AnyStr, BinaryIO, Iterable, Optional, Union
 from warnings import warn
 
-from ocrmypdf._logging import PageNumberFilter, TqdmConsole
+from ocrmypdf._logging import (  # pylint: disable=unused-import
+    PageNumberFilter,
+    TqdmConsole,
+)
 from ocrmypdf._plugin_manager import get_plugin_manager
 from ocrmypdf._sync import run_pipeline
 from ocrmypdf._validation import check_options
@@ -26,7 +31,10 @@ except ModuleNotFoundError:
     coloredlogs = None
 
 
-PathOrIO = Union[BinaryIO, os.PathLike, str, bytes]
+StrPath = Union[os.PathLike, AnyStr]
+PathOrIO = Union[BinaryIO, StrPath]
+
+_api_lock = threading.Lock()
 
 
 class Verbosity(IntEnum):
@@ -40,18 +48,23 @@ class Verbosity(IntEnum):
 
 def configure_logging(
     verbosity: Verbosity,
+    *,
     progress_bar_friendly: bool = True,
     manage_root_logger: bool = False,
+    plugin_manager=None,
 ):
     """Set up logging.
 
     Before calling :func:`ocrmypdf.ocr()`, you can use this function to
-    configure logging, if you want ocrmypdf's output to look like the ocrmypdf
+    configure logging if you want ocrmypdf's output to look like the ocrmypdf
     command line interface. It will register log handlers, log filters, and
     formatters, configure color logging to standard error, and adjust the log
     levels of third party libraries. Details of this are fine-tuned and subject
     to change. The ``verbosity`` argument is equivalent to the argument
-    ``--verbose`` and applies those settings.
+    ``--verbose`` and applies those settings. If you have a wrapper
+    script for ocrmypdf and you want it to be very similar to ocrmypdf, use this
+    function; if you are using ocrmypdf as part of an application that manages
+    its own logging, you probably do not want this function.
 
     If this function is not called, ocrmypdf will not configure logging, and it
     is up to the caller of ``ocrmypdf.ocr()`` to set up logging as it wishes using
@@ -69,12 +82,11 @@ def configure_logging(
     their own debug logging.
 
     Args:
-        verbosity (Verbosity): Verbosity level.
-        progress_bar_friendly (bool): Install the TqdmConsole log handler, which is
-            compatible with the tqdm progress bar; without this log messages will
-            overwrite the progress bar
-        manage_root_logger (bool): Configure the process's root logger, to ensure
-            all log output is sent through
+        verbosity: Verbosity level.
+        progress_bar_friendly: If True (the default), install a custom log handler
+            that is compatible with progress bars and colored output.
+        manage_root_logger: Configure the process's root logger.
+        plugin_manager: The plugin manager, used for obtaining the custom log handler.
 
     Returns:
         The toplevel logger for ocrmypdf (or the root logger, if we are managing it).
@@ -85,9 +97,11 @@ def configure_logging(
     log = logging.getLogger(prefix)
     log.setLevel(logging.DEBUG)
 
-    if progress_bar_friendly:
-        console = logging.StreamHandler(stream=TqdmConsole(sys.stderr))
-    else:
+    console = None
+    if plugin_manager and progress_bar_friendly:
+        console = plugin_manager.hook.get_logging_console()
+
+    if not console:
         console = logging.StreamHandler(stream=sys.stderr)
 
     if verbosity < 0:
@@ -173,14 +187,14 @@ def create_options(
         else:
             raise TypeError(f"{arg}: {val} ({type(val)})")
 
-    try:
-        cmdline.append(os.fspath(input_file))
-    except TypeError:
+    if isinstance(input_file, (BinaryIO, IOBase)):
         cmdline.append('stream://input_file')
-    try:
-        cmdline.append(os.fspath(output_file))
-    except TypeError:
+    else:
+        cmdline.append(os.fspath(input_file))
+    if isinstance(output_file, (BinaryIO, IOBase)):
         cmdline.append('stream://output_file')
+    else:
+        cmdline.append(os.fspath(output_file))
 
     parser._api_mode = True
     options = parser.parse_args(cmdline)
@@ -202,7 +216,7 @@ def ocr(  # pylint: disable=unused-argument
     language: Iterable[str] = None,
     image_dpi: int = None,
     output_type=None,
-    sidecar: os.PathLike = None,
+    sidecar: Optional[StrPath] = None,
     jobs: int = None,
     use_threads: bool = None,
     title: str = None,
@@ -239,7 +253,8 @@ def ocr(  # pylint: disable=unused-argument
     user_words: os.PathLike = None,
     user_patterns: os.PathLike = None,
     fast_web_view: float = None,
-    plugins: Iterable[Union[str, Path]] = None,
+    plugins: Iterable[StrPath] = None,
+    plugin_manager=None,
     keep_temporary_files: bool = None,
     progress_bar: bool = None,
     **kwargs,
@@ -261,7 +276,7 @@ def ocr(  # pylint: disable=unused-argument
             read.
         output_file: If a :class:`pathlib.Path`, ``str`` or ``bytes``, this is
             interpreted as file system path to the output file. If the object
-            appears to be a writable stream (with methods such as ``.read()`` and
+            appears to be a writable stream (with methods such as ``.write()`` and
             ``.seek()``), the output will be written to this stream. If
             ``output_file`` is ``"-"``, the output will be written to ``sys.stdout``
             (provided that standard output does not seem to be a terminal device).
@@ -291,6 +306,9 @@ def ocr(  # pylint: disable=unused-argument
     Returns:
         :class:`ocrmypdf.ExitCode`
     """
+    if plugins and plugin_manager:
+        raise ValueError("plugins= and plugin_manager are mutually exclusive")
+
     if not plugins:
         plugins = []
     elif isinstance(plugins, (str, Path)):
@@ -298,18 +316,25 @@ def ocr(  # pylint: disable=unused-argument
     else:
         plugins = list(plugins)
 
-    parser = get_parser()
-    _plugin_manager = get_plugin_manager(plugins)
-    _plugin_manager.hook.add_options(parser=parser)  # pylint: disable=no-member
-
-    create_options_kwargs = {
-        k: v for k, v in locals().items() if not k.startswith('_') and k != 'kwargs'
-    }
+    # No new variable names should be assigned until these two steps are run
+    create_options_kwargs = {k: v for k, v in locals().items() if k != 'kwargs'}
     create_options_kwargs.update(kwargs)
 
-    if 'verbose' in kwargs:
-        warn("ocrmypdf.ocr(verbose=) is ignored. Use ocrmypdf.configure_logging().")
+    parser = get_parser()
+    create_options_kwargs['parser'] = parser
 
-    options = create_options(**create_options_kwargs)
-    check_options(options, _plugin_manager)
-    return run_pipeline(options=options, plugin_manager=_plugin_manager, api=True)
+    with _api_lock:
+        # We can't allow multiple ocrmypdf.ocr() threads to run in parallel, because
+        # they might install different plugins, and generally speaking we have areas
+        # of code that use global state.
+
+        if not plugin_manager:
+            plugin_manager = get_plugin_manager(plugins)
+        plugin_manager.hook.add_options(parser=parser)  # pylint: disable=no-member
+
+        if 'verbose' in kwargs:
+            warn("ocrmypdf.ocr(verbose=) is ignored. Use ocrmypdf.configure_logging().")
+
+        options = create_options(**create_options_kwargs)
+        check_options(options, plugin_manager)
+        return run_pipeline(options=options, plugin_manager=plugin_manager, api=True)

src/ocrmypdf/builtin_plugins/__init__.py

@@ -3,3 +3,7 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file exists only mark builtin_plugins as a package.
+# The plugin manager will not load it, so anything defined here may not be
+# processed as a module.

src/ocrmypdf/builtin_plugins/concurrency.py

@@ -0,0 +1,172 @@
+# © 2020 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# © 2020 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+import logging.handlers
+import multiprocessing
+import os
+import queue
+import signal
+import sys
+import threading
+from contextlib import suppress
+from multiprocessing import Pool as ProcessPool
+from multiprocessing.pool import ThreadPool
+from typing import Callable, Iterable, Union
+
+from tqdm import tqdm
+
+from ocrmypdf import Executor, hookimpl
+from ocrmypdf._logging import TqdmConsole
+from ocrmypdf.exceptions import InputFileError
+from ocrmypdf.helpers import remove_all_log_handlers
+
+Queue = Union[multiprocessing.Queue, queue.Queue]
+
+
+def log_listener(q: Queue):
+    """Listen to the worker processes and forward the messages to logging
+
+    For simplicity this is a thread rather than a process. Only one process
+    should actually write to sys.stderr or whatever we're using, so if this is
+    made into a process the main application needs to be directed to it.
+
+    See https://docs.python.org/3/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes
+    """
+
+    while True:
+        try:
+            record = q.get()
+            if record is None:
+                break
+            logger = logging.getLogger(record.name)
+            logger.handle(record)
+        except Exception:  # pylint: disable=broad-except
+            import traceback  # pylint: disable=import-outside-toplevel
+
+            print("Logging problem", file=sys.stderr)
+            traceback.print_exc(file=sys.stderr)
+
+
+def process_sigbus(*args):
+    raise InputFileError("A worker process lost access to an input file")
+
+
+def process_init(q: Queue, user_init: Callable[[], None], loglevel):
+    """Initialize a process pool worker"""
+
+    # Ignore SIGINT (our parent process will kill us gracefully)
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+    # Install SIGBUS handler (so our parent process can abort somewhat gracefully)
+    with suppress(AttributeError):  # Windows and Cygwin do not have SIGBUS
+        # Windows and Cygwin do not have pthread_sigmask or SIGBUS
+        signal.signal(signal.SIGBUS, process_sigbus)
+
+    # Remove any log handlers that belong to the parent process
+    root = logging.getLogger()
+    remove_all_log_handlers(root)
+
+    # Set up our single log handler to forward messages to the parent
+    root.setLevel(loglevel)
+    root.addHandler(logging.handlers.QueueHandler(q))
+
+    user_init()
+    return
+
+
+def thread_init(_queue: Queue, user_init: Callable[[], None], _loglevel):
+    # As a thread, block SIGBUS so the main thread deals with it...
+    with suppress(AttributeError):
+        signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGBUS})
+
+    user_init()
+    return
+
+
+class StandardExecutor(Executor):
+    def _execute(
+        self,
+        *,
+        use_threads: bool,
+        max_workers: int,
+        tqdm_kwargs: dict,
+        worker_initializer: Callable,
+        task: Callable,
+        task_arguments: Iterable,
+        task_finished: Callable,
+    ):
+        if use_threads:
+            log_queue = queue.Queue(-1)
+            pool_class = ThreadPool
+            initializer = thread_init
+        else:
+            log_queue = multiprocessing.Queue(-1)
+            pool_class = ProcessPool
+            initializer = process_init
+
+        # Regardless of whether we use_threads for worker processes, the log_listener
+        # must be a thread. Make sure we create the listener after the worker pool,
+        # so that it does not get forked into the workers.
+        listener = threading.Thread(target=log_listener, args=(log_queue,))
+        listener.start()
+
+        with self.pbar_class(**tqdm_kwargs) as pbar:
+            pool = pool_class(
+                processes=max_workers,
+                initializer=initializer,
+                initargs=(log_queue, worker_initializer, logging.getLogger("").level),
+            )
+            try:
+                results = pool.imap_unordered(task, task_arguments)
+                for result in results:
+                    if task_finished:
+                        task_finished(result, pbar)
+                    else:
+                        pbar.update()
+            except KeyboardInterrupt:
+                # Terminate pool so we exit instantly
+                pool.terminate()
+                # Don't try listener.join() here, will deadlock
+                raise
+            except Exception:
+                if not os.environ.get("PYTEST_CURRENT_TEST", ""):
+                    # Unless inside pytest, exit immediately because no one wants
+                    # to wait for child processes to finalize results that will be
+                    # thrown away. Inside pytest, we want child processes to exit
+                    # cleanly so that they output an error messages or coverage data
+                    # we need from them.
+                    pool.terminate()
+                raise
+            finally:
+                # Terminate log listener
+                log_queue.put_nowait(None)
+                pool.close()
+                pool.join()
+
+        listener.join()
+
+
+@hookimpl
+def get_executor(progressbar_class):
+    return StandardExecutor(pbar_class=progressbar_class)
+
+
+@hookimpl
+def get_progressbar_class():
+    return tqdm
+
+
+@hookimpl
+def get_logging_console():
+    return logging.StreamHandler(stream=TqdmConsole(sys.stderr))

src/ocrmypdf/builtin_plugins/default_filters.py

@@ -0,0 +1,14 @@
+# © 2021 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from ocrmypdf import hookimpl
+
+
+@hookimpl
+def filter_pdf_page(
+    page, image_filename, output_pdf
+):  # pylint: disable=unused-argument
+    return output_pdf

src/ocrmypdf/builtin_plugins/ghostscript.py

@@ -79,12 +79,21 @@ def rasterize_pdf_page(
 
 
 @hookimpl
-def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdfa_part):
+def generate_pdfa(
+    pdf_pages,
+    pdfmark,
+    output_file,
+    compression,
+    pdf_version,
+    pdfa_part,
+    progressbar_class,
+):
     ghostscript.generate_pdfa(
         pdf_pages=[*pdf_pages, pdfmark],
         output_file=output_file,
         compression=compression,
         pdf_version=pdf_version,
         pdfa_part=pdfa_part,
+        progressbar_class=progressbar_class,
     )
     return output_file

src/ocrmypdf/builtin_plugins/tesseract_ocr.py

@@ -80,20 +80,14 @@ def check_options(options):
         program='tesseract',
         package={'linux': 'tesseract-ocr'},
         version_checker=tesseract.version,
-        need_version='4.0.0',  # using backport for Travis CI
+        need_version='4.0.0-beta.1',  # using backport for Travis CI
+        version_parser=tesseract.TesseractVersion,
     )
 
     # Decide on what renderer to use
     if options.pdf_renderer == 'auto':
         options.pdf_renderer = 'sandwich'
 
-    if options.pdf_renderer == 'sandwich' and not tesseract.has_textonly_pdf(
-        set(options.languages)
-    ):
-        raise MissingDependencyError(
-            "You are using an alpha version of Tesseract 4.0 that does not support "
-            "the textonly_pdf parameter. We don't support versions this old."
-        )
     if not tesseract.has_user_words() and (options.user_words or options.user_patterns):
         log.warning(
             "Tesseract 4.0 ignores --user-words and --user-patterns, so these "

src/ocrmypdf/cli.py

@@ -6,12 +6,15 @@
 
 
 import argparse
+from typing import Optional, Type, TypeVar
 
 from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME
 from ocrmypdf._version import __version__ as _VERSION
 
+T = TypeVar('T')
 
-def numeric(basetype, min_=None, max_=None):
+
+def numeric(basetype: Type[T], min_: Optional[T] = None, max_: Optional[T] = None):
     """Validator for numeric params"""
     min_ = basetype(min_) if min_ is not None else None
     max_ = basetype(max_) if max_ is not None else None
@@ -164,7 +167,8 @@ Online documentation is located at:
         metavar='FILE',
         help="Generate sidecar text files that contain the same text recognized "
         "by Tesseract. This may be useful for building a OCR text database. "
-        "If FILE is omitted, the sidecar file be named {output_file}.txt "
+        "If FILE is omitted, the sidecar file be named {output_file}.txt; the next "
+        "argument must NOT be the name of the input PDF. "
         "If FILE is set to '-', the sidecar is written to stdout (a "
         "convenient way to preview OCR quality). The output file and sidecar "
         "may not both use stdout at the same time.",
@@ -407,7 +411,7 @@ Online documentation is located at:
     )
     advanced.add_argument(
         '--pdf-renderer',
-        choices=['auto', 'hocr', 'sandwich'],
+        choices=['auto', 'hocr', 'sandwich', 'hocrdebug'],
         default='auto',
         help="Choose OCR PDF renderer - the default option is to let OCRmyPDF "
         "choose.  See documentation for discussion.",

src/ocrmypdf/extra_plugins/semfree.py

@@ -0,0 +1,191 @@
+# © 2021 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+"""Semaphore-free alternate executor.
+
+There are two popular environments that do not fully support the standard Python
+multiprocessing module: AWS Lambda, and Termux (a terminal emulator for Android).
+
+This alternate executor divvies up work among worker processes before processing,
+rather than having each worker consume work from a shared queue when they finish
+their task. This means workers have no need to coordinate with each other. Each
+worker communicates only with the main process.
+
+This is not without drawbacks. If the tasks are not "even" in size, which cannot
+be guaranteed, some workers may end up with too much work while others are idle.
+It is less efficient than the standard implementation, so not th edefault.
+"""
+
+import logging
+import logging.handlers
+import signal
+from contextlib import suppress
+from enum import Enum, auto
+from itertools import islice, repeat, takewhile, zip_longest
+from multiprocessing import Pipe, Process
+from multiprocessing.connection import Connection, wait
+from typing import Callable, Iterable, Iterator
+
+from ocrmypdf import Executor, hookimpl
+from ocrmypdf._concurrent import NullProgressBar
+from ocrmypdf.exceptions import InputFileError
+from ocrmypdf.helpers import remove_all_log_handlers
+
+
+class MessageType(Enum):
+    exception = auto()
+    result = auto()
+    complete = auto()
+
+
+def split_every(n: int, iterable: Iterable) -> Iterator:
+    """Split iterable into groups of n.
+
+    >>> list(split_every(4, range(10)))
+    [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
+
+    https://stackoverflow.com/a/22919323
+    """
+    iterator = iter(iterable)
+    return takewhile(bool, (list(islice(iterator, n)) for _ in repeat(None)))
+
+
+def process_sigbus(*args):
+    raise InputFileError("A worker process lost access to an input file")
+
+
+class ConnectionLogHandler(logging.handlers.QueueHandler):
+    def __init__(self, conn: Connection) -> None:
+        super().__init__(None)
+        self.conn = conn
+
+    def enqueue(self, record):
+        self.conn.send(('log', record))
+
+
+def process_loop(
+    conn: Connection, user_init: Callable[[], None], loglevel, task, task_args
+):
+    """Initialize a process pool worker"""
+
+    # Install SIGBUS handler (so our parent process can abort somewhat gracefully)
+    with suppress(AttributeError):  # Windows and Cygwin do not have SIGBUS
+        # Windows and Cygwin do not have pthread_sigmask or SIGBUS
+        signal.signal(signal.SIGBUS, process_sigbus)
+
+    # Reconfigure the root logger for this process to send all messages to a queue
+    h = ConnectionLogHandler(conn)
+    root = logging.getLogger()
+    remove_all_log_handlers(root)
+    root.setLevel(loglevel)
+    root.addHandler(h)
+
+    user_init()
+
+    for args in task_args:
+        try:
+            result = task(args)
+        except Exception as e:
+            conn.send((MessageType.exception, e))
+            break
+        else:
+            conn.send((MessageType.result, result))
+
+    conn.send((MessageType.complete, None))
+    conn.close()
+    return
+
+
+class LambdaExecutor(Executor):
+    def _execute(
+        self,
+        *,
+        use_threads: bool,
+        max_workers: int,
+        tqdm_kwargs: dict,
+        worker_initializer: Callable,
+        task: Callable,
+        task_arguments: Iterable,
+        task_finished: Callable,
+    ):
+        if use_threads and max_workers == 1:
+            with self.pbar_class(**tqdm_kwargs) as pbar:
+                for args in task_arguments:
+                    result = task(args)
+                    task_finished(result, pbar)
+            return
+
+        task_arguments = list(task_arguments)
+        grouped_args = list(
+            zip_longest(*list(split_every(max_workers, task_arguments)))
+        )
+        if not grouped_args:
+            return
+
+        processes = []
+        connections = []
+        for chunk in grouped_args:
+            parent_conn, child_conn = Pipe()
+
+            worker_args = [args for args in chunk if args is not None]
+            process = Process(
+                target=process_loop,
+                args=(
+                    child_conn,
+                    worker_initializer,
+                    logging.getLogger("").level,
+                    task,
+                    worker_args,
+                ),
+            )
+            process.daemon = True
+            processes.append(process)
+            connections.append(parent_conn)
+
+        for process in processes:
+            process.start()
+
+        with self.pbar_class(**tqdm_kwargs) as pbar:
+            while connections:
+                for r in wait(connections):
+                    try:
+                        msg_type, msg = r.recv()
+                    except EOFError:
+                        connections.remove(r)
+                        continue
+
+                    if msg_type == MessageType.result:
+                        if task_finished:
+                            task_finished(msg, pbar)
+                    elif msg_type == 'log':
+                        record = msg
+                        logger = logging.getLogger(record.name)
+                        logger.handle(record)
+                    elif msg_type == MessageType.complete:
+                        connections.remove(r)
+                    elif msg_type == MessageType.exception:
+                        for process in processes:
+                            process.terminate()
+                        raise msg
+
+        for process in processes:
+            process.join()
+
+
+@hookimpl
+def get_executor(progressbar_class):
+    return LambdaExecutor(pbar_class=progressbar_class)
+
+
+@hookimpl
+def get_logging_console():
+    return logging.StreamHandler()
+
+
+@hookimpl
+def get_progressbar_class():
+    return NullProgressBar

src/ocrmypdf/helpers.py

@@ -15,9 +15,9 @@ from collections.abc import Iterable
 from contextlib import suppress
 from functools import wraps
 from io import StringIO
-from math import isclose
+from math import isclose, isfinite
 from pathlib import Path
-from typing import Any, Sequence, TypeVar
+from typing import Any, Sequence
 
 import pikepdf
 
@@ -25,19 +25,35 @@ log = logging.getLogger(__name__)
 
 
 class Resolution(namedtuple('Resolution', ('x', 'y'))):
-    """The number of pixels per inch in each 2D direction."""
+    """The number of pixels per inch in each 2D direction.
+
+    Resolution objects are considered "equal" for == purposes if they are
+    equal to a reasonable tolerance.
+    """
 
     __slots__ = ()
 
+    # rel_tol after converting from dpi to pixels per meter and saving
+    # as integer with rounding, as many file formats
+    CONVERSION_ERROR = 0.002
+
     def round(self, ndigits: int):
         return Resolution(round(self.x, ndigits), round(self.y, ndigits))
 
     def to_int(self):
         return Resolution(int(round(self.x)), int(round(self.y)))
 
+    @classmethod
+    def _isclose(cls, a, b):
+        return isclose(a, b, rel_tol=cls.CONVERSION_ERROR)
+
     @property
     def is_square(self) -> bool:
-        return isclose(self.x, self.y, rel_tol=1e-3)
+        return self._isclose(self.x, self.y)
+
+    @property
+    def is_finite(self) -> bool:
+        return isfinite(self.x) and isfinite(self.y)
 
     def take_max(self, vals, yvals=None):
         if yvals is not None:
@@ -54,13 +70,29 @@ class Resolution(namedtuple('Resolution', ('x', 'y'))):
     def __str__(self):
         return f"{self.x:f}x{self.y:f}"
 
-    def __repr__(self):
+    def __repr__(self):  # pragma: no cover
         return f"Resolution({self.x}x{self.y} dpi)"
 
+    def __eq__(self, other):
+        if isinstance(other, tuple) and len(other) == 2:
+            other = Resolution(*other)
+        if not isinstance(other, Resolution):
+            return NotImplemented
+        return self._isclose(self.x, other.x) and self._isclose(self.y, other.y)
+
+
+class NeverRaise(Exception):
+    """An exception that is never raised"""
+
 
 def safe_symlink(input_file: os.PathLike, soft_link_name: os.PathLike):
-    """
-    Helper function: relinks soft symbolic link if necessary
+    """Create a symbolic link at ``soft_link_name``, which references ``input_file``.
+
+    Think of this as copying ``input_file`` to ``soft_link_name`` with less overhead.
+
+    Use symlinks safely. Self-linking loops are prevented. On Windows, file copy is
+    used since symlinks may require administrator privileges. An existing link at the
+    destination is removed.
     """
     input_file = os.fspath(input_file)
     soft_link_name = os.fspath(soft_link_name)
@@ -68,8 +100,8 @@ def safe_symlink(input_file: os.PathLike, soft_link_name: os.PathLike):
     # Guard against soft linking to oneself
     if input_file == soft_link_name:
         log.warning(
-            "No symbolic link made. You are using "
-            "the original data directory as the working directory."
+            "No symbolic link created. You are using  the original data directory "
+            "as the working directory."
         )
         return
 
@@ -78,10 +110,7 @@ def safe_symlink(input_file: os.PathLike, soft_link_name: os.PathLike):
         # do not delete or overwrite real (non-soft link) file
         if not os.path.islink(soft_link_name):
             raise FileExistsError(f"{soft_link_name} exists and is not a link")
-        try:
-            os.unlink(soft_link_name)
-        except OSError:
-            log.debug("Can't unlink %s", soft_link_name)
+        os.unlink(soft_link_name)
 
     if not os.path.exists(input_file):
         raise FileNotFoundError(f"trying to create a broken symlink to {input_file}")
@@ -110,7 +139,7 @@ def is_iterable_notstr(thing: Any) -> bool:
 
 
 def monotonic(L: Sequence) -> bool:
-    """Does list increase monotonically?"""
+    """Does this sequence increase monotonically?"""
     return all(b > a for a, b in zip(L, L[1:]))
 
 
@@ -169,49 +198,61 @@ def is_file_writable(test_file: os.PathLike) -> bool:
 def check_pdf(input_file: Path) -> bool:
     """Check if a PDF complies with the PDF specification.
 
-    Checks for proper formatting and proper linearization.
+    Checks for proper formatting and proper linearization. Uses pikepdf (which in
+    turn, uses QPDF) to perform the checks.
     """
-    pdf = None
     try:
         pdf = pikepdf.open(input_file)
     except pikepdf.PdfError as e:
         log.error(e)
         return False
     else:
-        messages = pdf.check()
-        for msg in messages:
-            if 'error' in msg.lower():
-                log.error(msg)
+        with pdf:
+            messages = pdf.check()
+            for msg in messages:
+                if 'error' in msg.lower():
+                    log.error(msg)
+                else:
+                    log.warning(msg)
+
+            sio = StringIO()
+            linearize_msgs = ''
+            try:
+                # If linearization is missing entirely, we do not complain. We do
+                # complain if linearization is present but incorrect.
+                pdf.check_linearization(sio)
+            except RuntimeError:
+                pass
+            except (
+                # Workaround for a problematic pikepdf version
+                # pragma: no cover
+                getattr(pikepdf, 'ForeignObjectError')
+                if pikepdf.__version__ == '2.1.0'
+                else NeverRaise
+            ):
+                pass
             else:
-                log.warning(msg)
+                linearize_msgs = sio.getvalue()
+                if linearize_msgs:
+                    log.warning(linearize_msgs)
 
-        sio = StringIO()
-        linearize = None
-        try:
-            pdf.check_linearization(sio)
-        except RuntimeError:
-            pass
-        else:
-            linearize = sio.getvalue()
-            if linearize:
-                log.warning(linearize)
-
-        if not messages and not linearize:
-            return True
-        return False
-    finally:
-        if pdf:
-            pdf.close()
+            if not messages and not linearize_msgs:
+                return True
+            return False
 
 
-T = TypeVar('T')
-
-
-def clamp(n: T, smallest: T, largest: T) -> T:
-    """Clamps the value of n to between smallest and largest."""
+def clamp(n, smallest, largest):  # mypy doesn't understand types for this
+    """Clamps the value of ``n`` to between ``smallest`` and ``largest``."""
     return max(smallest, min(n, largest))
 
 
+def remove_all_log_handlers(logger):
+    "Remove all log handlers, usually used in a child process."
+    for handler in logger.handlers[:]:
+        logger.removeHandler(handler)
+        handler.close()  # To ensure handlers with opened resources are released
+
+
 def pikepdf_enable_mmap():
     # try:
     #     if pikepdf._qpdf.set_access_default_mmap(True):
@@ -220,12 +261,13 @@ def pikepdf_enable_mmap():
     #     log.debug("pikepdf mmap not available")
     # We found a race condition probably related to pybind issue #2252 that can
     # cause a crash. For now, disable pikepdf mmap to be on the safe side.
-    log.debug("pikepdf mmap disabled")
+    # Fix is not in pybind11 2.6.0
+    # log.debug("pikepdf mmap disabled")
     return
 
 
 def deprecated(func):
-    """Warn that function is deprecated"""
+    """Warn that function is deprecated."""
 
     @wraps(func)
     def new_func(*args, **kwargs):

src/ocrmypdf/hocrtransform.py

@@ -31,18 +31,81 @@
 import argparse
 import os
 import re
-from collections import namedtuple
-from itertools import chain
 from math import atan, cos, sin
 from pathlib import Path
-from typing import Optional, Tuple, Union
+from typing import Any, NamedTuple, Optional, Tuple, Union
 from xml.etree import ElementTree
 
 from reportlab.lib.colors import black, cyan, magenta, red
 from reportlab.lib.units import inch
 from reportlab.pdfgen.canvas import Canvas
 
-Rect = namedtuple('Rect', ['x1', 'y1', 'x2', 'y2'])
+# According to Wikipedia these languages are supported in the ISO-8859-1 character
+# set, meaning reportlab can generate them and they are compatible with hocr,
+# assuming Tesseract has the necessary languages installed. Note that there may
+# not be language packs for them.
+HOCR_OK_LANGS = frozenset(
+    [
+        # Languages fully covered by Latin-1:
+        'afr',  # Afrikaans
+        'alb',  # Albanian
+        'ast',  # Leonese
+        'baq',  # Basque
+        'bre',  # Breton
+        'cos',  # Corsican
+        'eng',  # English
+        'eus',  # Basque
+        'fao',  # Faoese
+        'gla',  # Scottish Gaelic
+        'glg',  # Galician
+        'glv',  # Manx
+        'ice',  # Icelandic
+        'ind',  # Indonesian
+        'isl',  # Icelandic
+        'ita',  # Italian
+        'ltz',  # Luxembourgish
+        'mal',  # Malay Rumi
+        'mga',  # Irish
+        'nor',  # Norwegian
+        'oci',  # Occitan
+        'por',  # Portugeuse
+        'roh',  # Romansh
+        'sco',  # Scots
+        'sma',  # Sami
+        'spa',  # Spanish
+        'sqi',  # Albanian
+        'swa',  # Swahili
+        'swe',  # Swedish
+        'tgl',  # Tagalog
+        'wln',  # Walloon
+        # Languages supported by Latin-1 except for a few rare characters that OCR
+        # is probably not trained to recognize anyway:
+        'cat',  # Catalan
+        'cym',  # Welsh
+        'dan',  # Danish
+        'deu',  # German
+        'dut',  # Dutch
+        'est',  # Estonian
+        'fin',  # Finnish
+        'fra',  # French
+        'hun',  # Hungarian
+        'kur',  # Kurdish
+        'nld',  # Dutch
+        'wel',  # Welsh
+    ]
+)
+
+
+Element = ElementTree.Element
+
+
+class Rect(NamedTuple):  # pylint: disable=inherit-non-class
+    """A rectangle for managing PDF coordinates."""
+
+    x1: Any
+    y1: Any
+    x2: Any
+    y2: Any
 
 
 class HocrTransformError(Exception):
@@ -69,7 +132,7 @@ class HocrTransform:
         {'ff': 'ff', 'ffi': 'f‌f‌i', 'ffl': 'f‌f‌l', 'fi': 'fi', 'fl': 'fl'}
     )
 
-    def __init__(self, hocr_filename: Union[str, Path], dpi: float):
+    def __init__(self, *, hocr_filename: Union[str, Path], dpi: float):
         self.dpi = dpi
         self.hocr = ElementTree.parse(os.fspath(hocr_filename))
 
@@ -105,7 +168,7 @@ class HocrTransform:
         else:
             return ''
 
-    def _get_element_text(self, element):
+    def _get_element_text(self, element: Element):
         """
         Return the textual content of the element and its children
         """
@@ -119,7 +182,7 @@ class HocrTransform:
         return text
 
     @classmethod
-    def element_coordinates(cls, element) -> Rect:
+    def element_coordinates(cls, element: Element) -> Rect:
         """
         Returns a tuple containing the coordinates of the bounding box around
         an element
@@ -133,7 +196,7 @@ class HocrTransform:
         return out
 
     @classmethod
-    def baseline(cls, element) -> Tuple[float, float]:
+    def baseline(cls, element: Element) -> Tuple[float, float]:
         """
         Returns a tuple containing the baseline slope and intercept.
         """
@@ -149,7 +212,7 @@ class HocrTransform:
         """
         return Rect._make((c / self.dpi * inch) for c in pxl)
 
-    def _child_xpath(self, html_tag, html_class=None):
+    def _child_xpath(self, html_tag: str, html_class: Optional[str] = None) -> str:
         xpath = f".//{self.xmlns}{html_tag}"
         if html_class:
             xpath += f"[@class='{html_class}']"
@@ -174,6 +237,7 @@ class HocrTransform:
 
     def to_pdf(
         self,
+        *,
         out_filename: Path,
         image_filename: Optional[Path] = None,
         show_bounding_boxes: bool = False,
@@ -232,13 +296,11 @@ class HocrTransform:
                 )
 
         found_lines = False
-        for line in sorted(
-            chain(
-                self.hocr.iterfind(self._child_xpath('span', 'ocr_header')),
-                self.hocr.iterfind(self._child_xpath('span', 'ocr_line')),
-                self.hocr.iterfind(self._child_xpath('span', 'ocr_textfloat')),
-            ),
-            key=self.topdown_position,
+        for line in (
+            element
+            for element in self.hocr.iterfind(self._child_xpath('span'))
+            if 'class' in element.attrib
+            and element.attrib['class'] in {'ocr_header', 'ocr_line', 'ocr_textfloat'}
         ):
             found_lines = True
             self._do_line(
@@ -280,13 +342,15 @@ class HocrTransform:
     def _do_line(
         self,
         pdf: Canvas,
-        line,
+        line: Optional[Element],
         elemclass: str,
         fontname: str,
         invisible_text: bool,
         interword_spaces: bool,
         show_bounding_boxes: bool,
     ):
+        if not line:
+            return
         pxl_line_coords = self.element_coordinates(line)
         line_box = self.pt_from_pixel(pxl_line_coords)
         line_height = line_box.y2 - line_box.y1
@@ -423,10 +487,10 @@ if __name__ == "__main__":
     parser.add_argument('outputfile', help='Path to the PDF file to be generated')
     args = parser.parse_args()
 
-    hocr = HocrTransform(args.hocrfile, args.resolution)
+    hocr = HocrTransform(hocr_filename=args.hocrfile, dpi=args.resolution)
     hocr.to_pdf(
-        args.outputfile,
-        args.image,
-        args.boundingboxes,
+        out_filename=args.outputfile,
+        image_filename=args.image,
+        show_bounding_boxes=args.boundingboxes,
         interword_spaces=args.interword_spaces,
     )

src/ocrmypdf/leptonica.py

@@ -15,7 +15,6 @@ import logging
 import os
 import sys
 import threading
-import warnings
 from collections import deque
 from collections.abc import Sequence
 from contextlib import suppress
@@ -24,18 +23,20 @@ from functools import lru_cache
 from io import BytesIO, UnsupportedOperation
 from os import fspath
 from tempfile import TemporaryFile
+from warnings import warn
 
 from ocrmypdf.exceptions import MissingDependencyError
 from ocrmypdf.lib._leptonica import ffi
-from ocrmypdf.subprocess import shim_paths_with_program_files
 
 # pylint: disable=protected-access
 
 logger = logging.getLogger(__name__)
 
 if os.name == 'nt':
+    from ocrmypdf.subprocess._windows import shim_env_path
+
     libname = 'liblept-5'
-    os.environ['PATH'] = shim_paths_with_program_files()
+    os.environ['PATH'] = shim_env_path()
 else:
     libname = 'lept'
 _libpath = find_library(libname)
@@ -58,9 +59,9 @@ if not _libpath:
         ---------------------------------------------------------------------
         """
     )
-if os.name == 'nt':  
-    # On Windows, recent versions of libpng require zlib. We have to make sure 
-    # the zlib version being loaded is the same one that libpng was built with. 
+if os.name == 'nt':
+    # On Windows, recent versions of libpng require zlib. We have to make sure
+    # the zlib version being loaded is the same one that libpng was built with.
     # This tries to import zlib from Tesseract's installation folder, falling back
     # to find_library() if liblept is being loaded from somewhere else.
     # Loading zlib from other places could cause a version mismatch
@@ -87,9 +88,9 @@ except ffi.error as e:
 
 class _LeptonicaErrorTrap_Redirect:
     """
-    Context manager to trap errors reported by Leptonica.
+    Context manager to trap errors reported by Leptonica < 1.79 or on Apple Silicon.
 
-    Leptonica's error return codes don't provide much informatino about what
+    Leptonica's error return codes don't provide much information about what
     went wrong. Leptonica does, however, write more detailed errors to stderr
     (provided this is not disabled at compile time). The Leptonica source
     code is very consistent in its use of macros to generate errors.
@@ -113,8 +114,10 @@ class _LeptonicaErrorTrap_Redirect:
         # Save the old stderr, and redirect stderr to temporary file
         self.leptonica_lock.acquire()
         try:
-            with suppress(AttributeError):
-                sys.stderr.flush()
+            # It would make sense to do sys.stderr.flush() here, but that can deadlock
+            # due to https://bugs.python.org/issue6721. So don't flush. Pretend
+            # there's nothing important in sys.stderr. If the user cared they would
+            # be using Leptonica 1.79 or later anyway to avoid this mess.
             self.copy_of_stderr = os.dup(sys.stderr.fileno())
             os.dup2(self.tmpfile.fileno(), sys.stderr.fileno(), inheritable=False)
         except AttributeError:
@@ -169,20 +172,6 @@ tls = threading.local()
 tls.trap = None
 
 
-@ffi.callback("void(char *)")
-def _stderr_handler(cstr):
-    msg = ffi.string(cstr).decode(errors='replace')
-    if msg.startswith("Error"):
-        logger.error(msg)
-    elif msg.startswith("Warning"):
-        logger.warning(msg)
-    else:
-        logger.debug(msg)
-    if tls.trap is not None:
-        tls.trap.append(msg)
-    return
-
-
 class _LeptonicaErrorTrap_Queue:
     def __init__(self):
         self.queue = deque()
@@ -203,18 +192,40 @@ class _LeptonicaErrorTrap_Queue:
         if 'Error' in output:
             if 'image file not found' in output:
                 raise FileNotFoundError()
-            if 'pixWrite: stream not opened' in output:
+            elif 'pixWrite: stream not opened' in output:
                 raise LeptonicaIOError()
-            if 'index not valid' in output:
+            elif 'index not valid' in output:
                 raise IndexError()
-            raise LeptonicaError(output)
+            elif 'pixGetInvBackgroundMap: w and h must be >= 5' in output:
+                logger.warning(
+                    "Leptonica attempted to remove background from a low resolution - "
+                    "you may want to review in a PDF viewer"
+                )
+            else:
+                raise LeptonicaError(output)
         return False
 
 
 try:
+
+    @ffi.callback("void(char *)")
+    def _stderr_handler(cstr):
+        msg = ffi.string(cstr).decode(errors='replace')
+        if msg.startswith("Error"):
+            logger.error(msg)
+        elif msg.startswith("Warning"):
+            logger.warning(msg)
+        else:
+            logger.debug(msg)
+        if tls.trap is not None:
+            tls.trap.append(msg)
+        return
+
     lept.leptSetStderrHandler(_stderr_handler)
-except ffi.error:
+except (ffi.error, MemoryError):
     # Pre-1.79 Leptonica does not have leptSetStderrHandler
+    # And some platforms, notably Apple ARM 64, do not allow the write+execute
+    # memory needed to set up the callback function.
     _LeptonicaErrorTrap = _LeptonicaErrorTrap_Redirect
 else:
     # 1.79 have this new symbol
@@ -386,7 +397,7 @@ class Pix(LeptonicaObject):
 
     @classmethod
     def read(cls, path):
-        warnings.warn('Use Pix.open() instead', DeprecationWarning)
+        warn('Use Pix.open() instead', DeprecationWarning)
         return cls.open(path)
 
     @classmethod
@@ -651,6 +662,9 @@ class Pix(LeptonicaObject):
         bg_val=200,
         smooth_kernel=(2, 1),
     ):
+        if self.width < tile_size[0] or self.height < tile_size[1]:
+            logger.info("Skipped pixMaskedThreshOnBackgroundNorm on small image")
+            return self
         # Background norm doesn't work on color mapped Pix, so remove colormap
         target_pix = self.remove_colormap(lept.REMOVE_CMAP_BASED_ON_SRC)
         with _LeptonicaErrorTrap():

src/ocrmypdf/optimize.py

@@ -9,12 +9,9 @@ import logging
 import sys
 import tempfile
 from collections import defaultdict
-from functools import partial
-from io import BytesIO
 from os import fspath
 from pathlib import Path
 from typing import (
-    Any,
     Callable,
     Dict,
     Iterator,
@@ -25,21 +22,19 @@ from typing import (
     Optional,
     Sequence,
     Tuple,
-    Union,
 )
 
 import img2pdf
 import pikepdf
 from pikepdf import Dictionary, Name, Object, Pdf, PdfImage
 from PIL import Image
-from tqdm import tqdm
 
 from ocrmypdf import leptonica
-from ocrmypdf._concurrent import exec_progress_pool
+from ocrmypdf._concurrent import Executor, SerialExecutor
 from ocrmypdf._exec import jbig2enc, pngquant
 from ocrmypdf._jobcontext import PdfContext
 from ocrmypdf.exceptions import OutputFileAccessError
-from ocrmypdf.helpers import deprecated, safe_symlink
+from ocrmypdf.helpers import safe_symlink
 
 log = logging.getLogger(__name__)
 
@@ -50,7 +45,7 @@ DEFAULT_PNG_QUALITY = 70
 Xref = NewType('Xref', int)
 
 
-class XrefExt(NamedTuple):
+class XrefExt(NamedTuple):  # pylint: disable=inherit-non-class
     xref: Xref
     ext: str
 
@@ -67,18 +62,20 @@ def jpg_name(root: Path, xref: Xref) -> Path:
     return img_name(root, xref, '.jpg')
 
 
-def tif_name(root: Path, xref: Xref) -> Path:
-    return img_name(root, xref, '.tif')
-
-
 def extract_image_filter(
     pike: Pdf, root: Path, image: Object, xref: Xref
 ) -> Optional[Tuple[PdfImage, Tuple[Name, Object]]]:
+    del pike  # unused args
+    del root
+
     if image.Subtype != Name.Image:
         return None
     if image.Length < 100:
         log.debug(f"Skipping small image, xref {xref}")
         return None
+    if image.Width < 8 or image.Height < 8:  # Issue 732
+        log.debug(f"Skipping oddly sized image, xref {xref}")
+        return None
 
     pim = PdfImage(image)
 
@@ -95,6 +92,10 @@ def extract_image_filter(
         log.debug(f"Skipping JPEG2000 iamge, xref {xref}")
         return None  # Don't do JPEG2000
 
+    if filtdp[0] == Name.CCITTFaxDecode and filtdp[1].get('/K', 0) >= 0:
+        log.debug(f"Skipping CCITT Group 3 image, xref {xref}")
+        return None  # pikepdf doesn't support Group 3 yet
+
     if Name.Decode in image:
         log.debug(f"Skipping image with Decode table, xref {xref}")
         return None  # Don't mess with custom Decode tables
@@ -105,6 +106,8 @@ def extract_image_filter(
 def extract_image_jbig2(
     *, pike: pikepdf.Pdf, root: Path, image: Object, xref: Xref, options
 ) -> Optional[XrefExt]:
+    del options  # unused arg
+
     result = extract_image_filter(pike, root, image, xref)
     if result is None:
         return None
@@ -115,25 +118,29 @@ def extract_image_jbig2(
         and filtdp[0] != Name.JBIG2Decode
         and jbig2enc.available()
     ):
-        try:
-            # Save any colorspace associated with the image, so that we
-            # will export a pure 1-bit PNG with no palette or ICC profile.
-            # Showing the palette or ICC to jbig2enc will cause it to perform
-            # colorspace transform to 1bpp, which will conflict the palette or
-            # ICC if it exists.
-            colorspace = pim.obj.ColorSpace
-            # Set to DeviceGray temporarily; we already in 1 bpc.
-            pim.obj.ColorSpace = pikepdf.Name.DeviceGray
-            imgname = root / f'{xref:08d}'
-            with imgname.open('wb') as f:
-                ext = pim.extract_to(stream=f)
-            imgname.rename(imgname.with_suffix(ext))
-        except pikepdf.UnsupportedImageTypeError:
-            return None
-        finally:
-            # Restore image colorspace after temporarily setting it to DeviceGray
-            pim.obj.ColorSpace = colorspace
-        return XrefExt(xref, ext)
+        # Save any colorspace associated with the image, so that we
+        # will export a pure 1-bit PNG with no palette or ICC profile.
+        # Showing the palette or ICC to jbig2enc will cause it to perform
+        # colorspace transform to 1bpp, which will conflict the palette or
+        # ICC if it exists.
+        colorspace = pim.obj.get(pikepdf.Name.ColorSpace, None)
+        if colorspace is not None or pim.image_mask:
+            try:
+                # Set to DeviceGray temporarily; we already in 1 bpc.
+                pim.obj.ColorSpace = pikepdf.Name.DeviceGray
+                imgname = root / f'{xref:08d}'
+                with imgname.open('wb') as f:
+                    ext = pim.extract_to(stream=f)
+                imgname.rename(imgname.with_suffix(ext))
+            except pikepdf.UnsupportedImageTypeError:
+                return None
+            finally:
+                # Restore image colorspace after temporarily setting it to DeviceGray
+                if colorspace is not None:
+                    pim.obj.ColorSpace = colorspace
+                else:
+                    del pim.obj.ColorSpace
+            return XrefExt(xref, ext)
     return None
 
 
@@ -149,11 +156,6 @@ def extract_image_generic(
     if pim.bits_per_component == 1:
         return None
 
-    try:
-        pim.indexed  # pikepdf 1.6.3 can't handle [/Indexed [/Array...]]
-    except NotImplementedError:
-        return None
-
     if filtdp[0] == Name.DCTDecode and options.optimize >= 2:
         # This is a simple heuristic derived from some training data, that has
         # about a 70% chance of guessing whether the JPEG is high quality,
@@ -209,7 +211,10 @@ def extract_image_generic(
 
 
 def extract_images(
-    pike: Pdf, root: Path, options, extract_fn: Callable[..., Optional[XrefExt]],
+    pike: Pdf,
+    root: Path,
+    options,
+    extract_fn: Callable[..., Optional[XrefExt]],
 ) -> Iterator[Tuple[int, XrefExt]]:
     """Extract image using extract_fn
 
@@ -256,8 +261,8 @@ def extract_images(
             result = extract_fn(
                 pike=pike, root=root, image=image, xref=xref, options=options
             )
-        except Exception as e:  # pylint: disable=broad-except
-            log.debug("Image xref %s, error %s", xref, repr(e))
+        except Exception:  # pylint: disable=broad-except
+            log.exception(f"While extracting image xref {xref}, an error occurred")
             errors += 1
         else:
             if result:
@@ -290,26 +295,22 @@ def extract_images_jbig2(pike: Pdf, root: Path, options) -> Dict[int, List[XrefE
         group = pageno // options.jbig2_page_group_size
         jbig2_groups[group].append(xref_ext)
 
-    # Elide empty groups
-    jbig2_groups = {
-        group: xrefs for group, xrefs in jbig2_groups.items() if len(xrefs) > 0
-    }
     log.debug("Optimizable images: JBIG2 groups: %s", (len(jbig2_groups),))
     return jbig2_groups
 
 
 def _produce_jbig2_images(
-    jbig2_groups: Dict[int, List[XrefExt]], root: Path, options
+    jbig2_groups: Dict[int, List[XrefExt]], root: Path, options, executor: Executor
 ) -> None:
     """Produce JBIG2 images from their groups"""
 
     def jbig2_group_args(root: Path, groups: Dict[int, List[XrefExt]]):
         for group, xref_exts in groups.items():
             prefix = f'group{group:08d}'
-            yield dict(
-                cwd=fspath(root),
-                infiles=(img_name(root, xref, ext) for xref, ext in xref_exts),
-                out_prefix=prefix,
+            yield (
+                fspath(root),  # =cwd
+                (img_name(root, xref, ext) for xref, ext in xref_exts),  # =infiles
+                prefix,  # =out_prefix
             )
 
     def jbig2_single_args(root, groups: Dict[int, List[XrefExt]]):
@@ -318,23 +319,20 @@ def _produce_jbig2_images(
             # Second loop is to ensure multiple images per page are unpacked
             for n, xref_ext in enumerate(xref_exts):
                 xref, ext = xref_ext
-                yield dict(
-                    cwd=fspath(root),
-                    infile=img_name(root, xref, ext),
-                    outfile=root / f'{prefix}.{n:04d}',
+                yield (
+                    fspath(root),
+                    img_name(root, xref, ext),
+                    root / f'{prefix}.{n:04d}',
                 )
 
-    def convert_generic(fn, kwargs_dict):
-        return fn(**kwargs_dict)
-
     if options.jbig2_page_group_size > 1:
         jbig2_args = jbig2_group_args
-        jbig2_convert = partial(convert_generic, jbig2enc.convert_group)
+        jbig2_convert = jbig2enc.convert_group_mp
     else:
         jbig2_args = jbig2_single_args
-        jbig2_convert = partial(convert_generic, jbig2enc.convert_single)
+        jbig2_convert = jbig2enc.convert_single_mp
 
-    exec_progress_pool(
+    executor(
         use_threads=True,
         max_workers=options.jobs,
         tqdm_kwargs=dict(
@@ -349,7 +347,11 @@ def _produce_jbig2_images(
 
 
 def convert_to_jbig2(
-    pike: Pdf, jbig2_groups: Dict[int, List[XrefExt]], root: Path, options
+    pike: Pdf,
+    jbig2_groups: Dict[int, List[XrefExt]],
+    root: Path,
+    options,
+    executor: Executor,
 ) -> None:
     """Convert images to JBIG2 and insert into PDF.
 
@@ -364,7 +366,7 @@ def convert_to_jbig2(
     and needs no dictionary. Currently this must be lossless JBIG2.
     """
 
-    _produce_jbig2_images(jbig2_groups, root, options)
+    _produce_jbig2_images(jbig2_groups, root, options, executor)
 
     for group, xref_exts in jbig2_groups.items():
         prefix = f'group{group:08d}'
@@ -388,27 +390,53 @@ def convert_to_jbig2(
             )
 
 
-def transcode_jpegs(pike: Pdf, jpegs: Sequence[Xref], root: Path, options) -> None:
-    for xref in tqdm(
-        jpegs, desc="JPEGs", unit='image', disable=not options.progress_bar
-    ):
-        in_jpg = jpg_name(root, xref)
-        opt_jpg = in_jpg.with_suffix('.opt.jpg')
+def _optimize_jpeg(args):
+    xref, in_jpg, opt_jpg, jpeg_quality = args
 
-        # This produces a debug warning from PIL
-        # DEBUG:PIL.Image:Error closing: 'NoneType' object has no attribute
-        # 'close'.  Seems to be mostly harmless
-        # https://github.com/python-pillow/Pillow/issues/1144
-        with Image.open(in_jpg) as im:
-            im.save(opt_jpg, optimize=True, quality=options.jpeg_quality)
+    # This may produce a debug warning from PIL
+    # DEBUG:PIL.Image:Error closing: 'NoneType' object has no attribute
+    # 'close'.  Seems to be mostly harmless
+    # https://github.com/python-pillow/Pillow/issues/1144
+    with Image.open(in_jpg) as im:
+        im.save(opt_jpg, optimize=True, quality=jpeg_quality)
 
-        if opt_jpg.stat().st_size > in_jpg.stat().st_size:
-            log.debug("xref %s, jpeg, made larger - skip", xref)
-            continue
+    if opt_jpg.stat().st_size > in_jpg.stat().st_size:
+        log.debug("xref %s, jpeg, made larger - skip", xref)
+        opt_jpg.unlink()
+        opt_jpg = None
+    return xref, opt_jpg
 
-        compdata = leptonica.CompressedData.open(opt_jpg)
-        im_obj = pike.get_object(xref, 0)
-        im_obj.write(compdata.read(), filter=Name.DCTDecode)
+
+def transcode_jpegs(
+    pike: Pdf, jpegs: Sequence[Xref], root: Path, options, executor
+) -> None:
+    def jpeg_args():
+        for xref in jpegs:
+            in_jpg = jpg_name(root, xref)
+            opt_jpg = in_jpg.with_suffix('.opt.jpg')
+            yield xref, in_jpg, opt_jpg, options.jpeg_quality
+
+    def finish_jpeg(result, pbar):
+        xref, opt_jpg = result
+        if opt_jpg:
+            compdata = leptonica.CompressedData.open(opt_jpg)
+            im_obj = pike.get_object(xref, 0)
+            im_obj.write(compdata.read(), filter=Name.DCTDecode)
+        pbar.update()
+
+    executor(
+        use_threads=True,  # Processes are significantly slower at this task
+        max_workers=options.jobs,
+        tqdm_kwargs=dict(
+            desc="JPEGs",
+            total=len(jpegs),
+            unit='image',
+            disable=not options.progress_bar,
+        ),
+        task=_optimize_jpeg,
+        task_arguments=jpeg_args(),
+        task_finished=finish_jpeg,
+    )
 
 
 def _transcode_png(pike: Pdf, filename: Path, xref: Xref) -> bool:
@@ -458,6 +486,7 @@ def transcode_pngs(
     image_name_fn: Callable[[Path, Xref], Path],
     root: Path,
     options,
+    executor,
 ) -> None:
     modified: MutableSet[Xref] = set()
     if options.optimize >= 2:
@@ -477,10 +506,7 @@ def transcode_pngs(
                 )
                 modified.add(xref)
 
-        def pngquant_fn(args):
-            pngquant.quantize(*args)
-
-        exec_progress_pool(
+        executor(
             use_threads=True,
             max_workers=options.jobs,
             tqdm_kwargs=dict(
@@ -489,7 +515,7 @@ def transcode_pngs(
                 unit='image',
                 disable=not options.progress_bar,
             ),
-            task=pngquant_fn,
+            task=pngquant.quantize_mp,
             task_arguments=pngquant_args(),
         )
 
@@ -498,82 +524,13 @@ def transcode_pngs(
         _transcode_png(pike, filename, xref)
 
 
-@deprecated
-def rewrite_png_as_g4(pike: Pdf, im_obj: Object, compdata) -> None:
-    im_obj.BitsPerComponent = 1
-    im_obj.Width = compdata.w
-    im_obj.Height = compdata.h
-
-    im_obj.write(compdata.read())
-
-    log.debug(f"PNG to G4 {im_obj.objgen}")
-    if Name.Predictor in im_obj:
-        del im_obj.Predictor
-    if Name.DecodeParms in im_obj:
-        del im_obj.DecodeParms
-    im_obj.DecodeParms = Dictionary(
-        K=-1, BlackIs1=bool(compdata.minisblack), Columns=compdata.w
-    )
-
-    im_obj.Filter = Name.CCITTFaxDecode
-    return
-
-
-@deprecated
-def rewrite_png(pike: Pdf, im_obj: Object, compdata) -> None:
-    # When a PNG is inserted into a PDF, we more or less copy the IDAT section from
-    # the PDF and transfer the rest of the PNG headers to PDF image metadata.
-    # One thing we have to do is tell the PDF reader whether a predictor was used
-    # on the image before Flate encoding. (Typically one is.)
-    # According to Leptonica source, PDF readers don't actually need us
-    # to specify the correct predictor, they just need a value of either:
-    #   1 - no predictor
-    #   10-14 - there is a predictor
-    # Leptonica's compdata->predictor only tells TRUE or FALSE
-    # 10-14 means the actual predictor is specified in the data, so for any
-    # number >= 10 the PDF reader will use whatever the PNG data specifies.
-    # In practice Leptonica should use Paeth, 14, but 15 seems to be the
-    # designated value for "optimal". So we will use 15.
-    # See:
-    #   - PDF RM 7.4.4.4 Table 10
-    #   - https://github.com/DanBloomberg/leptonica/blob/master/src/pdfio2.c#L757
-    predictor = 15 if compdata.predictor > 0 else 1
-    dparms = Dictionary(Predictor=predictor)
-    if predictor > 1:
-        dparms.BitsPerComponent = compdata.bps  # Yes, this is redundant
-        dparms.Colors = compdata.spp
-        dparms.Columns = compdata.w
-
-    im_obj.BitsPerComponent = compdata.bps
-    im_obj.Width = compdata.w
-    im_obj.Height = compdata.h
-
-    log.debug(
-        f"PNG {im_obj.objgen}: palette={compdata.ncolors} spp={compdata.spp} bps={compdata.bps}"
-    )
-    if compdata.ncolors > 0:
-        # .ncolors is the number of colors in the palette, not the number of
-        # colors used in a true color image. The palette string is always
-        # given as RGB tuples even when the image is grayscale; see
-        # https://github.com/DanBloomberg/leptonica/blob/master/src/colormap.c#L2067
-        palette_pdf_string = compdata.get_palette_pdf_string()
-        palette_data = pikepdf.Object.parse(palette_pdf_string)
-        palette_stream = pikepdf.Stream(pike, bytes(palette_data))
-        palette = [Name.Indexed, Name.DeviceRGB, compdata.ncolors - 1, palette_stream]
-        cs = palette
-    else:
-        # ncolors == 0 means we are using a colorspace without a palette
-        if compdata.spp == 1:
-            cs = Name.DeviceGray
-        elif compdata.spp == 3:
-            cs = Name.DeviceRGB
-        elif compdata.spp == 4:
-            cs = Name.DeviceCMYK
-    im_obj.ColorSpace = cs
-    im_obj.write(compdata.read(), filter=Name.FlateDecode, decode_parms=dparms)
-
-
-def optimize(input_file: Path, output_file: Path, context, save_settings) -> None:
+def optimize(
+    input_file: Path,
+    output_file: Path,
+    context,
+    save_settings,
+    executor: Executor = SerialExecutor(),
+) -> None:
     options = context.options
     if options.optimize == 0:
         safe_symlink(input_file, output_file)
@@ -591,14 +548,14 @@ def optimize(input_file: Path, output_file: Path, context, save_settings) -> Non
         root.mkdir(exist_ok=True)
 
         jpegs, pngs = extract_images_generic(pike, root, options)
-        transcode_jpegs(pike, jpegs, root, options)
+        transcode_jpegs(pike, jpegs, root, options, executor)
         # if options.optimize >= 2:
         # Try pngifying the jpegs
         #    transcode_pngs(pike, jpegs, jpg_name, root, options)
-        transcode_pngs(pike, pngs, png_name, root, options)
+        transcode_pngs(pike, pngs, png_name, root, options, executor)
 
         jbig2_groups = extract_images_jbig2(pike, root, options)
-        convert_to_jbig2(pike, jbig2_groups, root, options)
+        convert_to_jbig2(pike, jbig2_groups, root, options, executor)
 
         target_file = output_file.with_suffix('.opt.pdf')
         pike.remove_unreferenced_resources()
@@ -613,7 +570,7 @@ def optimize(input_file: Path, output_file: Path, context, save_settings) -> Non
         )
     ratio = input_size / output_size
     savings = 1 - output_size / input_size
-    log.info(f"Optimize ratio: {ratio:.2f} savings: {(100 * savings):.1f}%")
+    log.info(f"Optimize ratio: {ratio:.2f} savings: {(savings):.1%}")
 
     if savings < 0:
         log.info("Image optimization did not improve the file - discarded")

src/ocrmypdf/pdfa.py

@@ -33,6 +33,7 @@ def _postscript_objdef(
     objtype = '/stream' if stream_name else '/dict'
 
     if stream_name:
+        assert stream_data is not None
         a85_data = base64.a85encode(stream_data, adobe=True).decode('ascii')
         yield f'{stream_name} ' + a85_data
         yield 'def'
@@ -111,7 +112,7 @@ def generate_pdfa_ps(target_filename: Path, icc: str = 'sRGB'):
 
 
 def file_claims_pdfa(filename: Path):
-    """Determines if the file claims to be PDF/A compliant
+    """Determines if the file claims to be PDF/A compliant.
 
     This only checks if the XMP metadata contains a PDF/A marker. It does not
     do full PDF/A validation.

src/ocrmypdf/pdfinfo/info.py

@@ -6,23 +6,25 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
+import atexit
 import logging
 import re
 from collections import defaultdict, namedtuple
+from contextlib import ExitStack
 from decimal import Decimal
 from enum import Enum
 from functools import partial
-from math import hypot, isclose
+from math import hypot, inf, isclose
 from os import PathLike
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Container, Iterator, Optional, Tuple, Union
 from warnings import warn
 
 import pikepdf
-from pikepdf import PdfMatrix
+from pikepdf import Object, Pdf, PdfMatrix
 
-from ocrmypdf._concurrent import exec_progress_pool
-from ocrmypdf.exceptions import EncryptedPdfError
+from ocrmypdf._concurrent import Executor, SerialExecutor
+from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
 from ocrmypdf.helpers import Resolution, available_cpu_count, pikepdf_enable_mmap
 from ocrmypdf.pdfinfo.layout import get_page_analysis, get_text_boxes
 
@@ -115,7 +117,7 @@ def _normalize_stack(graphobjs):
             yield (operands, operator)
 
 
-def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
+def _interpret_contents(contentstream: Object, initial_shorthand=UNIT_SQUARE):
     """Interpret the PDF content stream.
 
     The stack represents the state of the PDF graphics stack.  We are only
@@ -204,7 +206,7 @@ def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
     )
 
 
-def _get_dpi(ctm_shorthand, image_size):
+def _get_dpi(ctm_shorthand, image_size) -> Resolution:
     """Given the transformation matrix and image size, find the image DPI.
 
     PDFs do not include image resolution information within image data.
@@ -254,25 +256,29 @@ def _get_dpi(ctm_shorthand, image_size):
     a, b, c, d, _, _ = ctm_shorthand
 
     # Calculate the width and height of the image in PDF units
-    image_drawn_width = hypot(a, b)
-    image_drawn_height = hypot(c, d)
+    image_drawn = hypot(a, b), hypot(c, d)
 
-    # The scale of the image is pixels per unit of default user space (1/72")
-    scale_w = image_size[0] / image_drawn_width
-    scale_h = image_size[1] / image_drawn_height
-
-    # DPI = scale * 72
-    dpi_w = scale_w * 72.0
-    dpi_h = scale_h * 72.0
+    def calc(drawn, pixels, inches_per_pt=72.0):
+        # The scale of the image is pixels per unit of default user space (1/72")
+        scale = pixels / drawn if drawn != 0 else inf
+        dpi = scale * inches_per_pt
+        return dpi
 
+    dpi_w, dpi_h = (calc(image_drawn[n], image_size[n]) for n in range(2))
     return Resolution(dpi_w, dpi_h)
 
 
 class ImageInfo:
     DPI_PREC = Decimal('1.000')
 
-    def __init__(self, *, name='', pdfimage=None, inline=None, shorthand=None):
-
+    def __init__(
+        self,
+        *,
+        name='',
+        pdfimage: Optional[Object] = None,
+        inline: Optional[Object] = None,
+        shorthand=None,
+    ):
         self._name = str(name)
         self._shorthand = shorthand
 
@@ -282,6 +288,8 @@ class ImageInfo:
         elif pdfimage is not None:
             self._origin = 'xobject'
             pim = pikepdf.PdfImage(pdfimage)
+        else:
+            raise ValueError("Either pdfimage or inline must be set")
         self._width = pim.width
         self._height = pim.height
 
@@ -355,6 +363,10 @@ class ImageInfo:
     def enc(self):
         return self._enc
 
+    @property
+    def renderable(self):
+        return self.dpi.is_finite and self.width >= 0 and self.height >= 0
+
     @property
     def dpi(self):
         return _get_dpi(self._shorthand, (self._width, self._height))
@@ -371,7 +383,7 @@ class ImageInfo:
         ).format(**class_locals)
 
 
-def _find_inline_images(contentsinfo):
+def _find_inline_images(contentsinfo: ContentsInfo) -> Iterator[ImageInfo]:
     "Find inline images in the contentstream"
 
     for n, inline in enumerate(contentsinfo.inline_images):
@@ -380,7 +392,7 @@ def _find_inline_images(contentsinfo):
         )
 
 
-def _image_xobjects(container):
+def _image_xobjects(container) -> Iterator[Tuple[Object, str]]:
     """Search for all XObject-based images in the container
 
     Usually the container is a page, but it could also be a Form XObject
@@ -400,7 +412,7 @@ def _image_xobjects(container):
         return
     xobjs = resources['/XObject'].as_dict()
     for xobj in xobjs:
-        candidate = xobjs[xobj]
+        candidate: Object = xobjs[xobj]
         if not '/Subtype' in candidate:
             continue
         if candidate['/Subtype'] == '/Image':
@@ -408,7 +420,9 @@ def _image_xobjects(container):
             yield (pdfimage, xobj)
 
 
-def _find_regular_images(container, contentsinfo):
+def _find_regular_images(
+    container: Object, contentsinfo: ContentsInfo
+) -> Iterator[ImageInfo]:
     """Find images stored in the container's /Resources /XObject
 
     Usually the container is a page, but it could also be a Form XObject
@@ -432,7 +446,7 @@ def _find_regular_images(container, contentsinfo):
             yield ImageInfo(name=draw.name, pdfimage=pdfimage, shorthand=draw.shorthand)
 
 
-def _find_form_xobject_images(pdf, container, contentsinfo):
+def _find_form_xobject_images(pdf: Pdf, container: Object, contentsinfo: ContentsInfo):
     """Find any images that are in Form XObjects in the container
 
     The container may be a page, or a parent Form XObject.
@@ -464,7 +478,9 @@ def _find_form_xobject_images(pdf, container, contentsinfo):
             )
 
 
-def _process_content_streams(*, pdf, container, shorthand=None):
+def _process_content_streams(
+    *, pdf: Pdf, container: Object, shorthand=None
+) -> Iterator[Union[VectorMarker, TextMarker, ImageInfo]]:
     """Find all individual instances of images drawn in the container
 
     Usually the container is a page, but it may also be a Form XObject.
@@ -526,7 +542,7 @@ def _page_has_text(text_blocks, page_width, page_height) -> bool:
         margin_ratio * ph,  # bottom  (first quadrant: bottom < top)
     )
 
-    def rects_intersect(a, b):
+    def rects_intersect(a, b) -> bool:
         """
         Where (a,b) are 4-tuple rects (left-0, top-1, right-2, bottom-3)
         https://stackoverflow.com/questions/306316/determine-if-two-rectangles-overlap-each-other
@@ -542,7 +558,7 @@ def _page_has_text(text_blocks, page_width, page_height) -> bool:
     return has_text
 
 
-def simplify_textboxes(miner, textbox_getter):
+def simplify_textboxes(miner, textbox_getter) -> Iterator[TextboxInfo]:
     """Extract only limited content from text boxes
 
     We do this to save memory and ensure that our objects are pickleable.
@@ -556,101 +572,51 @@ def simplify_textboxes(miner, textbox_getter):
         yield TextboxInfo(box.bbox, visible, corrupt)
 
 
-def _pdf_get_pageinfo(
-    pdf, pageno: int, infile: PathLike, check_pages, detailed_analysis: bool
-):
-    pageinfo: Dict[str, Any] = {}
-    pageinfo['pageno'] = pageno
-    pageinfo['images'] = []
-
-    page = pdf.pages[pageno]
-    mediabox = [Decimal(d) for d in page.MediaBox.as_list()]
-    width_pt = mediabox[2] - mediabox[0]
-    height_pt = mediabox[3] - mediabox[1]
-
-    check_this_page = pageno in check_pages
-
-    if check_this_page and detailed_analysis:
-        pscript5_mode = str(pdf.docinfo.get('/Creator')).startswith('PScript5')
-        miner = get_page_analysis(infile, pageno, pscript5_mode)
-        pageinfo['textboxes'] = list(simplify_textboxes(miner, get_text_boxes))
-        bboxes = (box.bbox for box in pageinfo['textboxes'])
-
-        pageinfo['has_text'] = _page_has_text(bboxes, width_pt, height_pt)
-    else:
-        pageinfo['textboxes'] = []
-        pageinfo['has_text'] = None  # i.e. "no information"
-
-    userunit = page.get('/UserUnit', Decimal(1.0))
-    if not isinstance(userunit, Decimal):
-        userunit = Decimal(userunit)
-    pageinfo['userunit'] = userunit
-    pageinfo['width_inches'] = width_pt * userunit / Decimal(72.0)
-    pageinfo['height_inches'] = height_pt * userunit / Decimal(72.0)
-
-    try:
-        pageinfo['rotate'] = int(page['/Rotate'])
-    except KeyError:
-        pageinfo['rotate'] = 0
-
-    userunit_shorthand = (userunit, 0, 0, userunit, 0, 0)
-
-    if check_this_page:
-        pageinfo['has_vector'] = False
-        pageinfo['has_text'] = False
-        pageinfo['images'] = []
-        for ci in _process_content_streams(
-            pdf=pdf, container=page, shorthand=userunit_shorthand
-        ):
-            if isinstance(ci, VectorMarker):
-                pageinfo['has_vector'] = True
-            elif isinstance(ci, TextMarker):
-                pageinfo['has_text'] = True
-            elif isinstance(ci, ImageInfo):
-                pageinfo['images'].append(ci)
-            else:
-                raise NotImplementedError()
-    else:
-        pageinfo['has_vector'] = None  # i.e. "no information"
-        pageinfo['has_text'] = None
-        pageinfo['images'] = None
-
-    if pageinfo['images']:
-        dpi = Resolution(0.0, 0.0).take_max(image.dpi for image in pageinfo['images'])
-        pageinfo['dpi'] = dpi
-        pageinfo['width_pixels'] = int(round(dpi.x * float(pageinfo['width_inches'])))
-        pageinfo['height_pixels'] = int(round(dpi.y * float(pageinfo['height_inches'])))
-
-    return pageinfo
-
-
 worker_pdf = None
 
 
-def _pdf_pageinfo_sync_init(infile):
+def _pdf_pageinfo_sync_init(pdf: Pdf, infile: Path, pdfminer_loglevel):
     global worker_pdf  # pylint: disable=global-statement
     pikepdf_enable_mmap()
-    # If this function is called as a thread initializer, we need a messy hack
-    # to close worker_pdf. If called as a process, it will be released when the
-    # process is terminated.
-    worker_pdf = pikepdf.open(infile)
+
+    logging.getLogger('pdfminer').setLevel(pdfminer_loglevel)
+
+    # If the pdf is not opened, open a copy for our worker process to use
+    if pdf is None:
+        worker_pdf = pikepdf.open(infile)
+
+        def on_process_close():
+            worker_pdf.close()
+
+        # Close when this process exits
+        atexit.register(on_process_close)
 
 
 def _pdf_pageinfo_sync(args):
-    global worker_pdf  # pylint: disable=global-statement
-    pageno, infile, check_pages, detailed_analysis = args
-    page = PageInfo(worker_pdf, pageno, infile, check_pages, detailed_analysis)
-    return page
+    pageno, thread_pdf, infile, check_pages, detailed_analysis = args
+    pdf = thread_pdf if thread_pdf is not None else worker_pdf
+    with ExitStack() as stack:
+        if not pdf:  # When called with SerialExecutor
+            pdf = stack.enter_context(pikepdf.open(infile))
+        page = PageInfo(pdf, pageno, infile, check_pages, detailed_analysis)
+        return page
 
 
 def _pdf_pageinfo_concurrent(
-    pdf, infile, progbar, max_workers, check_pages, detailed_analysis=False
+    pdf,
+    executor: Executor,
+    infile,
+    progbar,
+    max_workers,
+    check_pages,
+    detailed_analysis=False,
 ):
-    global worker_pdf  # pylint: disable=global-statement
     pages = [None] * len(pdf.pages)
 
     def update_pageinfo(result, pbar):
         page = result
+        if not page:
+            raise InputFileError("Could read a page in the PDF")
         pages[page.pageno] = page
         pbar.update()
 
@@ -658,7 +624,6 @@ def _pdf_pageinfo_concurrent(
         max_workers = available_cpu_count()
 
     total = len(pdf.pages)
-    contexts = ((n, infile, check_pages, detailed_analysis) for n in range(total))
 
     use_threads = False  # No performance gain if threaded due to GIL
     n_workers = min(1 + len(pages) // 4, max_workers)
@@ -667,34 +632,116 @@ def _pdf_pageinfo_concurrent(
         # a separate process.
         use_threads = True
 
-    try:
-        exec_progress_pool(
-            use_threads=use_threads,
-            max_workers=n_workers,
-            tqdm_kwargs=dict(
-                total=total, desc="Scanning contents", unit='page', disable=not progbar
-            ),
-            task_initializer=partial(_pdf_pageinfo_sync_init, infile),
-            task=_pdf_pageinfo_sync,
-            task_arguments=contexts,
-            task_finished=update_pageinfo,
-        )
-    finally:
-        if worker_pdf and use_threads:
-            assert n_workers == 1, "Should have only one worker when threaded"
-            # This is messy, but if we ran in thread, close worker_pdf
-            worker_pdf.close()
+    # If we use a thread, we can pass the already-open Pdf for them to use
+    # If we use processes, we pass a None which tells the init function to open its
+    # own
+    initial_pdf = pdf if use_threads else None
+
+    contexts = (
+        (n, initial_pdf, infile, check_pages, detailed_analysis) for n in range(total)
+    )
+    assert n_workers == 1 if use_threads else n_workers >= 1, "Not multithreadable"
+    executor(
+        use_threads=use_threads,
+        max_workers=n_workers,
+        tqdm_kwargs=dict(
+            total=total, desc="Scanning contents", unit='page', disable=not progbar
+        ),
+        worker_initializer=partial(
+            _pdf_pageinfo_sync_init,
+            initial_pdf,
+            infile,
+            logging.getLogger('pdfminer').level,
+        ),
+        task=_pdf_pageinfo_sync,
+        task_arguments=contexts,
+        task_finished=update_pageinfo,
+    )
     return pages
 
 
 class PageInfo:
-    def __init__(self, pdf, pageno, infile, check_pages, detailed_analysis=False):
+    def __init__(
+        self,
+        pdf: Pdf,
+        pageno: int,
+        infile: PathLike,
+        check_pages: Container[int],
+        detailed_analysis: bool = False,
+    ):
         self._pageno = pageno
         self._infile = infile
         self._detailed_analysis = detailed_analysis
-        self._pageinfo = _pdf_get_pageinfo(
-            pdf, pageno, infile, check_pages, detailed_analysis
-        )
+        self._gather_pageinfo(pdf, pageno, infile, check_pages, detailed_analysis)
+
+    def _gather_pageinfo(
+        self,
+        pdf: Pdf,
+        pageno: int,
+        infile: PathLike,
+        check_pages: Container[int],
+        detailed_analysis: bool,
+    ):
+        page = pdf.pages[pageno]
+        mediabox = [Decimal(d) for d in page.MediaBox.as_list()]
+        width_pt = mediabox[2] - mediabox[0]
+        height_pt = mediabox[3] - mediabox[1]
+
+        check_this_page = pageno in check_pages
+
+        if check_this_page and detailed_analysis:
+            pscript5_mode = str(pdf.docinfo.get('/Creator')).startswith('PScript5')
+            miner = get_page_analysis(infile, pageno, pscript5_mode)
+            self._textboxes = list(simplify_textboxes(miner, get_text_boxes))
+            bboxes = (box.bbox for box in self._textboxes)
+
+            self._has_text = _page_has_text(bboxes, width_pt, height_pt)
+        else:
+            self._textboxes = []
+            self._has_text = None  # i.e. "no information"
+
+        userunit = page.get('/UserUnit', Decimal(1.0))
+        if not isinstance(userunit, Decimal):
+            userunit = Decimal(userunit)
+        self._userunit = userunit
+        self._width_inches = width_pt * userunit / Decimal(72.0)
+        self._height_inches = height_pt * userunit / Decimal(72.0)
+
+        try:
+            self._rotate = int(page['/Rotate'])
+        except KeyError:
+            self._rotate = 0
+
+        userunit_shorthand = (userunit, 0, 0, userunit, 0, 0)
+
+        if check_this_page:
+            self._has_vector = False
+            self._has_text = False
+            self._images = []
+            for ci in _process_content_streams(
+                pdf=pdf, container=page, shorthand=userunit_shorthand
+            ):
+                if isinstance(ci, VectorMarker):
+                    self._has_vector = True
+                elif isinstance(ci, TextMarker):
+                    self._has_text = True
+                elif isinstance(ci, ImageInfo):
+                    self._images.append(ci)
+                else:
+                    raise NotImplementedError()
+        else:
+            self._has_vector = None  # i.e. "no information"
+            self._has_text = None
+            self._images = None
+
+        self._dpi = None
+        if self._images:
+            dpi = Resolution(0.0, 0.0).take_max(
+                image.dpi for image in self._images if image.renderable
+            )
+            self._dpi = dpi
+            self._width_pixels = int(round(dpi.x * float(self._width_inches)))
+            self._height_pixels = int(round(dpi.y * float(self._height_inches)))
 
     @property
     def pageno(self) -> int:
@@ -702,25 +749,25 @@ class PageInfo:
 
     @property
     def has_text(self) -> bool:
-        return self._pageinfo['has_text']
+        return self._has_text
 
     @property
     def has_corrupt_text(self) -> bool:
         if not self._detailed_analysis:
             raise NotImplementedError('Did not do detailed analysis')
-        return any(tbox.is_corrupt for tbox in self._pageinfo['textboxes'])
+        return any(tbox.is_corrupt for tbox in self._textboxes)
 
     @property
     def has_vector(self) -> bool:
-        return self._pageinfo['has_vector']
+        return self._has_vector
 
     @property
     def width_inches(self) -> Decimal:
-        return self._pageinfo['width_inches']
+        return self._width_inches
 
     @property
     def height_inches(self) -> Decimal:
-        return self._pageinfo['height_inches']
+        return self._height_inches
 
     @property
     def width_pixels(self) -> int:
@@ -732,18 +779,18 @@ class PageInfo:
 
     @property
     def rotation(self) -> int:
-        return self._pageinfo.get('rotate', None)
+        return self._rotate
 
     @rotation.setter
     def rotation(self, value):
         if value in (0, 90, 180, 270, 360, -90, -180, -270):
-            self._pageinfo['rotate'] = value
+            self._rotate = value
         else:
             raise ValueError("rotation must be a cardinal angle")
 
     @property
     def images(self):
-        return self._pageinfo['images']
+        return self._images
 
     def get_textareas(
         self, visible: Optional[bool] = None, corrupt: Optional[bool] = None
@@ -758,24 +805,22 @@ class PageInfo:
                     result = False
             return result
 
-        if 'textboxes' not in self._pageinfo:
+        if not self._textboxes:
             if visible is not None and corrupt is not None:
                 raise NotImplementedError('Incomplete information on textboxes')
-            return self._pageinfo['bboxes']
+            return self._textboxes
 
-        return (
-            obj.bbox
-            for obj in self._pageinfo['textboxes']
-            if predicate(obj, visible, corrupt)
-        )
+        return (obj.bbox for obj in self._textboxes if predicate(obj, visible, corrupt))
 
     @property
     def dpi(self) -> Resolution:
-        return self._pageinfo.get('dpi', Resolution(0.0, 0.0))
+        if self._dpi is None:
+            return Resolution(0.0, 0.0)
+        return self._dpi
 
     @property
     def userunit(self) -> Decimal:
-        return self._pageinfo.get('userunit', None)
+        return self._userunit
 
     @property
     def min_version(self) -> str:
@@ -798,10 +843,12 @@ class PdfInfo:
     def __init__(
         self,
         infile,
+        *,
         detailed_analysis: bool = False,
         progbar: bool = False,
         max_workers: int = None,
         check_pages=None,
+        executor: Executor = SerialExecutor(),
     ):
         self._infile = infile
         if check_pages is None:
@@ -812,18 +859,19 @@ class PdfInfo:
                 raise EncryptedPdfError()  # Triggered by encryption with empty passwd
             self._pages = _pdf_pageinfo_concurrent(
                 pdf,
+                executor,
                 infile,
                 progbar,
                 max_workers,
                 check_pages=check_pages,
                 detailed_analysis=detailed_analysis,
             )
-            self._needs_rendering = pdf.root.get('/NeedsRendering', False)
+            self._needs_rendering = pdf.Root.get('/NeedsRendering', False)
             self._has_acroform = False
-            if '/AcroForm' in pdf.root:
-                if len(pdf.root.AcroForm.get('/Fields', [])) > 0:
+            if '/AcroForm' in pdf.Root:
+                if len(pdf.Root.AcroForm.get('/Fields', [])) > 0:
                     self._has_acroform = True
-                elif '/XFA' in pdf.root.AcroForm:
+                elif '/XFA' in pdf.Root.AcroForm:
                     self._has_acroform = True
 
     @property

src/ocrmypdf/pdfinfo/layout.py

@@ -21,7 +21,7 @@ from pdfminer.pdffont import PDFSimpleFont, PDFUnicodeNotDefined
 from pdfminer.pdfpage import PDFPage
 from pdfminer.utils import bbox2str, matrix2str
 
-from ocrmypdf.exceptions import EncryptedPdfError
+from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
 
 STRIP_NAME = re.compile(r'[0-9]+')
 
@@ -223,6 +223,7 @@ def get_page_analysis(infile, pageno, pscript5_mode):
     )
     interp = pdfminer.pdfinterp.PDFPageInterpreter(rman, dev)
 
+    patcher = None
     if pscript5_mode:
         patcher = patch.multiple(
             'pdfminer.pdffont.PDFType3Font',
@@ -235,12 +236,17 @@ def get_page_analysis(infile, pageno, pscript5_mode):
 
     try:
         with Path(infile).open('rb') as f:
-            page = PDFPage.get_pages(f, pagenos=[pageno], maxpages=0)
-            interp.process_page(next(page))
-    except PDFTextExtractionNotAllowed:
-        raise EncryptedPdfError()
+            page_iter = PDFPage.get_pages(f, pagenos=[pageno], maxpages=0)
+            page = next(page_iter, None)
+            if page is None:
+                raise InputFileError(
+                    f"pdfminer could not process page {pageno} (counting from 0)."
+                )
+            interp.process_page(page)
+    except PDFTextExtractionNotAllowed as e:
+        raise EncryptedPdfError() from e
     finally:
-        if pscript5_mode:
+        if patcher is not None:
             patcher.stop()
 
     return dev.get_result()

src/ocrmypdf/pluginspec.py

@@ -8,24 +8,41 @@
 from abc import ABC, abstractmethod, abstractstaticmethod
 from argparse import ArgumentParser, Namespace
 from collections import namedtuple
+from logging import Handler
 from pathlib import Path
 from typing import TYPE_CHECKING, AbstractSet, List, Optional
 
 import pluggy
 
+from ocrmypdf._concurrent import Executor
 from ocrmypdf.helpers import Resolution
 
 if TYPE_CHECKING:
     from PIL import Image
 
+    # pylint: disable=ungrouped-imports
     from ocrmypdf._jobcontext import PageContext
     from ocrmypdf.pdfinfo import PdfInfo
 
+    # pylint: enable=ungrouped-imports
+
 hookspec = pluggy.HookspecMarker('ocrmypdf')
 
 # pylint: disable=unused-argument
 
 
+@hookspec(firstresult=True)
+def get_logging_console() -> Handler:
+    """Returns a custom logging handler.
+
+    Generally this is necessary when both logging output and a progress bar are both
+    outputting to ``sys.stderr``.
+
+    Note:
+        This is a :ref:`firstresult hook<firstresult>`.
+    """
+
+
 @hookspec
 def add_options(parser: ArgumentParser) -> None:
     """Allows the plugin to add its own command line and API arguments.
@@ -59,7 +76,68 @@ def check_options(options: Namespace) -> None:
     Note:
         This hook will be called from the main process, and may modify global state
         before child worker processes are forked.
-	"""
+    """
+
+
+@hookspec(firstresult=True)
+def get_executor(progressbar_class) -> Executor:
+    """Called to obtain an object that manages parallel execution.
+
+    This may be used to replace OCRmyPDF's default parallel execution system
+    with a third party alternative. For example, you could make OCRmyPDF run in a
+    distributed environment.
+
+    OCRmyPDF's executors are analogous to the standard Python executors in
+    ``conconcurrent.futures``, but they do not work the same way. Executors may
+    be reused for different, unrelated batch operations, since all of the context
+    for a given job are passed to :meth:`Executor.__call__`.
+
+    Should be of type :class:`Executor` or otherwise conforming to the protocol
+    of that call.
+
+    Arguments:
+        progressbar_class: A progress bar class, which will be created when
+
+    Note:
+        This hook will be called from the main process, and may modify global state
+        before child worker processes are forked.
+    Note:
+        This is a :ref:`firstresult hook<firstresult>`.
+    """
+
+
+@hookspec(firstresult=True)
+def get_progressbar_class():
+    """Called to obtain a class that can be used to monitor progress.
+
+    A progress bar is assumed, but this could be used for any type of monitoring.
+
+    The class should follow a tqdm-like protocol. Calling the class should return
+    a new progress bar object, which is activated with ``__enter__`` and terminated
+    ``__exit__``. An update method is called whenever the progress bar is updated.
+    Progress bar objects will not be reused; a new one will be created for each
+    group of tasks.
+
+    The progress bar is held in the main process/thread and not updated by child
+    process/threads. When a child notifies the parent of completed work, the
+    parent updates the progress bar.
+
+    The arguments are the same as `tqdm <https://github.com/tqdm/tqdm>`_ accepts.
+
+    Progress bars should never write to ``sys.stdout``, or they will corrupt the
+    output if OCRmyPDF writes a PDF to standard output.
+
+    The type of events that OCRmyPDF reports to a progress bar may change in
+    minor releases.
+
+    Here is how OCRmyPDF will use the progress bar:
+
+    Example:
+        pbar_class = pm.hook.get_progressbar_class()
+        with pbar_class(**tqdm_kwargs) as pbar:
+            ...
+            pbar.update(1)
+    """
 
 
 @hookspec
@@ -163,11 +241,11 @@ def filter_page_image(page: 'PageContext', image_filename: Path) -> Path:
     will be resized and the OCR layer misaligned. OCRmyPDF does not nothing
     to enforce these constraints; it is up to the plugin to do sensible things.
 
-    OCRmyPDF will create the PDF page based on the image format used. If you
-    convert the image to a JPEG, the output page will be created as a JPEG, etc.
-    If you change the colorspace, that change will be kept. Note that the
-    OCRmyPDF image optimization stage, if enabled, may ultimately chose a
-    different format.
+    OCRmyPDF will create the PDF page based on the image format used (unless the
+    hook is overriden). If you convert the image to a JPEG, the output page will
+    be created as a JPEG, etc. If you change the colorspace, that change will be
+    kept. Note that the OCRmyPDF image optimization stage, if enabled, may
+    ultimately chose a different format.
 
     If the return value is a file that does not exist, ``FileNotFoundError``
     will occur. The return value should be a path to a file in the same folder
@@ -185,6 +263,50 @@ def filter_page_image(page: 'PageContext', image_filename: Path) -> Path:
     """
 
 
+@hookspec(firstresult=True)
+def filter_pdf_page(
+    page: 'PageContext', image_filename: Path, output_pdf: Path
+) -> Path:
+    """Called to convert a filtered whole page image into a PDF.
+
+    A whole page image is only produced when preprocessing command line arguments
+    are issued or when ``--force-ocr`` is issued. If no whole page is image is
+    produced for a given page, this function will not be called. This is not
+    the image that will be shown to OCR. The whole page image is filtered in
+    the hook above, ``filter_page_image``, then this function is called for
+    PDF conversion.
+
+    This function will only be called when OCRmyPDF runs in a mode such as
+    "force OCR" mode where rasterizing of all content is performed.
+
+    Clever things could be done at this stage such as segmenting the page image into
+    color regions or vector equivalents.
+
+    The provider of the hook implementation is responsible for ensuring that the
+    OCR text layer is aligned with the PDF produced here, or text misalignment
+    will result.
+
+    Currently this function must produce a single page PDF or the pipeline will
+    fail.  If the intent is to remove the PDF, then create a single page empty
+    PDF.
+
+    Args:
+        page: Context for this page.
+        image_filename: Filename of the input image used to create output_pdf,
+            for "reference" if recreating the output_pdf entirely.
+        output_pdf: The previous created output_pdf.
+
+    Returns:
+        output_pdf
+
+    Note:
+        This hook will be called from child processes. Modifying global state
+        will not affect the main process or other child processes.
+    Note:
+        This is a :ref:`firstresult hook<firstresult>`.
+    """
+
+
 OrientationConfidence = namedtuple('OrientationConfidence', ('angle', 'confidence'))
 """Expresses an OCR engine's confidence in page rotation.
 
@@ -280,6 +402,7 @@ def generate_pdfa(
     compression: str,
     pdf_version: str,
     pdfa_part: str,
+    progressbar_class,
 ) -> Path:
     """Generate a PDF/A.
 
@@ -302,10 +425,21 @@ def generate_pdfa(
             At its own discretion, the PDF/A generator may raise the version,
             but should not lower it.
         pdfa_part: The desired PDF/A compliance level, such as ``'2B'``.
+        progressbar_class: The class of a progress bar with a tqdm-like API. An
+            instance of this class will be initialized when PDF/A conversion
+            begins, using
+            ``instance = progressbar_class(total: int, desc: str, unit:str)``,
+            defining the number of work units, a user-visible description,
+            and the name of the work units ("page"). Then ``instance.update()``
+            will be called when a work unit is completed. If ``None``, no
+            progress information is reported.
 
     Returns:
         Path: If successful, the hook should return ``output_file``.
 
     Note:
         This is a :ref:`firstresult hook<firstresult>`.
+
+    See also:
+        https://github.com/tqdm/tqdm
     """

src/ocrmypdf/py.typed

@@ -0,0 +1 @@
+# ocrmypdf is typed

src/ocrmypdf/subprocess/__init__.py

@@ -10,29 +10,94 @@
 import logging
 import os
 import re
-import shutil
 import sys
 from collections.abc import Mapping
 from contextlib import suppress
-from distutils.version import LooseVersion
+from distutils.version import LooseVersion, Version
 from functools import lru_cache
 from pathlib import Path
-from subprocess import PIPE, STDOUT, CalledProcessError
+from subprocess import PIPE, STDOUT, CalledProcessError, CompletedProcess, Popen
 from subprocess import run as subprocess_run
+from typing import Callable, Optional, Type, Union
 
 from ocrmypdf.exceptions import MissingDependencyError
 
+# pylint: disable=logging-format-interpolation
+
 log = logging.getLogger(__name__)
 
 
-def run(args, *, env=None, **kwargs):
+def run(args, *, env=None, logs_errors_to_stdout=False, **kwargs):
     """Wrapper around :py:func:`subprocess.run`
 
     The main purpose of this wrapper is to log subprocess output in an orderly
     fashion that indentifies the responsible subprocess. An additional
     task is that this function goes to greater lengths to find possible Windows
     locations of our dependencies when they are not on the system PATH.
+
+    Arguments should be identical to ``subprocess.run``, except for following:
+
+    Arguments:
+        logs_errors_to_stdout: If True, indicates that the process writes its error
+            messages to stdout rather than stderr, so stdout should be logged
+            if there is an error. If False, stderr is logged. Could be used with
+            stderr=STDOUT, stdout=PIPE for example.
     """
+    args, env, process_log, _text = _fix_process_args(args, env, kwargs)
+
+    stderr = None
+    stderr_name = 'stderr' if not logs_errors_to_stdout else 'stdout'
+    try:
+        proc = subprocess_run(args, env=env, **kwargs)
+    except CalledProcessError as e:
+        stderr = getattr(e, stderr_name, None)
+        raise
+    else:
+        stderr = getattr(proc, stderr_name, None)
+    finally:
+        if process_log.isEnabledFor(logging.DEBUG) and stderr:
+            with suppress(AttributeError, UnicodeDecodeError):
+                stderr = stderr.decode('utf-8', 'replace')
+            if logs_errors_to_stdout:
+                process_log.debug("stdout/stderr = %s", stderr)
+            else:
+                process_log.debug("stderr = %s", stderr)
+    return proc
+
+
+def run_polling_stderr(args, *, callback, check=False, env=None, **kwargs):
+    """Run a process like ``ocrmypdf.subprocess.run``, and poll stderr.
+
+    Every line of produced by stderr will be forwarded to the callback function.
+    The intended use is monitoring progress of subprocesses that output their
+    own progress indicators. In addition, each line will be logged if debug
+    logging is enabled.
+
+    Requires stderr to be opened in text mode for ease of handling errors. In
+    addition the expected encoding= and errors= arguments should be set. Note
+    that if stdout is already set up, it need not be binary.
+    """
+    args, env, process_log, text = _fix_process_args(args, env, kwargs)
+    assert text, "Must use text=True"
+
+    with Popen(args, env=env, **kwargs) as proc:
+        lines = []
+        while proc.poll() is None:
+            for msg in iter(proc.stderr.readline, ''):
+                if process_log.isEnabledFor(logging.DEBUG):
+                    process_log.debug(msg.strip())
+                callback(msg)
+                lines.append(msg)
+        stderr = ''.join(lines)
+
+        if check and proc.returncode != 0:
+            raise CalledProcessError(proc.returncode, args, output=None, stderr=stderr)
+        return CompletedProcess(args, proc.returncode, None, stderr=stderr)
+
+
+def _fix_process_args(args, env, kwargs):
+    assert 'universal_newlines' not in kwargs, "Use text= instead of universal_newlines"
+
     if not env:
         env = os.environ
 
@@ -40,53 +105,23 @@ def run(args, *, env=None, **kwargs):
     program = args[0]
 
     if os.name == 'nt':
-        args = _fix_windows_args(program, args, env)
+        from ocrmypdf.subprocess._windows import fix_windows_args
+
+        args = fix_windows_args(program, args, env)
 
     log.debug("Running: %s", args)
-    process_log = log.getChild('subprocess.' + os.path.basename(program))
-    if sys.version_info < (3, 7) and os.name == 'nt':
-        # Can't use close_fds=True on Windows with Python 3.6 or older
-        # https://bugs.python.org/issue19575, etc.
-        kwargs['close_fds'] = False
-
-    stderr = None
-    try:
-        proc = subprocess_run(args, env=env, **kwargs)
-    except CalledProcessError as e:
-        stderr = getattr(e, 'stderr', None)
-        raise
-    else:
-        stderr = getattr(proc, 'stderr', None)
-    finally:
-        if process_log.isEnabledFor(logging.DEBUG) and stderr:
-            with suppress(AttributeError, UnicodeDecodeError):
-                stderr = stderr.decode('utf-8', 'replace')
-            process_log.debug("stderr = %s", stderr)
-    return proc
-
-
-def _fix_windows_args(program, args, env):
-    """Adjust our desired program and command line arguments for use on Windows"""
-
-    if sys.version_info < (3, 8):
-        # bpo-33617 - Windows needs manual Path -> str conversion
-        args = [os.fspath(arg) for arg in args]
-        program = os.fspath(program)
-
-    # If we are running a .py on Windows, ensure we call it with this Python
-    # (to support test suite shims)
-    if program.lower().endswith('.py'):
-        args = [sys.executable] + args
-
-    paths = os.pathsep.join(os.get_exec_path(env))
-    if not shutil.which(args[0], path=paths):
-        # If the program we want is not on the PATH, add some interesting
-        # locations in %PROGRAMFILES% to the PATH and try again
-        shimmed_path = shim_paths_with_program_files(env)
-        new_args0 = shutil.which(args[0], path=shimmed_path)
-        if new_args0:
-            args[0] = new_args0
-    return args
+    process_log = log.getChild(os.path.basename(program))
+    text = kwargs.get('text', False)
+    if sys.version_info < (3, 7):
+        if os.name == 'nt':
+            # Can't use close_fds=True on Windows with Python 3.6 or older
+            # https://bugs.python.org/issue19575, etc.
+            kwargs['close_fds'] = False
+        if 'text' in kwargs:
+            # Convert run(...text=) to run(...universal_newlines=) for Python 3.6
+            kwargs['universal_newlines'] = kwargs['text']
+            del kwargs['text']
+    return args, env, process_log, text
 
 
 @lru_cache(maxsize=None)
@@ -107,7 +142,7 @@ def get_version(
         proc = run(
             args_prog,
             close_fds=True,
-            universal_newlines=True,
+            text=True,
             stdout=PIPE,
             stderr=STDOUT,
             check=True,
@@ -126,44 +161,18 @@ def get_version(
         raise MissingDependencyError(
             f"Could not find program '{program}' on the PATH"
         ) from e
-    try:
-        version = re.match(regex, output.strip()).group(1)
-    except AttributeError as e:
+
+    match = re.match(regex, output.strip())
+    if not match:
         raise MissingDependencyError(
             f"The program '{program}' did not report its version. "
             f"Message was:\n{output}"
         )
+    version = match.group(1)
 
     return version
 
 
-def shim_paths_with_program_files(env=None):
-    if not env:
-        env = os.environ
-    program_files = env.get('PROGRAMFILES', '')
-    if not program_files:
-        return env.get('PATH', '')
-
-    def path_walker():
-        for path in Path(program_files).iterdir():
-            if not path.is_dir():
-                continue
-            if path.name.lower() == 'tesseract-ocr':
-                yield path
-            elif path.name.lower() == 'gs':
-                yield from (p for p in path.glob('**/bin') if p.is_dir())
-
-    paths = sorted(
-        (p for p in path_walker()), key=lambda p: (p.name, p.parent.name), reverse=True
-    )
-    paths.extend(
-        Path(str_path)
-        for str_path in os.get_exec_path(env)
-        if Path(str_path) not in set(paths)
-    )
-    return os.pathsep.join(str(p) for p in paths)
-
-
 missing_program = '''
 The program '{program}' could not be executed or was not found on your
 system PATH.
@@ -258,13 +267,30 @@ def _error_old_version(program, package, need_version, found_version, required_f
 
 def check_external_program(
     *,
-    program,
-    package,
-    version_checker,
-    need_version,
-    required_for=None,
+    program: str,
+    package: str,
+    version_checker: Union[str, Callable],
+    need_version: str,
+    required_for: Optional[str] = None,
     recommended=False,
+    version_parser: Type[Version] = LooseVersion,
 ):
+    """Check for required version of external program and raise exception if not.
+
+    Args:
+        program: The name of the program to test.
+        package: The name of a software package that typically supplies this program.
+            Usually the same as program.
+        version_check: A callable without arguments that retrieves the installed
+            version of program.
+        need_version: The minimum required version.
+        required_for: The name of an argument of feature that requires this program.
+        recommended: If this external program is recommended, instead of raising
+            an exception, log a warning and allow execution to continue.
+        version_parser: A class that should be used to parse and compare version
+            numbers. Used when version numbers do not follow standard conventions.
+    """
+
     try:
         if callable(version_checker):
             found_version = version_checker()
@@ -273,7 +299,7 @@ def check_external_program(
     except (CalledProcessError, FileNotFoundError, MissingDependencyError):
         _error_missing_program(program, package, required_for, recommended)
         if not recommended:
-            raise MissingDependencyError()
+            raise MissingDependencyError(program)
         return
 
     def remove_leading_v(s):
@@ -284,9 +310,9 @@ def check_external_program(
     found_version = remove_leading_v(found_version)
     need_version = remove_leading_v(need_version)
 
-    if found_version and LooseVersion(found_version) < LooseVersion(need_version):
+    if found_version and version_parser(found_version) < version_parser(need_version):
         _error_old_version(program, package, need_version, found_version, required_for)
         if not recommended:
-            raise MissingDependencyError()
+            raise MissingDependencyError(program)
 
     log.debug('Found %s %s', program, found_version)

src/ocrmypdf/subprocess/_windows.py

@@ -0,0 +1,162 @@
+# © 2020 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+import os
+import shutil
+import sys
+from distutils.version import LooseVersion
+from itertools import chain, filterfalse
+from pathlib import Path
+from typing import Any, Callable, Iterator, Optional, Tuple, TypeVar, cast
+
+try:
+    import winreg
+except ModuleNotFoundError as e:
+    raise ModuleNotFoundError("This module is for Windows only") from e
+
+log = logging.getLogger(__name__)
+
+T = TypeVar('T')
+
+
+def registry_enum(
+    key: winreg.HKEYType, enum_fn: Callable[[winreg.HKEYType, int], T]
+) -> Iterator[T]:
+    LIMIT = 999
+    n = 0
+    while n < LIMIT:
+        try:
+            yield enum_fn(key, n)
+            n += 1
+        except OSError:
+            break
+    if n == LIMIT:
+        raise ValueError(f"Too many registry keys under {key}")
+
+
+def registry_subkeys(key: winreg.HKEYType) -> Iterator[str]:
+    return registry_enum(key, winreg.EnumKey)
+
+
+def registry_values(key: winreg.HKEYType) -> Iterator[Tuple[str, Any, int]]:
+    return registry_enum(key, winreg.EnumValue)
+
+
+def registry_path_ghostscript(env=None) -> Iterator[Path]:
+    try:
+        with winreg.OpenKey(
+            winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Artifex\GPL Ghostscript"
+        ) as k:
+            latest_gs = max(registry_subkeys(k), key=LooseVersion)
+        with winreg.OpenKey(
+            winreg.HKEY_LOCAL_MACHINE, fr"SOFTWARE\Artifex\GPL Ghostscript\{latest_gs}"
+        ) as k:
+            _, gs_path, _ = next(registry_values(k))
+            yield Path(gs_path) / 'bin'
+    except OSError as e:
+        log.warning(e)
+
+
+def registry_path_tesseract(env=None) -> Iterator[Path]:
+    try:
+        with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Tesseract-OCR") as k:
+            for subkey, val, _valtype in registry_values(k):
+                if subkey == 'InstallDir':
+                    tesseract_path = Path(val)
+                    yield tesseract_path
+    except OSError as e:
+        log.warning(e)
+
+
+def program_files_paths(env=None) -> Iterator[Path]:
+    if not env:
+        env = os.environ
+    program_files = env.get('PROGRAMFILES', '')
+
+    def path_walker() -> Iterator[Path]:
+        for path in Path(program_files).iterdir():
+            if not path.is_dir():
+                continue
+            if path.name.lower() == 'tesseract-ocr':
+                yield path
+            elif path.name.lower() == 'gs':
+                yield from (p for p in path.glob('**/bin') if p.is_dir())
+
+    return iter(
+        sorted(
+            (p for p in path_walker()),
+            key=lambda p: (p.name, p.parent.name),
+            reverse=True,
+        )
+    )
+
+
+def paths_from_env(env=None) -> Iterator[Path]:
+    return (Path(p) for p in os.get_exec_path(env) if p)
+
+
+def shim_path(new_paths: Callable[[Any], Iterator[Path]], env=None) -> str:
+    if not env:
+        env = os.environ
+    return os.pathsep.join(str(p) for p in new_paths(env) if p)
+
+
+SHIMS = [
+    paths_from_env,
+    registry_path_ghostscript,
+    registry_path_tesseract,
+    program_files_paths,
+]
+
+
+def fix_windows_args(program, args, env):
+    """Adjust our desired program and command line arguments for use on Windows"""
+
+    if sys.version_info < (3, 8):
+        # bpo-33617 - Windows needs manual Path -> str conversion
+        args = [os.fspath(arg) for arg in args]
+        program = os.fspath(program)
+
+    # If we are running a .py on Windows, ensure we call it with this Python
+    # (to support test suite shims)
+    if program.lower().endswith('.py'):
+        args = [sys.executable] + args
+
+    # If the program we want is not on the PATH, check elsewhere
+    for shim in SHIMS:
+        shimmed_path = shim_path(shim, env)
+        new_args0 = shutil.which(args[0], path=shimmed_path)
+        if new_args0:
+            args[0] = new_args0
+            break
+
+    return args
+
+
+def unique_everseen(iterable, key=None):
+    "List unique elements, preserving order. Remember all elements ever seen."
+    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    # unique_everseen('ABBCcAD', str.lower) --> A B C D
+    seen = set()
+    seen_add = seen.add
+    if key is None:
+        key = lambda x: x
+    for element in iterable:
+        k = key(element)
+        if k not in seen:
+            seen_add(k)
+            yield element
+
+
+def shim_env_path(env=None):
+    if env is None:
+        env = os.environ
+
+    shim_paths = chain.from_iterable(shim(env) for shim in SHIMS)
+    return os.pathsep.join(
+        str(p) for p in unique_everseen(shim_paths, key=lambda p: str.casefold(str(p)))
+    )

tests/__init__.py

@@ -0,0 +1,7 @@
+# © 2021 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Empty __init__.py file

tests/cache/aspect/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin

@@ -9,7 +9,7 @@
   <meta name='ocr-capabilities' content='ocr_page ocr_carea ocr_par ocr_line ocrx_word ocrp_wconf'/>
  </head>
  <body>
-  <div class='ocr_page' id='page_1' title='image "/var/folders/2s/7t022mgj0h5cprbq0dtb1ksm0000gn/T/com.github.ocrmypdf.mrqsewbu/000001_ocr.png"; bbox 0 0 1000 800; ppageno 0'>
+  <div class='ocr_page' id='page_1' title='image "/tmp/ocrmypdf.io.zi7wmyqr/000001_ocr.png"; bbox 0 0 1000 800; ppageno 0'>
    <div class='ocr_carea' id='block_1_1' title="bbox 296 96 704 504">
     <p class='ocr_par' id='par_1_1' lang='eng' title="bbox 296 96 704 504">
      <span class='ocr_line' id='line_1_1' title="bbox 296 96 704 504; baseline 0 296; x_size 169.33333; x_descenders 42.333332; x_ascenders 42.333336">

tests/cache/cardinal/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin

@@ -9,7 +9,7 @@
   <meta name='ocr-capabilities' content='ocr_page ocr_carea ocr_par ocr_line ocrx_word ocrp_wconf'/>
  </head>
  <body>
-  <div class='ocr_page' id='page_1' title='image "/var/folders/2s/7t022mgj0h5cprbq0dtb1ksm0000gn/T/com.github.ocrmypdf.ces85e5u/000001_ocr.png"; bbox 0 0 2550 3300; ppageno 0'>
+  <div class='ocr_page' id='page_1' title='image "/tmp/ocrmypdf.io.idkfgigs/000001_ocr.png"; bbox 0 0 2550 3300; ppageno 0'>
    <div class='ocr_carea' id='block_1_1' title="bbox 582 131 1968 303">
     <p class='ocr_par' id='par_1_1' lang='eng' title="bbox 582 131 1968 303">
      <span class='ocr_header' id='line_1_1' title="bbox 882 131 1657 217; baseline 0.001 -17; x_size 85; x_descenders 16; x_ascenders 19">

tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt/txt.bin

@@ -103,7 +103,7 @@ ERASE, REPEAT, PLAY/STOP, or LOCATE.
 
 © Will sync to standard LinnDrum or Linn 9000 sync tone.
 
-® Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation.
+© Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation.
 * TEMPO may be specified in BEATS-PER-MINUTE or FRAMES-PER-BEAT at 24, 25, or 30 frames per second,
 
 (even drop frame!)
@@ -115,9 +115,9 @@ on the TAP TEMPO button.
 ¢ TEMPO CHANGES may be programmed into a sequence, with smooth transitions if desired.
 ¢ Any TIME SIGNATURE may be used, and may be changed within a song.
 
-nn
-
+linn
 Linn Electronics, Inc.
+
 18720 Oxnard Street, Tarzana, CA 91356
 (818) 708-8131 TELEX #298949 LINN UR
 

tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt/txt.bin

@@ -103,7 +103,7 @@ ERASE, REPEAT, PLAY/STOP, or LOCATE.
 
 © Will sync to standard LinnDrum or Linn 9000 sync tone.
 
-® Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation.
+© Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation.
 * TEMPO may be specified in BEATS-PER-MINUTE or FRAMES-PER-BEAT at 24, 25, or 30 frames per second,
 
 (even drop frame!)
@@ -115,9 +115,9 @@ on the TAP TEMPO button.
 ¢ TEMPO CHANGES may be programmed into a sequence, with smooth transitions if desired.
 ¢ Any TIME SIGNATURE may be used, and may be changed within a song.
 
-nn
-
+linn
 Linn Electronics, Inc.
+
 18720 Oxnard Street, Tarzana, CA 91356
 (818) 708-8131 TELEX #298949 LINN UR
 

tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt/txt.bin

@@ -1,128 +1,123 @@
-2A NNI‘I 6F6867# XATALL IE18-80L (818)
+The LinnSequencer
+32 Track MIDI Sequence Recorder
 
-9SEI6 VO “BUBZIRY, “J0aNS PIPUXO OZLEI
-“Uy ‘soTUOMOI,q UUrT
+The LinnSequencer is a state-of-the-art composition and performance tool for the professional musician. It is
 
-uut]
+extremely powerful, yet amazingly simple to learn and use. It’s many remarkable features include:
 
-“‘SUOS B UIJIM pasueyo oq ABU pue ‘posn oq AWW AYN IVNOIS AWLL AUV
-“parlsop Jr SUOTIISUBI} YIOOUIS YIM “BoueNbas eB OJUI pourtueIZOId 9q ABU SFONWHO OdINAL e
+¢ Operation is similar to multi-track tape recorder with PLAY, STOP, RECORD, FAST
+FORWARD, REWIND, and LOCATE controls.
 
-‘uonng OdNAL dV L 9) uO
+e Each of the 100 sequences contains 32 simultaneous, polyphonic tracks. Each track may
+be assigned to one of 16 MIDI channels. Simultaneously plays up to 16 polyphonic
 
-sojou Jayienb Suiddy} Aq 10 ‘syUSTIOIOUI oINUTIAI-J8g-Jesg & JO sys} UL ofquisn(pe ‘ATTeouIAUINU paiajus oq ABU OdINALL e
+synthesizers!
 
-(jouer doup u3a9)
+¢ Ultra-fast 3%” disk drive stores complex songs in seconds and holds over 110,000 notes
 
-“puooes Jed souely O€ 10 “SZ “pz 18 [LVAG-MAd-SHN VU 10 ALOANIWAAd-SLVAd U! patyoeds aq kewl OAL «
-‘uoTe1odo [SVx JO} Aj[eusoyUT JoyndUIOd 11g 9] 98108 ZHI 8g ‘poeds-ysry Bann soz] e
+per disk!
 
-"9U0} DUAS 0006 UUL] Jo wNIqUUr] prepue}s 0} OUAS [ITAA ©
+¢ One or all tracks may be TRANSPOSED at the touch of a key.
+e Exclusive real-time ERASE function makes editing FAST.
+* Exclusive REPEAT function automatically repeats any held notes at a pre-selected
 
-“ONYBA 9}OU poloapes Aue Je sas—nd jndyno 07 pewureigold 3q ACW SL Ad LNO YADONAL OML
+rhythmic value.
 
-"ALVOOT 10 GOLS/AV 1d ‘LWddad “ASV
+¢ TIMING CORRECTION works during playback and operates without ‘chopping’ notes.
 
-SUIpNpoUr ‘suOTIOUN] posn A[UOUILUOS 94] JO AUBUT [O1]UOD AJ9]OWIAI 0} PousIsse oq ACUI ST AdNI HOLIMSLOO OME «
-“SUIPIONAI I[IYM P2sesd JOU Iv $3}OU BUTISIXO—ZUIPIOIA SATON.ASOP-UON
+¢ Optional SMPTE time code synchronization.
 
-‘suoneurldxa peuoyippe sdeydsip uowng g1TqH
+© Optional remote control.
 
-oy] ‘pepsau JI ‘suoneiodo [ye yYsnosy] NOA sapins ApIespo Avfdsip QO] Joey Z7¢ 9y3—uoeIodo Urea] 0} Ased ‘aus «
+Recording a Sequence
 
-jUorel]suowtap & IO} Aepol Jayeap uur’] INOA dag ‘dISHUL
-INOA 0} UONUS}]¥ PaplAIPUN INOA SUTJOASp ITY ps pue
-p1osai ‘asoduod no Jay 0} pausisap st 1s0uenbesuur’] oy)
-Aum Aposiooid $,Jeu], ‘SS9d0Id SATTBS1D OY} YIM SOIOJIOIUT
-yey) xo]dwWI0d Os dq JOA9U P[NoUsS osn NOA AZopOuYdE} oy
+To record a sequence, simply press RECORD and PLAY,
+then play your MIDI keyboard in time to the Sequencer’s
+click track. When the sequence loops back around to bar 1,
+you’ ll hear what you played—only all timing errors will be
 
-ISTUMOIAUIO?) NOAA UOHISOdWIO)
+corrected! (Timing correction may be adjusted or defeated).
 
-"NOSpr] B Oy ‘AONUTJUT yada 0} seq Maz Se] BY] Jas UdAd
-uvd NOA ‘palisap JJ ‘souanbes Mou ¥B OVUT sjied ou] [Te Adoo
-ATesrewO Ne WI) [IM ONOS ALVAAO JeyIe80} wey}
-,deyd,, 0} UOTOUNJ ONOS ALVA ou] asn usy] ‘saouanbes
-JENPIAIpUt UI (“949 ‘snJOYD ‘aS1OA) UOTIDIS JIseq Yes
-Pl0da1 OF ST ABM JOuIOUY “(812g 666 01 dn) ysnory) ABM
+Any additional notes played will be added into the track
+— existing notes are not erased while recording!
 
-dU} [fe YORI] YORs p10991 0} ST SUOS B 9789I9 0} ABM SUG,
+FAST FORWARD, REWIND, and LOCATE controls
+may be used at any time to quickly access any location in
+your sequence for spot-recording. To overdub a new part,
+select a different track and start recording—while you
+record, the first track will play in perfect sync (unless you
+MUTE it, or SOLO another track). In this way, up to 32
+tracks may be overdubbed! All MIDI effects are recorded
+including pitch bend, modulation, velocity, aftertouch,
+sustain pedal, and program changes!
 
-SUOS & SUTVAID
+Editing
 
-*suoT}oes poJUBMUN
+To erase a wrong note, simply hold ERASE and press
+the note to be erased just before it plays in the sequence—
+when played back, it will be gone. Notes may also be
 
-SAOUIOI 0} ABM SWS dU} SoyeIodo SUV ALATAaG
+added, erased, or changed using the SINGLE STEP func-
+tion. To overdub notes at specific points within a sequence,
 
-“OBPLIq dy} PUB SNIOY PUOdAS dT]] Ud9MIAQ SIDA ISI
+Additional Features
 
-ay) Jo Adoo B JJasuT WYSE NOAA ‘afdwexs 10.f ‘UO JUSIN]JIP
+simply use LOCATE, FAST FORWARD, or REWIND to
+find the desired bar number, then start recording.
 
-B IO aouaNbas sues OY} UI—JOY OUP 0} UOTIEIO] 9UO WOT]
-$1Bq JAOUI OF NOA sMOTIe WOTIOUNS AdOO/IMASNI OULL
+The INSERT/COPY function allows you to move bars
+from one location to another—in the same sequence or a
+different one. For example, you might insert a copy of the
+first verse between the second chorus and the bridge.
+DELETE BARS operates the same way to remove
+unwanted sections,
 
-‘SUIPIONAI JIVIS Udy) “OQuINU eq porisop ay} puy
+Creating a Song
 
-0} CNIMAY 10 ‘CYVM Od LSWA “AEVOOT esn Apduns
+One way to create a song is to record each track all the
+way through (up to 999 bars). Another way is to record
+each basic section (verse, chorus, etc.) in individual
+sequences, then use the CREATE SONG function to “chain”
+them together. CREATE SONG will then automatically
+copy all the parts into a new sequence. If desired, you can
+even set the last few bars to repeat infinitely, for a fadeout.
 
-sainjeay [PUOHIPPY
+Composition Without Compromise
 
-‘gouanbas & UTYIIM s]UTOd a1y1dads 3¥ $9100 QnPIOAO OL "UOT}
--ouns dALLS ATONIS 24) Suisn pasueyo Jo ‘pasesa ‘pappe
-aq osye ABUT S9]ON ‘U0 9q ]IIM 1 “yoeq podeyd uayM
-—aouanbas oy] ul skeyd 71 a10J9q Isnf posers oq 0} d]0U ayy
-ssaid pue ASvwug ploy Aydunis ‘jou Suomm & aseso OL
+The technology you use should never be so complex that
+it interferes with the creative process. That’s precisely why
+the LinnSequencer is designed to let you compose, record
+and edit while devoting your undivided attention to your
+music. See your Linn dealer today for a demonstration!
 
-sunipa
+* Simple, easy to learn operation—the 32 character LCD display clearly guides you through all operations. If needed, the
 
-jsesdueyo ureisoid pue ‘fepod ureysns
-‘yonoplalje ‘AWOOTOA ‘UOTyeTNpow ‘pusg youd Surpnyour
-pep10del are $199JJ2 TCTIN [WV iPeqqnpseao aq Aeur syoen
-Ze 07 dn ‘Kem sie Uy *(foeI} JOyOUR OJOS 10 ALLAN
-NOA ssofum) duAS yOaysod ul Avy [[IM Yow] ISI 93 “prooar
-NOA 3[IYM—SUIPIOIA LIBIS PU YORI) TUdIOTJIP B JOaTas
-*y1ed MOU B QNPIsA0 OL, “SuIps0daJ-jods 10} aouanbes mno0k
-UI UOHBIO] Aue ssad0e ATYOIND 0} owt} Aue ye pasn aq AvUE
-SJONUOD FLIVOOT pur ‘ANIMA ‘CYVMaYOd LSVd
-{SUIPIOSAI {IY posesa JOU se So]OU SuTsTXO—
-yous} 3U} OUT poppe aq JIM poteyd sajou yeuonippe Auy
+HELP button displays additional explanations.
 
-*(povesjap 10 poysn{pe oq ABW UOTIIII0D BUTUTT]) j{paqoeLI09
+* Non-destructive recording—existing notes are not erased while recording.
+¢ Two FOOTSWITCH INPUTS may be assigned to remotely control many of the commonly used functions, including
 
-2q ][IM S1OLIe Sur [fe ATUO—patey]d nod Jey Jedy ]],NOA
+ERASE, REPEAT, PLAY/STOP, or LOCATE.
 
-‘] req 0] punose yoeq sdoo] sduanbas ay] Udy AA “YOu Yor
+¢ Iwo TRIGGER OUTPUTS may be programmed to output pulses at any selected note value.
 
-§,sa0uaNbas at} O] SUIT) UI preogday [IW] INO Avy usy3
-AV'1d pue (YOON ssoid Ayduus ‘aousnbes & p1o09es OF,
+© Will sync to standard LinnDrum or Linn 9000 sync tone.
 
-g0uaNbas & SUIP10I0y]
+© Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation.
+* TEMPO may be specified in BEATS-PER-MINUTE or FRAMES-PER-BEAT at 24, 25, or 30 frames per second,
 
-‘JONWOD s}JouNaI TeuONdGO e
+(even drop frame!)
 
-"UOTJEZIUOIYUAS OPOS UIT} FLAWS [euondo e
+¢ TEMPO may be entered numerically, adjustable in tenths of a Beat-Per-Minute increments, or by tapping quarter notes
 
-‘sou .sulddoys, noyyM sayelodo pue yoegdvyd ZuLINp S¥IOM NOLLOANNYOO ONIWILL e
+on the TAP TEMPO button.
 
-‘onqea ory AY
+¢ TEMPO CHANGES may be programmed into a sequence, with smooth transitions if desired.
+¢ Any TIME SIGNATURE may be used, and may be changed within a song.
 
-pojoojes-oid & ye sajou pyoy Aue syeadas ATTeONewWO Ne UOTOUNS [WAdAY OAISNOX e
-‘LSVJ SUnIpS soyeu UOTOUN ASV UA OUlN-[eal SAISNIOXY e
-‘Koy B JO YONO} 941 12 CASOdSNVALL 0g ABU Syde] [Te 10 9UC e
+linn
+Linn Electronics, Inc.
 
-i ASIP Jed
-
-S9}0U OOO‘OTT JOA SpfOy puv SpUOdeS UT SBUOS Xa[AUIOD So10}S DALIP YSIP , 74 € ISCJ-CNIN
-
-jSIOZISOUJUAS
-
-stuoydAjod of 0} dn skeyd A[snoourynuls ‘spouueYd [IW 9T JO duo 0} pousisse oq
-ABUL YORI] YOR ‘syous) oruoydAjod ‘snoouelnurs 7¢ SuTeJUOS ssouUaNbas QO] OY} JO YORA e
-
-‘SJONUOS ATWOOT pur ‘GNIMAY ‘GaVM OA
-LSVd ‘GYOOde AOLS ‘AV Td YIM Jopsocas ade} Yowsj-N[NU O} eps st UOTLISdO @
-LOPNOUT SaINjeoy s[quyIeUlss AUB S.JJ ‘OSN pue UIes] 0} o[duns A[suIzeUe JOA ‘PnJsomod APOUIOITXO
-St 1] “UeIOIsNUL feUOIssajoid oY} 10 JOO} soUBULIOJIJAd pue UOTIsOduIOS 11e-dY1-JO-9}e)s B SI IONUANbDaguUT] ay
-
-JOps1odady soUINbIS [GTI YVAL ZE
-Jgouanbaguury oy
+18720 Oxnard Street, Tarzana, CA 91356
+(818) 708-8131 TELEX #298949 LINN UR
 

tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/txt.bin

@@ -1,128 +1,123 @@
-2A NNI‘I 6F6867# XATALL IE18-80L (818)
+The LinnSequencer
+32 Track MIDI Sequence Recorder
 
-9SEI6 VO “BUBZIRY, “J0aNS PIPUXO OZLEI
-“Uy ‘soTUOMOI,q UUrT
+The LinnSequencer is a state-of-the-art composition and performance tool for the professional musician. It is
 
-uut]
+extremely powerful, yet amazingly simple to learn and use. It’s many remarkable features include:
 
-“‘SUOS B UIJIM pasueyo oq ABU pue ‘posn oq AWW AYN IVNOIS AWLL AUV
-“parlsop Jr SUOTIISUBI} YIOOUIS YIM “BoueNbas eB OJUI pourtueIZOId 9q ABU SFONWHO OdINAL e
+¢ Operation is similar to multi-track tape recorder with PLAY, STOP, RECORD, FAST
+FORWARD, REWIND, and LOCATE controls.
 
-‘uonng OdNAL dV L 9) uO
+e Each of the 100 sequences contains 32 simultaneous, polyphonic tracks. Each track may
+be assigned to one of 16 MIDI channels. Simultaneously plays up to 16 polyphonic
 
-sojou Jayienb Suiddy} Aq 10 ‘syUSTIOIOUI oINUTIAI-J8g-Jesg & JO sys} UL ofquisn(pe ‘ATTeouIAUINU paiajus oq ABU OdINALL e
+synthesizers!
 
-(jouer doup u3a9)
+¢ Ultra-fast 3%” disk drive stores complex songs in seconds and holds over 110,000 notes
 
-“puooes Jed souely O€ 10 “SZ “pz 18 [LVAG-MAd-SHN VU 10 ALOANIWAAd-SLVAd U! patyoeds aq kewl OAL «
-‘uoTe1odo [SVx JO} Aj[eusoyUT JoyndUIOd 11g 9] 98108 ZHI 8g ‘poeds-ysry Bann soz] e
+per disk!
 
-"9U0} DUAS 0006 UUL] Jo wNIqUUr] prepue}s 0} OUAS [ITAA ©
+¢ One or all tracks may be TRANSPOSED at the touch of a key.
+e Exclusive real-time ERASE function makes editing FAST.
+* Exclusive REPEAT function automatically repeats any held notes at a pre-selected
 
-“ONYBA 9}OU poloapes Aue Je sas—nd jndyno 07 pewureigold 3q ACW SL Ad LNO YADONAL OML
+rhythmic value.
 
-"ALVOOT 10 GOLS/AV 1d ‘LWddad “ASV
+¢ TIMING CORRECTION works during playback and operates without ‘chopping’ notes.
 
-SUIpNpoUr ‘suOTIOUN] posn A[UOUILUOS 94] JO AUBUT [O1]UOD AJ9]OWIAI 0} PousIsse oq ACUI ST AdNI HOLIMSLOO OME «
-“SUIPIONAI I[IYM P2sesd JOU Iv $3}OU BUTISIXO—ZUIPIOIA SATON.ASOP-UON
+¢ Optional SMPTE time code synchronization.
 
-‘suoneurldxa peuoyippe sdeydsip uowng g1TqH
+© Optional remote control.
 
-oy] ‘pepsau JI ‘suoneiodo [ye yYsnosy] NOA sapins ApIespo Avfdsip QO] Joey Z7¢ 9y3—uoeIodo Urea] 0} Ased ‘aus «
+Recording a Sequence
 
-jUorel]suowtap & IO} Aepol Jayeap uur’] INOA dag ‘dISHUL
-INOA 0} UONUS}]¥ PaplAIPUN INOA SUTJOASp ITY ps pue
-p1osai ‘asoduod no Jay 0} pausisap st 1s0uenbesuur’] oy)
-Aum Aposiooid $,Jeu], ‘SS9d0Id SATTBS1D OY} YIM SOIOJIOIUT
-yey) xo]dwWI0d Os dq JOA9U P[NoUsS osn NOA AZopOuYdE} oy
+To record a sequence, simply press RECORD and PLAY,
+then play your MIDI keyboard in time to the Sequencer’s
+click track. When the sequence loops back around to bar 1,
+you’ ll hear what you played—only all timing errors will be
 
-ISTUMOIAUIO?) NOAA UOHISOdWIO)
+corrected! (Timing correction may be adjusted or defeated).
 
-"NOSpr] B Oy ‘AONUTJUT yada 0} seq Maz Se] BY] Jas UdAd
-uvd NOA ‘palisap JJ ‘souanbes Mou ¥B OVUT sjied ou] [Te Adoo
-ATesrewO Ne WI) [IM ONOS ALVAAO JeyIe80} wey}
-,deyd,, 0} UOTOUNJ ONOS ALVA ou] asn usy] ‘saouanbes
-JENPIAIpUt UI (“949 ‘snJOYD ‘aS1OA) UOTIDIS JIseq Yes
-Pl0da1 OF ST ABM JOuIOUY “(812g 666 01 dn) ysnory) ABM
+Any additional notes played will be added into the track
+— existing notes are not erased while recording!
 
-dU} [fe YORI] YORs p10991 0} ST SUOS B 9789I9 0} ABM SUG,
+FAST FORWARD, REWIND, and LOCATE controls
+may be used at any time to quickly access any location in
+your sequence for spot-recording. To overdub a new part,
+select a different track and start recording—while you
+record, the first track will play in perfect sync (unless you
+MUTE it, or SOLO another track). In this way, up to 32
+tracks may be overdubbed! All MIDI effects are recorded
+including pitch bend, modulation, velocity, aftertouch,
+sustain pedal, and program changes!
 
-SUOS & SUTVAID
+Editing
 
-*suoT}oes poJUBMUN
+To erase a wrong note, simply hold ERASE and press
+the note to be erased just before it plays in the sequence—
+when played back, it will be gone. Notes may also be
 
-SAOUIOI 0} ABM SWS dU} SoyeIodo SUV ALATAaG
+added, erased, or changed using the SINGLE STEP func-
+tion. To overdub notes at specific points within a sequence,
 
-“OBPLIq dy} PUB SNIOY PUOdAS dT]] Ud9MIAQ SIDA ISI
+Additional Features
 
-ay) Jo Adoo B JJasuT WYSE NOAA ‘afdwexs 10.f ‘UO JUSIN]JIP
+simply use LOCATE, FAST FORWARD, or REWIND to
+find the desired bar number, then start recording.
 
-B IO aouaNbas sues OY} UI—JOY OUP 0} UOTIEIO] 9UO WOT]
-$1Bq JAOUI OF NOA sMOTIe WOTIOUNS AdOO/IMASNI OULL
+The INSERT/COPY function allows you to move bars
+from one location to another—in the same sequence or a
+different one. For example, you might insert a copy of the
+first verse between the second chorus and the bridge.
+DELETE BARS operates the same way to remove
+unwanted sections,
 
-‘SUIPIONAI JIVIS Udy) “OQuINU eq porisop ay} puy
+Creating a Song
 
-0} CNIMAY 10 ‘CYVM Od LSWA “AEVOOT esn Apduns
+One way to create a song is to record each track all the
+way through (up to 999 bars). Another way is to record
+each basic section (verse, chorus, etc.) in individual
+sequences, then use the CREATE SONG function to “chain”
+them together. CREATE SONG will then automatically
+copy all the parts into a new sequence. If desired, you can
+even set the last few bars to repeat infinitely, for a fadeout.
 
-sainjeay [PUOHIPPY
+Composition Without Compromise
 
-‘gouanbas & UTYIIM s]UTOd a1y1dads 3¥ $9100 QnPIOAO OL "UOT}
--ouns dALLS ATONIS 24) Suisn pasueyo Jo ‘pasesa ‘pappe
-aq osye ABUT S9]ON ‘U0 9q ]IIM 1 “yoeq podeyd uayM
-—aouanbas oy] ul skeyd 71 a10J9q Isnf posers oq 0} d]0U ayy
-ssaid pue ASvwug ploy Aydunis ‘jou Suomm & aseso OL
+The technology you use should never be so complex that
+it interferes with the creative process. That’s precisely why
+the LinnSequencer is designed to let you compose, record
+and edit while devoting your undivided attention to your
+music. See your Linn dealer today for a demonstration!
 
-sunipa
+* Simple, easy to learn operation—the 32 character LCD display clearly guides you through all operations. If needed, the
 
-jsesdueyo ureisoid pue ‘fepod ureysns
-‘yonoplalje ‘AWOOTOA ‘UOTyeTNpow ‘pusg youd Surpnyour
-pep10del are $199JJ2 TCTIN [WV iPeqqnpseao aq Aeur syoen
-Ze 07 dn ‘Kem sie Uy *(foeI} JOyOUR OJOS 10 ALLAN
-NOA ssofum) duAS yOaysod ul Avy [[IM Yow] ISI 93 “prooar
-NOA 3[IYM—SUIPIOIA LIBIS PU YORI) TUdIOTJIP B JOaTas
-*y1ed MOU B QNPIsA0 OL, “SuIps0daJ-jods 10} aouanbes mno0k
-UI UOHBIO] Aue ssad0e ATYOIND 0} owt} Aue ye pasn aq AvUE
-SJONUOD FLIVOOT pur ‘ANIMA ‘CYVMaYOd LSVd
-{SUIPIOSAI {IY posesa JOU se So]OU SuTsTXO—
-yous} 3U} OUT poppe aq JIM poteyd sajou yeuonippe Auy
+HELP button displays additional explanations.
 
-*(povesjap 10 poysn{pe oq ABW UOTIIII0D BUTUTT]) j{paqoeLI09
+* Non-destructive recording—existing notes are not erased while recording.
+¢ Two FOOTSWITCH INPUTS may be assigned to remotely control many of the commonly used functions, including
 
-2q ][IM S1OLIe Sur [fe ATUO—patey]d nod Jey Jedy ]],NOA
+ERASE, REPEAT, PLAY/STOP, or LOCATE.
 
-‘] req 0] punose yoeq sdoo] sduanbas ay] Udy AA “YOu Yor
+¢ Iwo TRIGGER OUTPUTS may be programmed to output pulses at any selected note value.
 
-§,sa0uaNbas at} O] SUIT) UI preogday [IW] INO Avy usy3
-AV'1d pue (YOON ssoid Ayduus ‘aousnbes & p1o09es OF,
+© Will sync to standard LinnDrum or Linn 9000 sync tone.
 
-g0uaNbas & SUIP10I0y]
+© Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation.
+* TEMPO may be specified in BEATS-PER-MINUTE or FRAMES-PER-BEAT at 24, 25, or 30 frames per second,
 
-‘JONWOD s}JouNaI TeuONdGO e
+(even drop frame!)
 
-"UOTJEZIUOIYUAS OPOS UIT} FLAWS [euondo e
+¢ TEMPO may be entered numerically, adjustable in tenths of a Beat-Per-Minute increments, or by tapping quarter notes
 
-‘sou .sulddoys, noyyM sayelodo pue yoegdvyd ZuLINp S¥IOM NOLLOANNYOO ONIWILL e
+on the TAP TEMPO button.
 
-‘onqea ory AY
+¢ TEMPO CHANGES may be programmed into a sequence, with smooth transitions if desired.
+¢ Any TIME SIGNATURE may be used, and may be changed within a song.
 
-pojoojes-oid & ye sajou pyoy Aue syeadas ATTeONewWO Ne UOTOUNS [WAdAY OAISNOX e
-‘LSVJ SUnIpS soyeu UOTOUN ASV UA OUlN-[eal SAISNIOXY e
-‘Koy B JO YONO} 941 12 CASOdSNVALL 0g ABU Syde] [Te 10 9UC e
+linn
+Linn Electronics, Inc.
 
-i ASIP Jed
-
-S9}0U OOO‘OTT JOA SpfOy puv SpUOdeS UT SBUOS Xa[AUIOD So10}S DALIP YSIP , 74 € ISCJ-CNIN
-
-jSIOZISOUJUAS
-
-stuoydAjod of 0} dn skeyd A[snoourynuls ‘spouueYd [IW 9T JO duo 0} pousisse oq
-ABUL YORI] YOR ‘syous) oruoydAjod ‘snoouelnurs 7¢ SuTeJUOS ssouUaNbas QO] OY} JO YORA e
-
-‘SJONUOS ATWOOT pur ‘GNIMAY ‘GaVM OA
-LSVd ‘GYOOde AOLS ‘AV Td YIM Jopsocas ade} Yowsj-N[NU O} eps st UOTLISdO @
-LOPNOUT SaINjeoy s[quyIeUlss AUB S.JJ ‘OSN pue UIes] 0} o[duns A[suIzeUe JOA ‘PnJsomod APOUIOITXO
-St 1] “UeIOIsNUL feUOIssajoid oY} 10 JOO} soUBULIOJIJAd pue UOTIsOduIOS 11e-dY1-JO-9}e)s B SI IONUANbDaguUT] ay
-
-JOps1odady soUINbIS [GTI YVAL ZE
-Jgouanbaguury oy
+18720 Oxnard Street, Tarzana, CA 91356
+(818) 708-8131 TELEX #298949 LINN UR
 

tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt/txt.bin

@@ -1,124 +1,123 @@
-2A NNI‘I 6F6867# XATALL IE18-80L (818)
+The LinnSequencer
+32 Track MIDI Sequence Recorder
 
-9SEI6 VO “BUBZIRY, “J0aNS PIPUXO OZLEI
-“Uy ‘soTUOMOI,q UUrT
+The LinnSequencer is a state-of-the-art composition and performance tool for the professional musician. It is
 
-uu
+extremely powerful, yet amazingly simple to learn and use. It’s many remarkable features include:
 
-“‘SUOS B UIJIM pasueyo oq ABU pue ‘posn oq AWW AYN IVNOIS AWLL AUV
-“parlsop Jr SUOTIISUBI} YIOOUIS YIM “BoueNbas eB OJUI pourtueIZOId 9q ABU SFONWHO OdINAL e
+¢ Operation is similar to multi-track tape recorder with PLAY, STOP, RECORD, FAST
+FORWARD, REWIND, and LOCATE controls.
 
-‘uonng OdNAL dV L 9) uO
+e Each of the 100 sequences contains 32 simultaneous, polyphonic tracks. Each track may
+be assigned to one of 16 MIDI channels. Simultaneously plays up to 16 polyphonic
 
-sojou Jayienb Suiddy} Aq 10 ‘syUSTIOIOUI oINUTIAI-J8g-Jesg & JO sys} UL ofquisn(pe ‘ATTeouIAUINU paiajus oq ABU OdINALL e
+synthesizers!
 
-(jouer doup u3a9)
+¢ Ultra-fast 3%” disk drive stores complex songs in seconds and holds over 110,000 notes
 
-“puooes Jed souely O€ 10 “SZ “pz 18 [LVAG-MAd-SHN VU 10 ALOANIWAAd-SLVAd U! patyoeds aq kewl OAL «
-‘uoTe1odo [SVx JO} Aj[eusoyUT JoyndUIOd 11g 9] 98108 ZHI 8g ‘poeds-ysry Bann soz] e
+per disk!
 
-"9U0} DUAS 0006 UUL] Jo wNIqUUr] prepue}s 0} OUAS [ITAA ©
+¢ One or all tracks may be TRANSPOSED at the touch of a key.
+e Exclusive real-time ERASE function makes editing FAST.
+* Exclusive REPEAT function automatically repeats any held notes at a pre-selected
 
-“ONYBA 9}OU poloapes Aue Je sas—nd jndyno 07 pewureigold 3q ACW SL Ad LNO YADONAL OML
+rhythmic value.
 
-"ALVOOT 10 GOLS/AV 1d ‘LWddad “ASV
+¢ TIMING CORRECTION works during playback and operates without ‘chopping’ notes.
 
-SUIpNpoUr ‘suOTIOUN] posn A[UOUILUOS 94] JO AUBUT [O1]UOD AJ9]OWIAI 0} PousIsse oq ACUI ST AdNI HOLIMSLOO OME «
-“SUIPIONAI I[IYM P2sesd JOU Iv $3}OU BUTISIXO—ZUIPIOIA SATON.ASOP-UON
+¢ Optional SMPTE time code synchronization.
 
-‘suoneurldxa peuoyippe sdeydsip uowng g1TqH
+© Optional remote control.
 
-oy] ‘pepsau JI ‘suoneiodo [ye yYsnosy] NOA sapins ApIespo Avfdsip QO] Joey Z7¢ 9y3—uoeIodo Urea] 0} Ased ‘aus «
+Recording a Sequence
 
-jUorel]suowtap & IO} Aepol Jayeap uur’] INOA dag ‘dISHUL
-INOA 0} UONUS}]¥ PaplAIPUN INOA SUTJOASp ITY ps pue
-p1osai ‘asoduod no Jay 0} pausisap st 1s0uenbesuur’] oy)
-Aum Aposiooid $,Jeu], ‘SS9d0Id SATTBS1D OY} YIM SOIOJIOIUT
-yey) xo]dwWI0d Os dq JOA9U P[NoUsS osn NOA AZopOuYdE} oy
+To record a sequence, simply press RECORD and PLAY,
+then play your MIDI keyboard in time to the Sequencer’s
+click track. When the sequence loops back around to bar 1,
+you’ ll hear what you played—only all timing errors will be
 
-ISTUMOIAUIO?) NOAA UOHISOdWIO)
+corrected! (Timing correction may be adjusted or defeated).
 
-"NOSpr] B Oy ‘AONUTJUT yada 0} seq Maz Se] BY] Jas UdAd
-uvd NOA ‘palisap JJ ‘souanbes Mou ¥B OVUT sjied ou] [Te Adoo
-ATesrewO Ne WI) [IM ONOS ALVAAO JeyIe80} wey}
-,deyd,, 0} UOTOUNJ ONOS ALVA ou] asn usy] ‘saouanbes
-JENPIAIpUt UI (“949 ‘snJOYD ‘aS1OA) UOTIDIS JIseq Yes
-Pl0da1 OF ST ABM JOuIOUY “(812g 666 01 dn) ysnory) ABM
+Any additional notes played will be added into the track
+— existing notes are not erased while recording!
 
-dU} [fe YORI] YORs p10991 0} ST SUOS B 9789I9 0} ABM SUG,
+FAST FORWARD, REWIND, and LOCATE controls
+may be used at any time to quickly access any location in
+your sequence for spot-recording. To overdub a new part,
+select a different track and start recording—while you
+record, the first track will play in perfect sync (unless you
+MUTE it, or SOLO another track). In this way, up to 32
+tracks may be overdubbed! All MIDI effects are recorded
+including pitch bend, modulation, velocity, aftertouch,
+sustain pedal, and program changes!
 
-SUOS & SUTVAID
+Editing
 
-*suoT}oes poJUBMUN
+To erase a wrong note, simply hold ERASE and press
+the note to be erased just before it plays in the sequence—
+when played back, it will be gone. Notes may also be
 
-SAOUIOI 0} ABM SWS dU} SoyeIodo SUV ALATAaG
+added, erased, or changed using the SINGLE STEP func-
+tion. To overdub notes at specific points within a sequence,
 
-“OBPLIq dy} PUB SNIOY PUOdAS dT]] Ud9MIAQ SIDA ISI
+Additional Features
 
-ay) Jo Adoo B JJasuT WYSE NOAA ‘afdwexs 10.f ‘UO JUSIN]JIP
+simply use LOCATE, FAST FORWARD, or REWIND to
+find the desired bar number, then start recording.
 
-B IO aouaNbas sues OY} UI—JOY OUP 0} UOTIEIO] 9UO WOT]
-$1Bq JAOUI OF NOA sMOTIe WOTIOUNS AdOO/IMASNI OULL
+The INSERT/COPY function allows you to move bars
+from one location to another—in the same sequence or a
+different one. For example, you might insert a copy of the
+first verse between the second chorus and the bridge.
+DELETE BARS operates the same way to remove
+unwanted sections,
 
-‘SUIPIONAI JIVIS Udy) “OQuINU eq porisop ay} puy
+Creating a Song
 
-0} CNIMAY 10 ‘CYVM Od LSWA “AEVOOT esn Apduns
+One way to create a song is to record each track all the
+way through (up to 999 bars). Another way is to record
+each basic section (verse, chorus, etc.) in individual
+sequences, then use the CREATE SONG function to “chain”
+them together. CREATE SONG will then automatically
+copy all the parts into a new sequence. If desired, you can
+even set the last few bars to repeat infinitely, for a fadeout.
 
-sainjeay [PUOHIPPY
+Composition Without Compromise
 
-‘gouanbas & UTYIIM s]UTOd a1y1dads 3¥ $9100 QnPIOAO OL "UOT}
--ouns dALLS ATONIS 24) Suisn pasueyo Jo ‘pasesa ‘pappe
-aq osye ABUT S9]ON ‘U0 9q ]IIM 1 “yoeq podeyd uayM
-—aouanbas oy] ul skeyd 71 a10J9q Isnf posers oq 0} d]0U ayy
-ssaid pue ASvwug ploy Aydunis ‘jou Suomm & aseso OL
+The technology you use should never be so complex that
+it interferes with the creative process. That’s precisely why
+the LinnSequencer is designed to let you compose, record
+and edit while devoting your undivided attention to your
+music. See your Linn dealer today for a demonstration!
 
-sunipa
+* Simple, easy to learn operation—the 32 character LCD display clearly guides you through all operations. If needed, the
 
-jsesdueyo ureisoid pue ‘fepod ureysns
-‘yonoplalje ‘AWOOTOA ‘UOTyeTNpow ‘pusg youd Surpnyour
-pep10del are $199JJ2 TCTIN [WV iPeqqnpseao aq Aeur syoen
-Ze 07 dn ‘Kem sie Uy *(foeI} JOyOUR OJOS 10 ALLAN
-NOA ssofum) duAS yOaysod ul Avy [[IM Yow] ISI 93 “prooar
-NOA 3[IYM—SUIPIOIA LIBIS PU YORI) TUdIOTJIP B JOaTas
-*y1ed MOU B QNPIsA0 OL, “SuIps0daJ-jods 10} aouanbes mno0k
-UI UOHBIO] Aue ssad0e ATYOIND 0} owt} Aue ye pasn aq AvUE
-SJONUOD FLIVOOT pur ‘ANIMA ‘CYVMaYOd LSVd
-{SUIPIOSAI {IY posesa JOU se So]OU SuTsTXO—
-yous} 3U} OUT poppe aq JIM poteyd sajou yeuonippe Auy
-*(povesjap 10 poysn{pe oq ABW UOTIIII0D BUTUTT]) j{paqoeLI09
-2q ][IM S1OLIe Sur [fe ATUO—patey]d nod Jey Jedy ]],NOA
-‘] req 0] punose yoeq sdoo] sduanbas ay] Udy AA “YOu Yor
-§,sa0uaNbas at} O] SUIT) UI preogday [IW] INO Avy usy3
-AV'1d pue (YOON ssoid Ayduus ‘aousnbes & p1o09es OF,
+HELP button displays additional explanations.
 
-g0uaNbas & SUIP10I0y]
+* Non-destructive recording—existing notes are not erased while recording.
+¢ Two FOOTSWITCH INPUTS may be assigned to remotely control many of the commonly used functions, including
 
-‘JONWOD s}JouNaI TeuONdGO e
+ERASE, REPEAT, PLAY/STOP, or LOCATE.
 
-"UOTJEZIUOIYUAS OPOS UIT} FLAWS [euondo e
+¢ Iwo TRIGGER OUTPUTS may be programmed to output pulses at any selected note value.
 
-‘sou .sulddoys, noyyM sayelodo pue yoegdvyd ZuLINp S¥IOM NOLLOANNYOO ONIWILL e
+© Will sync to standard LinnDrum or Linn 9000 sync tone.
 
-‘onqea ory AY
+© Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation.
+* TEMPO may be specified in BEATS-PER-MINUTE or FRAMES-PER-BEAT at 24, 25, or 30 frames per second,
 
-pojoojes-oid & ye sajou pyoy Aue syeadas ATTeONewWO Ne UOTOUNS [WAdAY OAISNOX e
-‘LSVJ SUnIpS soyeu UOTOUN ASV UA OUlN-[eal SAISNIOXY e
-‘Koy B JO YONO} 941 12 CASOdSNVALL 0g ABU Syde] [Te 10 9UC e
+(even drop frame!)
 
-i ASIP Jed
+¢ TEMPO may be entered numerically, adjustable in tenths of a Beat-Per-Minute increments, or by tapping quarter notes
 
-S9}0U OOO‘OTT JOA SpfOy puv SpUOdeS UT SBUOS Xa[AUIOD So10}S DALIP YSIP , 74 € ISCJ-CNIN
+on the TAP TEMPO button.
 
-jSIOZISOUJUAS
+¢ TEMPO CHANGES may be programmed into a sequence, with smooth transitions if desired.
+¢ Any TIME SIGNATURE may be used, and may be changed within a song.
 
-stuoydAjod of 0} dn skeyd A[snoourynuls ‘spouueYd [IW 9T JO duo 0} pousisse oq
-ABUL YORI] YOR ‘syous) oruoydAjod ‘snoouelnurs 7¢ SuTeJUOS ssouUaNbas QO] OY} JO YORA e
+linn
+Linn Electronics, Inc.
 
-‘SJONUOS ATWOOT pur ‘GNIMAY ‘GaVM OA
-LSVd ‘GYOOde AOLS ‘AV Td YIM Jopsocas ade} Yowsj-N[NU O} eps st UOTLISdO @
-LOPNOUT SaINjeoy s[quyIeUlss AUB S.JJ ‘OSN pue UIes] 0} o[duns A[suIzeUe JOA ‘PnJsomod APOUIOITXO
-St 1] “UeIOIsNUL feUOIssajoid oY} 10 JOO} soUBULIOJIJAd pue UOTIsOduIOS 11e-dY1-JO-9}e)s B SI IONUANbDaguUT] ay
-
-JOps1odady soUINbIS [GTI YVAL ZE
-Jgouanbaguury oy
+18720 Oxnard Street, Tarzana, CA 91356
+(818) 708-8131 TELEX #298949 LINN UR
 

tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/txt.bin

@@ -1,124 +1,123 @@
-2A NNI‘I 6F6867# XATALL IE18-80L (818)
+The LinnSequencer
+32 Track MIDI Sequence Recorder
 
-9SEI6 VO “BUBZIRY, “J0aNS PIPUXO OZLEI
-“Uy ‘soTUOMOI,q UUrT
+The LinnSequencer is a state-of-the-art composition and performance tool for the professional musician. It is
 
-uu
+extremely powerful, yet amazingly simple to learn and use. It’s many remarkable features include:
 
-“‘SUOS B UIJIM pasueyo oq ABU pue ‘posn oq AWW AYN IVNOIS AWLL AUV
-“parlsop Jr SUOTIISUBI} YIOOUIS YIM “BoueNbas eB OJUI pourtueIZOId 9q ABU SFONWHO OdINAL e
+¢ Operation is similar to multi-track tape recorder with PLAY, STOP, RECORD, FAST
+FORWARD, REWIND, and LOCATE controls.
 
-‘uonng OdNAL dV L 9) uO
+e Each of the 100 sequences contains 32 simultaneous, polyphonic tracks. Each track may
+be assigned to one of 16 MIDI channels. Simultaneously plays up to 16 polyphonic
 
-sojou Jayienb Suiddy} Aq 10 ‘syUSTIOIOUI oINUTIAI-J8g-Jesg & JO sys} UL ofquisn(pe ‘ATTeouIAUINU paiajus oq ABU OdINALL e
+synthesizers!
 
-(jouer doup u3a9)
+¢ Ultra-fast 3%” disk drive stores complex songs in seconds and holds over 110,000 notes
 
-“puooes Jed souely O€ 10 “SZ “pz 18 [LVAG-MAd-SHN VU 10 ALOANIWAAd-SLVAd U! patyoeds aq kewl OAL «
-‘uoTe1odo [SVx JO} Aj[eusoyUT JoyndUIOd 11g 9] 98108 ZHI 8g ‘poeds-ysry Bann soz] e
+per disk!
 
-"9U0} DUAS 0006 UUL] Jo wNIqUUr] prepue}s 0} OUAS [ITAA ©
+¢ One or all tracks may be TRANSPOSED at the touch of a key.
+e Exclusive real-time ERASE function makes editing FAST.
+* Exclusive REPEAT function automatically repeats any held notes at a pre-selected
 
-“ONYBA 9}OU poloapes Aue Je sas—nd jndyno 07 pewureigold 3q ACW SL Ad LNO YADONAL OML
+rhythmic value.
 
-"ALVOOT 10 GOLS/AV 1d ‘LWddad “ASV
+¢ TIMING CORRECTION works during playback and operates without ‘chopping’ notes.
 
-SUIpNpoUr ‘suOTIOUN] posn A[UOUILUOS 94] JO AUBUT [O1]UOD AJ9]OWIAI 0} PousIsse oq ACUI ST AdNI HOLIMSLOO OME «
-“SUIPIONAI I[IYM P2sesd JOU Iv $3}OU BUTISIXO—ZUIPIOIA SATON.ASOP-UON
+¢ Optional SMPTE time code synchronization.
 
-‘suoneurldxa peuoyippe sdeydsip uowng g1TqH
+© Optional remote control.
 
-oy] ‘pepsau JI ‘suoneiodo [ye yYsnosy] NOA sapins ApIespo Avfdsip QO] Joey Z7¢ 9y3—uoeIodo Urea] 0} Ased ‘aus «
+Recording a Sequence
 
-jUorel]suowtap & IO} Aepol Jayeap uur’] INOA dag ‘dISHUL
-INOA 0} UONUS}]¥ PaplAIPUN INOA SUTJOASp ITY ps pue
-p1osai ‘asoduod no Jay 0} pausisap st 1s0uenbesuur’] oy)
-Aum Aposiooid $,Jeu], ‘SS9d0Id SATTBS1D OY} YIM SOIOJIOIUT
-yey) xo]dwWI0d Os dq JOA9U P[NoUsS osn NOA AZopOuYdE} oy
+To record a sequence, simply press RECORD and PLAY,
+then play your MIDI keyboard in time to the Sequencer’s
+click track. When the sequence loops back around to bar 1,
+you’ ll hear what you played—only all timing errors will be
 
-ISTUMOIAUIO?) NOAA UOHISOdWIO)
+corrected! (Timing correction may be adjusted or defeated).
 
-"NOSpr] B Oy ‘AONUTJUT yada 0} seq Maz Se] BY] Jas UdAd
-uvd NOA ‘palisap JJ ‘souanbes Mou ¥B OVUT sjied ou] [Te Adoo
-ATesrewO Ne WI) [IM ONOS ALVAAO JeyIe80} wey}
-,deyd,, 0} UOTOUNJ ONOS ALVA ou] asn usy] ‘saouanbes
-JENPIAIpUt UI (“949 ‘snJOYD ‘aS1OA) UOTIDIS JIseq Yes
-Pl0da1 OF ST ABM JOuIOUY “(812g 666 01 dn) ysnory) ABM
+Any additional notes played will be added into the track
+— existing notes are not erased while recording!
 
-dU} [fe YORI] YORs p10991 0} ST SUOS B 9789I9 0} ABM SUG,
+FAST FORWARD, REWIND, and LOCATE controls
+may be used at any time to quickly access any location in
+your sequence for spot-recording. To overdub a new part,
+select a different track and start recording—while you
+record, the first track will play in perfect sync (unless you
+MUTE it, or SOLO another track). In this way, up to 32
+tracks may be overdubbed! All MIDI effects are recorded
+including pitch bend, modulation, velocity, aftertouch,
+sustain pedal, and program changes!
 
-SUOS & SUTVAID
+Editing
 
-*suoT}oes poJUBMUN
+To erase a wrong note, simply hold ERASE and press
+the note to be erased just before it plays in the sequence—
+when played back, it will be gone. Notes may also be
 
-SAOUIOI 0} ABM SWS dU} SoyeIodo SUV ALATAaG
+added, erased, or changed using the SINGLE STEP func-
+tion. To overdub notes at specific points within a sequence,
 
-“OBPLIq dy} PUB SNIOY PUOdAS dT]] Ud9MIAQ SIDA ISI
+Additional Features
 
-ay) Jo Adoo B JJasuT WYSE NOAA ‘afdwexs 10.f ‘UO JUSIN]JIP
+simply use LOCATE, FAST FORWARD, or REWIND to
+find the desired bar number, then start recording.
 
-B IO aouaNbas sues OY} UI—JOY OUP 0} UOTIEIO] 9UO WOT]
-$1Bq JAOUI OF NOA sMOTIe WOTIOUNS AdOO/IMASNI OULL
+The INSERT/COPY function allows you to move bars
+from one location to another—in the same sequence or a
+different one. For example, you might insert a copy of the
+first verse between the second chorus and the bridge.
+DELETE BARS operates the same way to remove
+unwanted sections,
 
-‘SUIPIONAI JIVIS Udy) “OQuINU eq porisop ay} puy
+Creating a Song
 
-0} CNIMAY 10 ‘CYVM Od LSWA “AEVOOT esn Apduns
+One way to create a song is to record each track all the
+way through (up to 999 bars). Another way is to record
+each basic section (verse, chorus, etc.) in individual
+sequences, then use the CREATE SONG function to “chain”
+them together. CREATE SONG will then automatically
+copy all the parts into a new sequence. If desired, you can
+even set the last few bars to repeat infinitely, for a fadeout.
 
-sainjeay [PUOHIPPY
+Composition Without Compromise
 
-‘gouanbas & UTYIIM s]UTOd a1y1dads 3¥ $9100 QnPIOAO OL "UOT}
--ouns dALLS ATONIS 24) Suisn pasueyo Jo ‘pasesa ‘pappe
-aq osye ABUT S9]ON ‘U0 9q ]IIM 1 “yoeq podeyd uayM
-—aouanbas oy] ul skeyd 71 a10J9q Isnf posers oq 0} d]0U ayy
-ssaid pue ASvwug ploy Aydunis ‘jou Suomm & aseso OL
+The technology you use should never be so complex that
+it interferes with the creative process. That’s precisely why
+the LinnSequencer is designed to let you compose, record
+and edit while devoting your undivided attention to your
+music. See your Linn dealer today for a demonstration!
 
-sunipa
+* Simple, easy to learn operation—the 32 character LCD display clearly guides you through all operations. If needed, the
 
-jsesdueyo ureisoid pue ‘fepod ureysns
-‘yonoplalje ‘AWOOTOA ‘UOTyeTNpow ‘pusg youd Surpnyour
-pep10del are $199JJ2 TCTIN [WV iPeqqnpseao aq Aeur syoen
-Ze 07 dn ‘Kem sie Uy *(foeI} JOyOUR OJOS 10 ALLAN
-NOA ssofum) duAS yOaysod ul Avy [[IM Yow] ISI 93 “prooar
-NOA 3[IYM—SUIPIOIA LIBIS PU YORI) TUdIOTJIP B JOaTas
-*y1ed MOU B QNPIsA0 OL, “SuIps0daJ-jods 10} aouanbes mno0k
-UI UOHBIO] Aue ssad0e ATYOIND 0} owt} Aue ye pasn aq AvUE
-SJONUOD FLIVOOT pur ‘ANIMA ‘CYVMaYOd LSVd
-{SUIPIOSAI {IY posesa JOU se So]OU SuTsTXO—
-yous} 3U} OUT poppe aq JIM poteyd sajou yeuonippe Auy
-*(povesjap 10 poysn{pe oq ABW UOTIIII0D BUTUTT]) j{paqoeLI09
-2q ][IM S1OLIe Sur [fe ATUO—patey]d nod Jey Jedy ]],NOA
-‘] req 0] punose yoeq sdoo] sduanbas ay] Udy AA “YOu Yor
-§,sa0uaNbas at} O] SUIT) UI preogday [IW] INO Avy usy3
-AV'1d pue (YOON ssoid Ayduus ‘aousnbes & p1o09es OF,
+HELP button displays additional explanations.
 
-g0uaNbas & SUIP10I0y]
+* Non-destructive recording—existing notes are not erased while recording.
+¢ Two FOOTSWITCH INPUTS may be assigned to remotely control many of the commonly used functions, including
 
-‘JONWOD s}JouNaI TeuONdGO e
+ERASE, REPEAT, PLAY/STOP, or LOCATE.
 
-"UOTJEZIUOIYUAS OPOS UIT} FLAWS [euondo e
+¢ Iwo TRIGGER OUTPUTS may be programmed to output pulses at any selected note value.
 
-‘sou .sulddoys, noyyM sayelodo pue yoegdvyd ZuLINp S¥IOM NOLLOANNYOO ONIWILL e
+© Will sync to standard LinnDrum or Linn 9000 sync tone.
 
-‘onqea ory AY
+© Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation.
+* TEMPO may be specified in BEATS-PER-MINUTE or FRAMES-PER-BEAT at 24, 25, or 30 frames per second,
 
-pojoojes-oid & ye sajou pyoy Aue syeadas ATTeONewWO Ne UOTOUNS [WAdAY OAISNOX e
-‘LSVJ SUnIpS soyeu UOTOUN ASV UA OUlN-[eal SAISNIOXY e
-‘Koy B JO YONO} 941 12 CASOdSNVALL 0g ABU Syde] [Te 10 9UC e
+(even drop frame!)
 
-i ASIP Jed
+¢ TEMPO may be entered numerically, adjustable in tenths of a Beat-Per-Minute increments, or by tapping quarter notes
 
-S9}0U OOO‘OTT JOA SpfOy puv SpUOdeS UT SBUOS Xa[AUIOD So10}S DALIP YSIP , 74 € ISCJ-CNIN
+on the TAP TEMPO button.
 
-jSIOZISOUJUAS
+¢ TEMPO CHANGES may be programmed into a sequence, with smooth transitions if desired.
+¢ Any TIME SIGNATURE may be used, and may be changed within a song.
 
-stuoydAjod of 0} dn skeyd A[snoourynuls ‘spouueYd [IW 9T JO duo 0} pousisse oq
-ABUL YORI] YOR ‘syous) oruoydAjod ‘snoouelnurs 7¢ SuTeJUOS ssouUaNbas QO] OY} JO YORA e
+linn
+Linn Electronics, Inc.
 
-‘SJONUOS ATWOOT pur ‘GNIMAY ‘GaVM OA
-LSVd ‘GYOOde AOLS ‘AV Td YIM Jopsocas ade} Yowsj-N[NU O} eps st UOTLISdO @
-LOPNOUT SaINjeoy s[quyIeUlss AUB S.JJ ‘OSN pue UIes] 0} o[duns A[suIzeUe JOA ‘PnJsomod APOUIOITXO
-St 1] “UeIOIsNUL feUOIssajoid oY} 10 JOO} soUBULIOJIJAd pue UOTIsOduIOS 11e-dY1-JO-9}e)s B SI IONUANbDaguUT] ay
-
-JOps1odady soUINbIS [GTI YVAL ZE
-Jgouanbaguury oy
+18720 Oxnard Street, Tarzana, CA 91356
+(818) 708-8131 TELEX #298949 LINN UR
 

tests/cache/ccitt/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin

@@ -9,7 +9,7 @@
   <meta name='ocr-capabilities' content='ocr_page ocr_carea ocr_par ocr_line ocrx_word ocrp_wconf'/>
  </head>
  <body>
-  <div class='ocr_page' id='page_1' title='image "/var/folders/2s/7t022mgj0h5cprbq0dtb1ksm0000gn/T/com.github.ocrmypdf.ucmqv4tt/000001_ocr.png"; bbox 0 0 2550 3300; ppageno 0'>
+  <div class='ocr_page' id='page_1' title='image "/tmp/ocrmypdf.io.suvrek94/000001_ocr.png"; bbox 0 0 2550 3300; ppageno 0'>
    <div class='ocr_carea' id='block_1_1' title="bbox 582 131 1968 303">
     <p class='ocr_par' id='par_1_1' lang='eng' title="bbox 582 131 1968 303">
      <span class='ocr_header' id='line_1_1' title="bbox 882 131 1657 217; baseline 0.001 -17; x_size 85; x_descenders 16; x_ascenders 19">
@@ -29,7 +29,7 @@
     <p class='ocr_par' id='par_1_2' lang='eng' title="bbox 347 380 2188 423">
      <span class='ocr_header' id='line_1_3' title="bbox 347 380 2188 423; baseline -0.001 -12; x_size 38; x_descenders 8; x_ascenders 10">
       <span class='ocrx_word' id='word_1_8' title='bbox 347 380 412 410; x_wconf 93'>The</span>
-      <span class='ocrx_word' id='word_1_9' title='bbox 424 380 676 417; x_wconf 92'>LinnSequencer</span>
+      <span class='ocrx_word' id='word_1_9' title='bbox 424 380 676 417; x_wconf 91'>LinnSequencer</span>
       <span class='ocrx_word' id='word_1_10' title='bbox 688 380 712 411; x_wconf 96'>is</span>
       <span class='ocrx_word' id='word_1_11' title='bbox 724 390 743 411; x_wconf 96'>a</span>
       <span class='ocrx_word' id='word_1_12' title='bbox 754 381 1005 423; x_wconf 96'>state-of-the-art</span>
@@ -39,7 +39,7 @@
       <span class='ocrx_word' id='word_1_16' title='bbox 1536 380 1602 411; x_wconf 96'>tool</span>
       <span class='ocrx_word' id='word_1_17' title='bbox 1615 380 1663 411; x_wconf 97'>for</span>
       <span class='ocrx_word' id='word_1_18' title='bbox 1674 381 1725 410; x_wconf 96'>the</span>
-      <span class='ocrx_word' id='word_1_19' title='bbox 1737 380 1940 417; x_wconf 95'>professional</span>
+      <span class='ocrx_word' id='word_1_19' title='bbox 1737 380 1940 417; x_wconf 96'>professional</span>
       <span class='ocrx_word' id='word_1_20' title='bbox 1952 380 2112 411; x_wconf 96'>musician.</span>
       <span class='ocrx_word' id='word_1_21' title='bbox 2127 381 2152 410; x_wconf 96'>It</span>
       <span class='ocrx_word' id='word_1_22' title='bbox 2164 380 2188 410; x_wconf 96'>is</span>
@@ -209,7 +209,7 @@
    <div class='ocr_carea' id='block_1_8' title="bbox 382 1021 689 1065">
     <p class='ocr_par' id='par_1_10' lang='eng' title="bbox 382 1021 689 1065">
      <span class='ocr_line' id='line_1_15' title="bbox 382 1021 689 1065; baseline 0.003 -8; x_size 45; x_descenders 8; x_ascenders 12">
-      <span class='ocrx_word' id='word_1_135' title='bbox 382 1021 564 1065; x_wconf 96'>rhythmic</span>
+      <span class='ocrx_word' id='word_1_135' title='bbox 382 1021 564 1065; x_wconf 95'>rhythmic</span>
       <span class='ocrx_word' id='word_1_136' title='bbox 577 1021 689 1058; x_wconf 96'>value.</span>
      </span>
     </p>
@@ -246,7 +246,7 @@
    <div class='ocr_carea' id='block_1_11' title="bbox 349 1195 874 1240">
     <p class='ocr_par' id='par_1_13' lang='eng' title="bbox 349 1195 874 1240">
      <span class='ocr_line' id='line_1_18' title="bbox 349 1195 874 1240; baseline 0 -8; x_size 45; x_descenders 8; x_ascenders 12">
-      <span class='ocrx_word' id='word_1_154' title='bbox 349 1203 367 1222; x_wconf 74'>©</span>
+      <span class='ocrx_word' id='word_1_154' title='bbox 349 1203 367 1222; x_wconf 70'>©</span>
       <span class='ocrx_word' id='word_1_155' title='bbox 382 1195 560 1240; x_wconf 96'>Optional</span>
       <span class='ocrx_word' id='word_1_156' title='bbox 573 1201 709 1233; x_wconf 96'>remote</span>
       <span class='ocrx_word' id='word_1_157' title='bbox 723 1196 874 1233; x_wconf 95'>control.</span>
@@ -300,8 +300,8 @@
       <span class='ocrx_word' id='word_1_190' title='bbox 1213 1419 1232 1450; x_wconf 74'>1,</span>
      </span>
      <span class='ocr_line' id='line_1_23' title="bbox 346 1457 1223 1491; baseline 0.001 -7; x_size 34; x_descenders 7; x_ascenders 8">
-      <span class='ocrx_word' id='word_1_191' title='bbox 346 1457 430 1490; x_wconf 14'>you’</span>
-      <span class='ocrx_word' id='word_1_192' title='bbox 406 1453 436 1496; x_wconf 14'>ll</span>
+      <span class='ocrx_word' id='word_1_191' title='bbox 346 1457 430 1490; x_wconf 15'>you’</span>
+      <span class='ocrx_word' id='word_1_192' title='bbox 406 1453 439 1496; x_wconf 15'>ll</span>
       <span class='ocrx_word' id='word_1_193' title='bbox 441 1457 506 1485; x_wconf 96'>hear</span>
       <span class='ocrx_word' id='word_1_194' title='bbox 517 1458 590 1485; x_wconf 96'>what</span>
       <span class='ocrx_word' id='word_1_195' title='bbox 600 1466 654 1491; x_wconf 93'>you</span>
@@ -361,7 +361,7 @@
       <span class='ocrx_word' id='word_1_230' title='bbox 684 1617 844 1648; x_wconf 95'>REWIND,</span>
       <span class='ocrx_word' id='word_1_231' title='bbox 857 1616 912 1643; x_wconf 95'>and</span>
       <span class='ocrx_word' id='word_1_232' title='bbox 924 1616 1068 1643; x_wconf 96'>LOCATE</span>
-      <span class='ocrx_word' id='word_1_233' title='bbox 1079 1616 1199 1643; x_wconf 95'>controls</span>
+      <span class='ocrx_word' id='word_1_233' title='bbox 1079 1616 1199 1643; x_wconf 96'>controls</span>
      </span>
      <span class='ocr_line' id='line_1_28' title="bbox 346 1655 1202 1689; baseline 0 -7; x_size 34; x_descenders 6; x_ascenders 9">
       <span class='ocrx_word' id='word_1_234' title='bbox 346 1663 409 1688; x_wconf 92'>may</span>
@@ -400,9 +400,9 @@
      </span>
      <span class='ocr_line' id='line_1_31' title="bbox 346 1773 1203 1808; baseline 0.001 -8; x_size 33; x_descenders 6; x_ascenders 8">
       <span class='ocrx_word' id='word_1_263' title='bbox 346 1774 448 1806; x_wconf 96'>record,</span>
-      <span class='ocrx_word' id='word_1_264' title='bbox 460 1774 506 1801; x_wconf 97'>the</span>
-      <span class='ocrx_word' id='word_1_265' title='bbox 503 1769 577 1812; x_wconf 96'>first</span>
-      <span class='ocrx_word' id='word_1_266' title='bbox 581 1774 658 1801; x_wconf 96'>track</span>
+      <span class='ocrx_word' id='word_1_264' title='bbox 460 1774 506 1801; x_wconf 96'>the</span>
+      <span class='ocrx_word' id='word_1_265' title='bbox 517 1773 577 1801; x_wconf 96'>first</span>
+      <span class='ocrx_word' id='word_1_266' title='bbox 587 1774 662 1801; x_wconf 96'>track</span>
       <span class='ocrx_word' id='word_1_267' title='bbox 673 1774 726 1801; x_wconf 96'>will</span>
       <span class='ocrx_word' id='word_1_268' title='bbox 736 1774 799 1807; x_wconf 96'>play</span>
       <span class='ocrx_word' id='word_1_269' title='bbox 809 1774 836 1801; x_wconf 96'>in</span>
@@ -412,11 +412,11 @@
       <span class='ocrx_word' id='word_1_273' title='bbox 1148 1782 1203 1807; x_wconf 96'>you</span>
      </span>
      <span class='ocr_line' id='line_1_32' title="bbox 346 1813 1191 1847; baseline 0.002 -8; x_size 33; x_descenders 5; x_ascenders 9">
-      <span class='ocrx_word' id='word_1_274' title='bbox 346 1813 454 1840; x_wconf 95'>MUTE</span>
-      <span class='ocrx_word' id='word_1_275' title='bbox 464 1813 492 1845; x_wconf 95'>it,</span>
+      <span class='ocrx_word' id='word_1_274' title='bbox 346 1813 454 1840; x_wconf 94'>MUTE</span>
+      <span class='ocrx_word' id='word_1_275' title='bbox 464 1813 492 1845; x_wconf 94'>it,</span>
       <span class='ocrx_word' id='word_1_276' title='bbox 505 1821 537 1840; x_wconf 95'>or</span>
       <span class='ocrx_word' id='word_1_277' title='bbox 547 1813 642 1840; x_wconf 95'>SOLO</span>
-      <span class='ocrx_word' id='word_1_278' title='bbox 653 1814 769 1841; x_wconf 95'>another</span>
+      <span class='ocrx_word' id='word_1_278' title='bbox 653 1814 769 1841; x_wconf 96'>another</span>
       <span class='ocrx_word' id='word_1_279' title='bbox 779 1814 875 1847; x_wconf 94'>track).</span>
       <span class='ocrx_word' id='word_1_280' title='bbox 889 1814 920 1840; x_wconf 96'>In</span>
       <span class='ocrx_word' id='word_1_281' title='bbox 930 1813 984 1841; x_wconf 96'>this</span>
@@ -518,13 +518,13 @@
      <span class='ocr_line' id='line_1_41' title="bbox 345 2207 1228 2242; baseline 0.002 -8; x_size 35; x_descenders 7; x_ascenders 9">
       <span class='ocrx_word' id='word_1_349' title='bbox 345 2207 412 2234; x_wconf 96'>tion.</span>
       <span class='ocrx_word' id='word_1_350' title='bbox 424 2208 461 2235; x_wconf 93'>To</span>
-      <span class='ocrx_word' id='word_1_351' title='bbox 472 2208 592 2235; x_wconf 91'>overdub</span>
+      <span class='ocrx_word' id='word_1_351' title='bbox 472 2208 592 2235; x_wconf 92'>overdub</span>
       <span class='ocrx_word' id='word_1_352' title='bbox 603 2212 680 2235; x_wconf 96'>notes</span>
       <span class='ocrx_word' id='word_1_353' title='bbox 691 2212 718 2235; x_wconf 96'>at</span>
       <span class='ocrx_word' id='word_1_354' title='bbox 729 2208 841 2242; x_wconf 96'>specific</span>
       <span class='ocrx_word' id='word_1_355' title='bbox 851 2209 943 2242; x_wconf 97'>points</span>
-      <span class='ocrx_word' id='word_1_356' title='bbox 955 2208 1049 2236; x_wconf 96'>within</span>
-      <span class='ocrx_word' id='word_1_357' title='bbox 1060 2217 1076 2236; x_wconf 96'>a</span>
+      <span class='ocrx_word' id='word_1_356' title='bbox 955 2208 1049 2236; x_wconf 97'>within</span>
+      <span class='ocrx_word' id='word_1_357' title='bbox 1060 2217 1076 2236; x_wconf 97'>a</span>
       <span class='ocrx_word' id='word_1_358' title='bbox 1086 2216 1228 2242; x_wconf 96'>sequence,</span>
      </span>
     </p>
@@ -681,7 +681,7 @@
       <span class='ocrx_word' id='word_1_465' title='bbox 1810 1808 1911 1835; x_wconf 96'>SONG</span>
       <span class='ocrx_word' id='word_1_466' title='bbox 1923 1808 2050 1836; x_wconf 96'>function</span>
       <span class='ocrx_word' id='word_1_467' title='bbox 2060 1812 2089 1835; x_wconf 96'>to</span>
-      <span class='ocrx_word' id='word_1_468' title='bbox 2103 1808 2215 1836; x_wconf 93'>“chain”</span>
+      <span class='ocrx_word' id='word_1_468' title='bbox 2103 1808 2215 1836; x_wconf 91'>“chain”</span>
      </span>
      <span class='ocr_line' id='line_1_56' title="bbox 1295 1847 2135 1881; baseline 0.001 -7; x_size 34; x_descenders 7; x_ascenders 8">
       <span class='ocrx_word' id='word_1_469' title='bbox 1295 1848 1370 1874; x_wconf 96'>them</span>
@@ -874,10 +874,10 @@
    <div class='ocr_carea' id='block_1_25' title="bbox 347 2556 1768 2590">
     <p class='ocr_par' id='par_1_33' lang='eng' title="bbox 347 2556 1768 2590">
      <span class='ocr_header' id='line_1_70' title="bbox 347 2556 1768 2590; baseline 0.001 -8; x_size 34; x_descenders 7; x_ascenders 8">
-      <span class='ocrx_word' id='word_1_604' title='bbox 347 2561 361 2575; x_wconf 86'>¢</span>
-      <span class='ocrx_word' id='word_1_605' title='bbox 372 2556 433 2583; x_wconf 85'>Iwo</span>
+      <span class='ocrx_word' id='word_1_604' title='bbox 347 2561 361 2575; x_wconf 85'>¢</span>
+      <span class='ocrx_word' id='word_1_605' title='bbox 372 2556 433 2583; x_wconf 84'>Iwo</span>
       <span class='ocrx_word' id='word_1_606' title='bbox 443 2556 612 2583; x_wconf 96'>TRIGGER</span>
-      <span class='ocrx_word' id='word_1_607' title='bbox 623 2556 797 2584; x_wconf 96'>OUTPUTS</span>
+      <span class='ocrx_word' id='word_1_607' title='bbox 623 2556 797 2584; x_wconf 95'>OUTPUTS</span>
       <span class='ocrx_word' id='word_1_608' title='bbox 808 2565 871 2590; x_wconf 96'>may</span>
       <span class='ocrx_word' id='word_1_609' title='bbox 881 2557 915 2584; x_wconf 96'>be</span>
       <span class='ocrx_word' id='word_1_610' title='bbox 925 2557 1119 2590; x_wconf 96'>programmed</span>
@@ -887,7 +887,7 @@
       <span class='ocrx_word' id='word_1_614' title='bbox 1381 2561 1409 2584; x_wconf 96'>at</span>
       <span class='ocrx_word' id='word_1_615' title='bbox 1419 2565 1472 2590; x_wconf 96'>any</span>
       <span class='ocrx_word' id='word_1_616' title='bbox 1483 2557 1598 2584; x_wconf 96'>selected</span>
-      <span class='ocrx_word' id='word_1_617' title='bbox 1608 2561 1673 2584; x_wconf 97'>note</span>
+      <span class='ocrx_word' id='word_1_617' title='bbox 1608 2561 1673 2584; x_wconf 96'>note</span>
       <span class='ocrx_word' id='word_1_618' title='bbox 1683 2556 1768 2583; x_wconf 96'>value.</span>
      </span>
     </p>
@@ -911,15 +911,15 @@
    </div>
    <div class='ocr_carea' id='block_1_27' title="bbox 347 2648 2100 2727">
     <p class='ocr_par' id='par_1_35' lang='eng' title="bbox 347 2648 2100 2727">
-     <span class='ocr_header' id='line_1_72' title="bbox 347 2648 1664 2682; baseline 0.001 -7; x_size 33; x_descenders 6; x_ascenders 8">
-      <span class='ocrx_word' id='word_1_630' title='bbox 347 2654 360 2667; x_wconf 45'>©</span>
-      <span class='ocrx_word' id='word_1_631' title='bbox 372 2648 483 2675; x_wconf 95'>Utilizes</span>
-      <span class='ocrx_word' id='word_1_632' title='bbox 493 2648 564 2680; x_wconf 96'>ultra</span>
+     <span class='ocr_header' id='line_1_72' title="bbox 347 2648 1664 2682; baseline 0 -7; x_size 33; x_descenders 6; x_ascenders 8">
+      <span class='ocrx_word' id='word_1_630' title='bbox 347 2654 360 2667; x_wconf 51'>©</span>
+      <span class='ocrx_word' id='word_1_631' title='bbox 372 2648 483 2675; x_wconf 96'>Utilizes</span>
+      <span class='ocrx_word' id='word_1_632' title='bbox 493 2648 564 2677; x_wconf 96'>ultra</span>
       <span class='ocrx_word' id='word_1_633' title='bbox 573 2648 744 2682; x_wconf 96'>high-speed,</span>
       <span class='ocrx_word' id='word_1_634' title='bbox 757 2649 772 2676; x_wconf 95'>8</span>
       <span class='ocrx_word' id='word_1_635' title='bbox 783 2649 862 2675; x_wconf 94'>MHz</span>
-      <span class='ocrx_word' id='word_1_636' title='bbox 873 2649 954 2676; x_wconf 96'>80186</span>
-      <span class='ocrx_word' id='word_1_637' title='bbox 965 2649 994 2676; x_wconf 96'>16</span>
+      <span class='ocrx_word' id='word_1_636' title='bbox 873 2649 954 2676; x_wconf 95'>80186</span>
+      <span class='ocrx_word' id='word_1_637' title='bbox 965 2649 994 2676; x_wconf 95'>16</span>
       <span class='ocrx_word' id='word_1_638' title='bbox 1004 2648 1043 2676; x_wconf 96'>bit</span>
       <span class='ocrx_word' id='word_1_639' title='bbox 1054 2653 1197 2682; x_wconf 96'>computer</span>
       <span class='ocrx_word' id='word_1_640' title='bbox 1208 2649 1350 2682; x_wconf 96'>internally</span>
@@ -960,18 +960,18 @@
    <div class='ocr_carea' id='block_1_29' title="bbox 347 2777 2174 2811">
     <p class='ocr_par' id='par_1_37' lang='eng' title="bbox 347 2777 2174 2811">
      <span class='ocr_header' id='line_1_75' title="bbox 347 2777 2174 2811; baseline 0.001 -8; x_size 33; x_descenders 5; x_ascenders 9">
-      <span class='ocrx_word' id='word_1_664' title='bbox 347 2782 360 2796; x_wconf 81'>¢</span>
+      <span class='ocrx_word' id='word_1_664' title='bbox 347 2782 360 2796; x_wconf 79'>¢</span>
       <span class='ocrx_word' id='word_1_665' title='bbox 372 2777 504 2804; x_wconf 94'>TEMPO</span>
       <span class='ocrx_word' id='word_1_666' title='bbox 515 2785 578 2810; x_wconf 96'>may</span>
-      <span class='ocrx_word' id='word_1_667' title='bbox 588 2777 622 2804; x_wconf 96'>be</span>
+      <span class='ocrx_word' id='word_1_667' title='bbox 588 2777 622 2804; x_wconf 95'>be</span>
       <span class='ocrx_word' id='word_1_668' title='bbox 633 2778 741 2804; x_wconf 95'>entered</span>
       <span class='ocrx_word' id='word_1_669' title='bbox 751 2777 934 2811; x_wconf 96'>numerically,</span>
-      <span class='ocrx_word' id='word_1_670' title='bbox 946 2777 1101 2811; x_wconf 95'>adjustable</span>
-      <span class='ocrx_word' id='word_1_671' title='bbox 1111 2777 1139 2804; x_wconf 95'>in</span>
+      <span class='ocrx_word' id='word_1_670' title='bbox 946 2777 1101 2811; x_wconf 96'>adjustable</span>
+      <span class='ocrx_word' id='word_1_671' title='bbox 1111 2777 1139 2804; x_wconf 96'>in</span>
       <span class='ocrx_word' id='word_1_672' title='bbox 1149 2778 1239 2805; x_wconf 96'>tenths</span>
       <span class='ocrx_word' id='word_1_673' title='bbox 1250 2778 1282 2805; x_wconf 96'>of</span>
-      <span class='ocrx_word' id='word_1_674' title='bbox 1290 2786 1307 2805; x_wconf 91'>a</span>
-      <span class='ocrx_word' id='word_1_675' title='bbox 1317 2777 1567 2805; x_wconf 91'>Beat-Per-Minute</span>
+      <span class='ocrx_word' id='word_1_674' title='bbox 1290 2786 1307 2805; x_wconf 93'>a</span>
+      <span class='ocrx_word' id='word_1_675' title='bbox 1317 2777 1567 2805; x_wconf 92'>Beat-Per-Minute</span>
       <span class='ocrx_word' id='word_1_676' title='bbox 1577 2777 1748 2809; x_wconf 96'>increments,</span>
       <span class='ocrx_word' id='word_1_677' title='bbox 1760 2785 1792 2804; x_wconf 96'>or</span>
       <span class='ocrx_word' id='word_1_678' title='bbox 1803 2777 1839 2810; x_wconf 96'>by</span>