diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 4700004e..416a4af0 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -1,10 +1,19 @@ # OCRmyPDF # -FROM ubuntu:20.04 as base -FROM base as builder +FROM ubuntu:21.04 as base ENV LANG=C.UTF-8 +ENV TZ=UTC +RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 \ + libqpdf-dev \ + zlib1g \ + liblept5 + +FROM base as builder RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential autoconf automake libtool \ @@ -13,7 +22,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ python3-dev \ python3-distutils \ libffi-dev \ - libqpdf-dev \ ca-certificates \ curl \ git @@ -37,26 +45,15 @@ COPY . /app WORKDIR /app -RUN pip3 install --no-cache-dir \ - -r requirements/main.txt \ - -r requirements/webservice.txt \ - -r requirements/test.txt \ - -r requirements/watcher.txt \ - . +RUN pip3 install --no-cache-dir .[test,webservice,watcher] FROM base -ENV LANG=C.UTF-8 - RUN apt-get update && apt-get install -y --no-install-recommends \ ghostscript \ img2pdf \ - liblept5 \ libsm6 libxext6 libxrender-dev \ - zlib1g \ pngquant \ - python3 \ - qpdf \ tesseract-ocr \ tesseract-ocr-chi-sim \ tesseract-ocr-deu \ diff --git a/.github/ISSUE_TEMPLATE/3-feature_request.md b/.github/ISSUE_TEMPLATE/3-feature_request.md index bbcbbe7d..fc3ba503 100644 --- a/.github/ISSUE_TEMPLATE/3-feature_request.md +++ b/.github/ISSUE_TEMPLATE/3-feature_request.md @@ -14,7 +14,14 @@ A clear and concise description of what the problem is. Ex. I'm always frustrate A clear and concise description of what you want to happen. **Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. +A clear and concise description of any alternative solutions or features you've considered. Please include the versions of OCRmyPDF and other supporting programs (Tesseract OCR, Ghostscript) - maybe an alternative already exists in a newer version. + +**Example file** +If your issue concerns how OCRmyPDF processes certain files, and please provide an example file that helps illustrate how OCRmyPDF's output could be improve. + +Please provide an input file with no personal or confidential information. At your option you may [GPG-encrypt the file](https://github.com/jbarlow83/OCRmyPDF/wiki) for OCRmyPDF's author only. + +Links to files hosted elsewhere are perfectly acceptable. You could also look in ``tests/resources`` and see if any of those files reproduce your issue. **Additional context** Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cad70f3f..23820f1b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,7 @@ on: - master - ci - release/* + - feature/* tags: - v* paths-ignore: @@ -18,8 +19,24 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04] #, ubuntu-20.04] - python: ["3.6"] #, "3.7", "3.8", "3.9"] + include: + - os: ubuntu-18.04 + python: 3.6 + - os: ubuntu-18.04 + python: 3.7 + - os: ubuntu-20.04 + python: 3.8 + - os: ubuntu-20.04 + python: 3.9 + - os: ubuntu-latest + python: 3.9 + - os: ubuntu-20.04 + python: "pypy-3.6" + - os: ubuntu-latest + python: "pypy-3.7" + - os: ubuntu-latest + python: 3.9 + tesseract5: true env: OS: ${{ matrix.os }} @@ -35,6 +52,11 @@ jobs: with: python-version: ${{ matrix.python }} + - name: Install Tesseract 5 + if: matrix.tesseract5 + run: | + sudo add-apt-repository ppa:alex-p/tesseract-ocr-devel + - name: Install common packages run: | sudo apt-get update @@ -65,9 +87,17 @@ jobs: sudo apt-get install -y --no-install-recommends \ libexempi8 + - name: Install Ubuntu packages for PyPy + if: startsWith(matrix.python, 'pypy') + run: | + sudo apt-get install -y --no-install-recommends \ + libxml2-dev \ + libxslt1-dev \ + pypy3-dev + - name: Install Python packages run: | - python -m pip install -r requirements/main.txt -r requirements/test.txt . + python -m pip install .[test] - name: Report versions run: | @@ -124,7 +154,7 @@ jobs: - name: Install Python packages run: | python -m pip install --upgrade pip - python -m pip install -r requirements/main.txt -r requirements/test.txt . + python -m pip install .[test] - name: Report versions run: | @@ -174,7 +204,7 @@ jobs: - name: Install Python packages run: | python -m pip install --upgrade pip - python -m pip install -r requirements/main.txt -r requirements/test.txt . + python -m pip install .[test] - name: Test run: | @@ -232,6 +262,7 @@ jobs: name: Build Docker images needs: [wheel_sdist_linux, test_linux, test_macos, test_windows] runs-on: ubuntu-latest + if: github.event_name != 'pull_request' steps: - name: Set image tag to release or branch run: echo "DOCKER_IMAGE_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV diff --git a/.gitignore b/.gitignore index 60de406b..1481bb33 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,9 @@ !.gitattributes !.gitignore !.pre-commit-config.yaml -!.readthedocs.yml +!.readthedocs.yaml !.github/ +!.docker/ # Dev scratch *.ipynb diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cd240e2..20da1ef6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,23 +1,38 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.0.1 hooks: - id: check-case-conflict - id: check-merge-conflict - id: check-toml - id: check-yaml - id: debug-statements - - repo: https://github.com/asottile/seed-isort-config - rev: v2.2.0 - hooks: - - id: seed-isort-config - - repo: https://github.com/pre-commit/mirrors-isort - rev: v5.7.0 # pick the isort version you'd like to use from https://github.com/pre-commit/mirrors-isort/releases + - repo: https://github.com/pycqa/isort + rev: 5.9.3 hooks: - id: isort + args: ["--profile", "black"] - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 21.9b0 hooks: - id: black language_version: python exclude: ^src/ocrmypdf/lib/_leptonica.py + - repo: https://github.com/asottile/setup-cfg-fmt + rev: v1.17.0 + hooks: + - id: setup-cfg-fmt + - repo: https://github.com/asottile/pyupgrade + rev: v2.26.0 + hooks: + - id: pyupgrade + args: ["--py36-plus"] + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.910 + hooks: + - id: mypy + additional_dependencies: + - types-toml + - types-setuptools + - types-requests + - types-Pillow diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..3c67df11 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,22 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally build your docs in additional formats such as PDF +formats: + - pdf + +# Optionally set the version of Python and requirements required to build your docs +python: + version: 3.7 + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index de4ec366..00000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,10 +0,0 @@ -build: - image: latest - -python: - version: 3.6 - -formats: - - pdf - -requirements_file: requirements/main.txt diff --git a/README.md b/README.md index a0bf0a35..b03627a0 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ I searched the web for a free command line tool to OCR PDF files: I found many, ## Installation -Linux, Windows, macOS and FreeBSD are supported. Docker images are also available. +Linux, Windows, macOS and FreeBSD are supported. Docker images are also available, for both x64 and ARM. | Operating system | Install command | | ----------------------------- | ------------------------------| @@ -91,6 +91,11 @@ brew install tesseract-lang You can then pass the `-l LANG` argument to OCRmyPDF to give a hint as to what languages it should search for. Multiple languages can be requested. +OCRmyPDF supports Tesseract 4.0 and the beta versions of Tesseract 5.0. It will +automatically use whichever version it finds first on the `PATH` environment +variable. On Windows, if `PATH` does not provide a Tesseract binary, we use +the highest version number that is installed according to the Windows Registry. + ## Documentation and support Once OCRmyPDF is installed, the built-in help which explains the command syntax and options can be accessed via: @@ -115,6 +120,7 @@ In addition to the required Python version (3.6+), OCRmyPDF requires external pr - [heise Open Source, 09/2014: Texterkennung mit OCRmyPDF](https://heise.de/-2356670) - [heise Durchsuchbare PDF-Dokumente mit OCRmyPDF erstellen](https://www.heise.de/ratgeber/Durchsuchbare-PDF-Dokumente-mit-OCRmyPDF-erstellen-4607592.html) - [Excellent Utilities: OCRmyPDF](https://www.linuxlinks.com/excellent-utilities-ocrmypdf-add-ocr-text-layer-scanned-pdfs/) +- [LinuxUser Texterkennung mit OCRmyPDF und Scanbd automatisieren](https://www.linux-community.de/ausgaben/linuxuser/2021/06/texterkennung-mit-ocrmypdf-und-scanbd-automatisieren/) ## Business enquiries diff --git a/docs/advanced.rst b/docs/advanced.rst index 09a7376a..fa5489f4 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -228,8 +228,8 @@ preprocessing is specified, then the image layer is a new PDF. Unlike ``sandwich`` this renderer is implemented within OCRmyPDF; anyone looking to customize how OCR is presented should look here. A major disadvantage of this renderer is it not capable of correctly handling -text outside the Latin alphabet. Pull requests to improve the situation -are welcome. +text outside the Latin alphabet (specifically, it supports the ISO 8859-1 +character). Pull requests to improve the situation are welcome. Currently, this renderer has the best compatibility with Mozilla's PDF.js viewer. diff --git a/docs/api.rst b/docs/api.rst index 3459e8b8..f3e03534 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -12,7 +12,7 @@ subprocess call anyway, as this provides isolation of its activities. Example ======= -OCRmyPDF one high-level function to run its main engine from an +OCRmyPDF provides one high-level function to run its main engine from an application. The parameters are symmetric to the command line arguments and largely have the same functions. @@ -23,7 +23,7 @@ and largely have the same functions. if __name__ == '__main__': # To ensure correct behavior on Windows and macOS ocrmypdf.ocr('input.pdf', 'output.pdf', deskew=True) -With a few exceptions, all of the command line arguments are available +With some exceptions, all of the command line arguments are available and may be passed as equivalent keywords. A few differences are that ``verbose`` and ``quiet`` are not available. @@ -41,29 +41,29 @@ execution. To do this, it will: - manage the signal flags of its worker processes - execute other subprocesses (forking and executing other programs) -The Python process that calls ``ocrmypdf.ocr()`` must be sufficiently +The Python process that calls :func:`ocrmypdf.ocr()` must be sufficiently privileged to perform these actions. -There is no currently no option to manage how jobs are scheduled other +There currently is no option to manage how jobs are scheduled other than the argument ``jobs=`` which will limit the number of worker processes. -Creating a child process to call ``ocrmypdf.ocr()`` is suggested. That +Creating a child process to call :func:`ocrmypdf.ocr()` is suggested. That way your application will survive and remain interactive even if OCRmyPDF fails for any reason. -Programs that call ``ocrmypdf.ocr()`` should also install a SIGBUS signal +Programs that call :func:`ocrmypdf.ocr()` should also install a SIGBUS signal handler (except on Windows), to raise an exception if access to a memory mapped file fails. OCRmyPDF may use memory mapping. -``ocrmypdf.ocr()`` will take a threading lock to prevent multiple runs of itself +:func:`ocrmypdf.ocr()` will take a threading lock to prevent multiple runs of itself in the same Python interpreter process. This is not thread-safe, because of how OCRmyPDF's plugins and Python's library import system work. If you need to parallelize OCRmyPDF, use processes. .. warning:: - On Windows and macOS, the script that calls ``ocrmypdf.ocr()`` must be + On Windows and macOS, the script that calls :func:`ocrmypdf.ocr()` must be protected by an "ifmain" guard (``if __name__ == '__main__'``). If you do not take at least one of these steps, process semantics will prevent OCRmyPDF from working correctly. @@ -96,7 +96,7 @@ Exceptions OCRmyPDF may throw standard Python exceptions, ``ocrmypdf.exceptions.*`` exceptions, some exceptions related to multiprocessing, and -``KeyboardInterrupt``. The parent process should provide an exception +:exc:`KeyboardInterrupt`. The parent process should provide an exception handler. OCRmyPDF will clean up its temporary files and worker processes automatically when an exception occurs. diff --git a/docs/batch.rst b/docs/batch.rst index e8e97374..e66388df 100644 --- a/docs/batch.rst +++ b/docs/batch.rst @@ -111,7 +111,7 @@ Users may need to customize the script to meet their requirements. .. code-block:: bash - pip3 install -r requirements/watcher.txt + pip3 install ocrmypdf[watcher] env OCR_INPUT_DIRECTORY=/mnt/input-pdfs \ OCR_OUTPUT_DIRECTORY=/mnt/output-pdfs \ diff --git a/docs/conf.py b/docs/conf.py index 4a6dc0ac..2aa1ad24 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,9 +31,18 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.napoleon'] +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.autosummary', + 'sphinx.ext.napoleon', + 'sphinx_issues', +] +# Extension settings +intersphinx_mapping = {'https://docs.python.org/': None} napoleon_use_rtype = False +issues_github_path = "jbarlow83/OCRmyPDF" # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -54,7 +63,7 @@ master_doc = 'index' # General information about the project. project = 'ocrmypdf' copyright = ( - '2020, James R. Barlow. Licensed under Creative Commons Attribution-ShareAlike 4.0.' + '2021, James R. Barlow. Licensed under Creative Commons Attribution-ShareAlike 4.0.' ) author = 'James R. Barlow' @@ -86,11 +95,10 @@ if on_rtd: ] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) - -from pkg_resources import get_distribution, DistributionNotFound +from importlib_metadata import version as package_version # The full version, including alpha/beta/rc tags. -release = get_distribution('ocrmypdf').version +release = package_version('ocrmypdf') version = '.'.join(release.split('.')[:2]) @@ -273,7 +281,7 @@ htmlhelp_basename = 'ocrmypdfdoc' # -- Options for LaTeX output --------------------------------------------- -latex_elements = { +latex_elements = { # type: ignore # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', diff --git a/docs/images/logo-square.png b/docs/images/logo-square.png new file mode 100644 index 00000000..0f4ace15 Binary files /dev/null and b/docs/images/logo-square.png differ diff --git a/docs/images/logo-square.svg b/docs/images/logo-square.svg new file mode 100644 index 00000000..b5ae241e --- /dev/null +++ b/docs/images/logo-square.svg @@ -0,0 +1,233 @@ + +image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/index.rst b/docs/index.rst index 91eac433..721b73a9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,6 +1,8 @@ OCRmyPDF documentation ====================== +.. figure:: images/logo.svg + OCRmyPDF adds an optical character recognition (OCR) text layer to scanned PDF files, allowing them to be searched. diff --git a/docs/installation.rst b/docs/installation.rst index 6d096996..8439cdfe 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -23,7 +23,7 @@ These platforms have one-liner installs: +-------------------------------+-------------------------------+ | LinuxBrew | ``brew install ocrmypdf`` | +-------------------------------+-------------------------------+ -| FreeBSD | ``pkg install py37-ocrmypdf`` | +| FreeBSD | ``pkg install py38-ocrmypdf`` | +-------------------------------+-------------------------------+ | Conda (WSL, macOS, Linux) | ``conda install ocrmypdf`` | +-------------------------------+-------------------------------+ @@ -635,7 +635,7 @@ versions likely work but have not been tested. .. code-block:: bash - pkg install py37-ocrmypdf + pkg install py38-ocrmypdf To install a more recent version, you could attempt to first install the system version with ``pkg``, then use ``pip install --user ocrmypdf``. @@ -812,8 +812,7 @@ To install all of the development and test requirements: python3 -m venv source venv/bin/activate cd OCRmyPDF - pip install -e . - pip install -r requirements/dev.txt -r requirements/test.txt + pip install -e .[test] To add JBIG2 encoding, see :ref:`jbig2`. diff --git a/docs/introduction.rst b/docs/introduction.rst index 360d57ec..929a4242 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -2,7 +2,12 @@ Introduction ============ -OCRmyPDF is a Python 3 application and library that adds OCR layers to PDFs. +OCRmyPDF is an application and library that adds text "layers" to images +in PDFs, making scanned image PDFs searchable. It uses OCR to guess what text +is contained in images. It is written in Python. OCRmyPDF supports plugins +that allow customization of its processing steps, and is very tolerant of +PDFs that contain scanned images and "born digital" content that needs no +text recognition. About OCR ========= @@ -26,7 +31,7 @@ exactly. They contain `vector graphics `__ that can contain raster objects such as scanned images. Because PDFs can contain multiple pages (unlike many image formats) and can contain fonts -and text, it is a good formats for exchanging scanned documents. +and text, it is a good format for exchanging scanned documents. |image| @@ -35,9 +40,9 @@ have one image. Some scanners or scanning software will segment pages into monochromatic text and color regions for example, to improve the compression ratio and appearance of the page. -Rasterizing a PDF is the process of generating an image suitable for -display or analyzing with an OCR engine. OCR engines like Tesseract work -with images, not vector objects. +Rasterizing a PDF is the process of generating corresponding raster images. +OCR engines like Tesseract work with images, not scalable vector graphics +or mixed raster-vector-text graphics such as PDF. About PDF/A =========== @@ -76,7 +81,7 @@ OCRmyPDF analyzes each page of a PDF to determine the colorspace and resolution (DPI) needed to capture all of the information on that page without losing content. It uses `Ghostscript `__ to rasterize the page, and -then performs on OCR on the rasterized image to create an OCR "layer". +then performs on OCR the rasterized image to create an OCR "layer". The layer is then grafted back onto the original PDF. While one can use a program like Ghostscript or ImageMagick to get an @@ -84,9 +89,9 @@ image and put the image through Tesseract, that actually creates a new PDF and many details may be lost. OCRmyPDF can produce a minimally changed PDF as output. -OCRmyPDF also some image processing options like deskew which improve -the appearance of files and quality of OCR. When these are used, the OCR -layer is grafted onto the processed image instead. +OCRmyPDF also provides some image processing options, like deskew, which +improves the appearance of files and quality of OCR. When these are used, +the OCR layer is grafted onto the processed image instead. By default, OCRmyPDF produces archival PDFs – PDF/A, which are a stricter subset of PDF features designed for long term archives. If @@ -139,7 +144,7 @@ Limitations OCRmyPDF is limited by the Tesseract OCR engine. As such it experiences these limitations, as do any other programs that rely on Tesseract: -- The OCR is not as accurate as commercial solutions such as Abbyy. +- The OCR is not as accurate as commercial OCR solutions. - It is not capable of recognizing handwriting. - It may find gibberish and report this as OCR output. - If a document contains languages outside of those given in the @@ -207,8 +212,9 @@ consider one of these similar open source programs: - pdf2pdfocr - pdfsandwich -- pypdfocr -- pdfbeads + +Ghostscript recently added three "pdfocr" output devices. They work by +rasterizing all content and converting all pages to a single colour space. Web front-ends ============== diff --git a/docs/jbig2.rst b/docs/jbig2.rst index 81789b6f..c49f0e9c 100644 --- a/docs/jbig2.rst +++ b/docs/jbig2.rst @@ -9,11 +9,11 @@ encoding was patented for a long time. All known JBIG2 US patents have expired as of 2017, but it is possible that unknown patents exist. JBIG2 encoding is recommended for OCRmyPDF and is used to losslessly -create smaller PDFs. If JBIG2 encoding not available, lower quality +create smaller PDFs. If JBIG2 encoding is not available, lower quality encodings will be used. JBIG2 decoding is not patented and is performed automatically by most -PDF viewers. It is widely supported has been part of the PDF +PDF viewers. It is widely supported and has been part of the PDF specification since 2001. On macOS, Homebrew packages jbig2enc and OCRmyPDF includes it by @@ -37,7 +37,7 @@ Lossy mode JBIG2 OCRmyPDF provides lossy mode JBIG2 as an advanced feature. Users should `review the technical concerns with JBIG2 in lossy -mode `__ +mode `__ and decide if this feature is acceptable for their use case. JBIG2 lossy mode does achieve higher compression ratios than any other diff --git a/docs/pdfsecurity.rst b/docs/pdfsecurity.rst index 04ad4e90..a11c84c3 100644 --- a/docs/pdfsecurity.rst +++ b/docs/pdfsecurity.rst @@ -19,7 +19,7 @@ PDF is a rich, complex file format. The official PDF 1.7 specification, ISO 32000:2008, is hundreds of pages long and references several annexes each of which are similar in length. PDFs can contain video, audio, XML, JavaScript and other programming, and forms. In some cases, they can -open internet connections to pre-selected URLs. All of these possible +open internet connections to pre-selected URLs. All of these are possible attack vectors. In short, PDFs `may contain @@ -31,7 +31,7 @@ describes a high-paranoia method which allows potentially hostile PDFs to be viewed and rasterized safely in a disposable virtual machine. A trusted PDF created in this manner is converted to images and loses all information making it searchable and losing all compression. OCRmyPDF -could be used restore searchability. +could be used to restore searchability. How OCRmyPDF processes PDFs =========================== @@ -41,8 +41,8 @@ layer. First, it runs all PDFs through `pikepdf `__, a library based on `qpdf `__, a program that repairs PDFs with syntax errors. This is done because, in the author's experience, a -significant number of PDFs in the wild especially those created by -scanners are not well-formed files. qpdf makes it more likely that +significant number of PDFs in the wild, especially those created by +scanners, are not well-formed files. qpdf makes it more likely that OCRmyPDF will succeed, but offers no security guarantees. qpdf is also used to split the PDF into single page PDFs. @@ -66,8 +66,8 @@ service. OCRmyPDF relies on Ghostscript, and therefore, if deployed online one should be prepared to comply with Ghostscript's Affero GPL license, and any other licenses. -Setting aside these concerns, a side effect of OCRmyPDF is it may -incidentally sanitize PDFs that contain certain types of malware. It +Setting aside these concerns, a side effect of OCRmyPDF is that it may +incidentally sanitize PDFs containing certain types of malware. It repairs the PDF with pikepdf/libqpdf, which could correct malformed PDF structures that are part of an attack. When PDF/A output is selected (the default), the input PDF is partially reconstructed by Ghostscript. @@ -83,7 +83,7 @@ Limiting CPU usage OCRmyPDF will attempt to use all available CPUs and storage, so executing ``nice ocrmypdf`` or limiting the number of jobs with the ``-j`` argument may ensure the server remains available. Another option -would be run OCRmyPDF jobs inside a Docker container, a virtual machine, +would be to run OCRmyPDF jobs inside a Docker container, a virtual machine, or a cloud instance, which can impose its own limits on CPU usage and be terminated "from orbit" if it fails to complete. @@ -128,8 +128,9 @@ Commercial alternatives The author also provides professional services that include OCR and building databases around PDFs, and is happy to provide consultation. -Abbyy Cloud OCR is a viable commercial alternative with a web services -API. +Abbyy Cloud OCR is viable commercial alternative with a web services +API. Amazon Textract, Google Cloud Vision, and Microsoft Azure +Computer Vision provide advanced OCR but have less PDF rendering capability. Password protection, digital signatures and certification ========================================================= diff --git a/docs/pipeline.svg b/docs/pipeline.svg deleted file mode 100644 index dc9e3011..00000000 --- a/docs/pipeline.svg +++ /dev/null @@ -1,392 +0,0 @@ - - - - - - -Pipeline: - - -clustertasks - -Pipeline: - - - -t0 - - - - -ocrmypdf.pipeline.triage - - - -t1 - - - - -ocrmypdf.pipeline.repair_and_parse_pdf - - - -t0->t1 - - - - - -t2 - - -ocrmypdf.pipeline.marker_pages - - - -t1->t2 - - - - - -t16 - - - - -ocrmypdf.pipeline.weave_layers - - - -t1->t16 - - - - - -t17 - - - - -ocrmypdf.pipeline.generate_postscript_stub - - - -t1->t17 - - - - - -t18 - - -ocrmypdf.pipeline.metadata_fixup - - - -t1->t18 - - - - - -t3 - - -ocrmypdf.pipeline.ocr_or_skip - - - -t2->t3 - - - - - -t4 - - - - -ocrmypdf.pipeline.rasterize_preview - - - -t3->t4 - - - - - -t5 - - - - -ocrmypdf.pipeline.orient_page - - - -t3->t5 - - - - - -t4->t5 - - - - - -t6 - - - - -ocrmypdf.pipeline.rasterize_with_ghostscript - - - -t5->t6 - - - - - -t13 - -ocrmypdf.pipeline.select_image_layer - - - -t5->t13 - - - - - -t7 - - - - -ocrmypdf.pipeline.preprocess_remove_background - - - -t6->t7 - - - - - -t12 - -ocrmypdf.pipeline.select_visible_page_image - - - -t6->t12 - - - - - -t8 - - - - -ocrmypdf.pipeline.preprocess_deskew - - - -t7->t8 - - - - - -t7->t12 - - - - - -t9 - - - - -ocrmypdf.pipeline.preprocess_clean - - - -t8->t9 - - - - - -t8->t12 - - - - - -t10 - - - - -ocrmypdf.pipeline.select_ocr_image - - - -t9->t10 - - - - - -t9->t12 - - - - - -t11 - - - - -ocrmypdf.pipeline.ocr_tesseract_hocr - - - -t10->t11 - - - - - -t15 - - - - -ocrmypdf.pipeline.ocr_tesseract_textonly_pdf - - - -t10->t15 - - - - - -t14 - - - - -ocrmypdf.pipeline.render_hocr_page - - - -t11->t14 - - - - - -t19 - - -ocrmypdf.pipeline.merge_sidecars - - - -t11->t19 - - - - - -t14->t16 - - - - - -t15->t16 - - - - - -t15->t19 - - - - - -t12->t13 - - - - - -t13->t16 - - - - - -t16->t18 - - - - - -t17->t18 - - - - - -t20 - - - - -ocrmypdf.pipeline.optimize_pdf - - - -t18->t20 - - - - - -t21 - - -ocrmypdf.pipeline.copy_final - - - -t20->t21 - - - - - diff --git a/docs/plugins.rst b/docs/plugins.rst index 952af3fd..c8d3e5ae 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -61,9 +61,10 @@ similar to ``pytest`` packages such as ``pytest-cov`` (the package) and .. note:: - We strongly recommend plugin authors name their plugins with the prefix + We recommend plugin authors name their plugins with the prefix ``ocrmypdf-`` (for the package name on PyPI) and ``ocrmypdf_`` (for the - module), just like pytest plugins. + module), just like pytest plugins. At the same time, please make it clear + that your package is not official. Setuptools plugins ================== @@ -86,6 +87,13 @@ named ``ocrmypdf-exampleplugin``: entry_points={"ocrmypdf": ["exampleplugin = exampleplugin.pluginmodule"]}, ) +.. code-block:: ini + + # equivalent setup.cfg + [options.entry_points] + ocrmypdf = + exampleplugin = exampleplugin.pluginmodule + Plugin requirements =================== diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 2d171340..b03f84be 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -12,19 +12,127 @@ may be unreliable. Use the API to depend on precise behavior. The public API may be useful in scripts that launch OCRmyPDF processes or that wish to use some of its features for working with PDFs. +.. note:: + + Python 3.6 reaches end of life on December 23, 2021. We will end support + for Python 3.6 around that time. The change will be marked with a major + release. + +v12.7.0 +======= + +- Fixed test suite failure when using pikepdf 3.2.0 that was compiled with pybind11 + 2.8.0. :issue:`843` +- Improve advice to user about using ``--max-image-mpixels`` if OCR fails for this + reason. +- Minor documentation fixes. (Thanks to @mara004.) +- Don't require importlib-metadata and importlib-resources backports on versions of + Python where the standard library implementation is sufficient. + (Thanks to Marco Genasci.) + +v12.6.0 +======= + +- Implemented ``--output-type=none`` to skip producing PDFs for applications that + only want sidecar files (:issue:`787`). +- Fixed ambiguities in descriptions of behavior of ``--jbig2-lossy``. +- Various improvements to documentation. + +v12.5.0 +======= + +- Fixed build failure for the combination of PyPy 3.6 and pikepdf 3.0. This + combination can work in a source build but does not work with wheels. +- Accepted bot that wanted to upgrade our deprecated requirements.txt. +- Documentation updates. +- Replace pkg_resources and install dependency on setuptools with + importlib-metadata and importlib-resources. +- Fixed regression in hocrtransform causing text to be omitted when this + renderer was used. +- Fixed some typing errors. + +v12.4.0 +======= + +- When grafting text layers, use pikepdf's ``unparse_content_stream`` if available. +- Confirmed support for pluggy 1.0. (Thanks @QuLogic.) +- Fixed some typing issues, improved pre-commit settings, and fixed issues + flagged by linters. +- PyPy 7.3.3 (=Python 3.6) is now supported. Note that PyPy does not necessarily + run faster, because the vast majority of OCRmyPDF's execution time is spent + running OCR or generally executing native code. However, PyPy may bring speed + improvements in some areas. + +v12.3.3 +======= + +- watcher.py: fixed interpretation of boolean env vars (:issue:`821`). +- Adjust CI scripts to test Tesseract 5 betas. +- Document our support for the Tesseract 5 betas. + +v12.3.2 +======= + +- Indicate support for flask 2.x, watcher 2.x (:issue:`815, 816`). + +v12.3.1 +======= + +- Fixed issue with selection of text when using the hOCR renderer (:issue:`813`). +- Fixed build errors with the Docker image by upgrading to a newer Ubuntu. + Also set the timezone of this image to UTC. + +v12.3.0 +======= + +- Fixed a regression introduced in Pillow 8.3.0. Pillow no longer rounds DPI + for image resolutions. We now account for this (:issue:`802`). +- We no longer use some API calls that are deprecated in the latest versions of + pikepdf. +- Improved error message when a language is requested that doesn't look like a + typical ISO 639-2 code. +- Fixed some tests that attempted to symlink on Windows, breaking tests on a + Windows desktop but not usually on CI. +- Documentation fixes (thanks to @mara004) + +v12.2.0 +======= + +- Fixed invalid Tesseract version number on Windows (:issue:`795`). +- Documentation tweaks. Documentation build now depends on sphinx-issues package. + +v12.1.0 +======= + +- For security reasons we now require Pillow >= 8.2.x. (Older versions will continue + to work if upgrading is not an option.) +- The build system was reorganized to rely on ``setup.cfg`` instead of ``setup.py``. + All changes should work with previously supported versions of setuptools. +- The files in ``requirements/*`` are now considered deprecated but will be retained for v12. + Instead use ``pip install ocrmypdf[test]`` instead of ``requirements/test.txt``, etc. + These files will be removed in v13. + +v12.0.3 +======= + +- Expand the list of languages supported by the hocr PDF renderer. + Several languages were previously considered not supported, particularly those + non-European languages that use the Latin alphabet. +- Fixed a case where the exception stack trace was suppressed in verbose mode. +- Improved documentation around commercial OCR. v12.0.2 ======= -- Fix exception thrown when using ``--remove-background`` on files containing small - images (#769). +- Fixed exception thrown when using ``--remove-background`` on files containing small + images (:issue:`769`). - Improve documentation for description of adding language packs to the Docker image - and corrected name of French language pack. + and corrected name of French language pack. v12.0.1 ======= -- Fix "invalid version number" for untagged tesseract versions (#770). +- Fixed "invalid version number" for untagged tesseract versions (:issue:`770`). v12.0.0 ======= @@ -84,7 +192,7 @@ v12.0.0 which ships with Ubuntu 18.04). - OCRmyPDF can now parse all of Tesseract version numbers, since several schemes have been in use. -- Fixed an issue with parsing PDFs that contain images drawn at a scale of 0. (#761) +- Fixed an issue with parsing PDFs that contain images drawn at a scale of 0. (:issue:`761`) - Removed a frequently repeated message about disabling mmap. v11.7.3 @@ -114,7 +222,7 @@ v11.7.0 ======= - We now support using ``--sidecar`` in conjunction with ``--pages``; these arguments - used to be mutually exclusive. (#735) + used to be mutually exclusive. (:issue:`735`) - Fixed a possible issue with PDF/A-1b generation. Acrobat complained that our PDFs use object streams. More robust PDF/A validators like veraPDF don't consider this a problem, but we'll honor Acrobat's objection from here on. This may increase file @@ -124,13 +232,13 @@ v11.6.2 ======= - Fixed a regression where the wrong page orientation would be produced when using - arguments such as ``--deskew --rotate-pages`` (#730). + arguments such as ``--deskew --rotate-pages`` (:issue:`730`). v11.6.1 ======= - Fixed an issue with attempting optimize unusually narrow-width images by excluding - these images from optimization (#732). + these images from optimization (:issue:`732`). - Remove an obsolete compatibility shim for a version of pikepdf that is no longer supported. @@ -168,7 +276,7 @@ v11.4.5 v11.4.4 ======= -- Fixed ``AttributeError: 'NoneType' object has no attribute 'userunit'``, issue #700, +- Fixed ``AttributeError: 'NoneType' object has no attribute 'userunit'`` (:issue:`700`), related to OCRmyPDF not properly forwarded an error message from pdfminer.six. - Adjusted typing of some arguments. - ``ocrmypdf.ocr`` now takes a ``threading.Lock`` for reasons outlined in the @@ -211,7 +319,7 @@ v11.4.0 ``com.github.ocrmypdf`` to ``ocrmypdf.io``. Scripts that chose to depend on this prefix may need to be adjusted. (This has always been an implementation detail so is not considered part of the semantic versioning "contract".) -- Fixed issue #692, where a particular file with malformed fonts would flood an +- Fixed :issue:`692`, where a particular file with malformed fonts would flood an internal message cue by generating so many debug messages. - Fixed an exception on processing hOCR files with no page record. Tesseract is not known to generate such files. @@ -230,7 +338,7 @@ v11.3.3 ======= - If unpaper outputs non-UTF-8 data, quietly fix this rather than choke on the - conversion. (Possibly addresses #671.) + conversion. (Possibly addresses :issue:`671`.) v11.3.2 ======= @@ -243,7 +351,7 @@ v11.3.2 as optimization candidates. - On some systems, unpaper seems to be unable to process the PNGs we offer it as input. We now convert the input to PNM format, which unpaper always accepts. - Fixes #665 and #667. + Fixes :issue:`665` and :issue:`667`. - DPI sent to unpaper is now rounded to a more reasonable number of decimal digits. - Debug and error messages from unpaper were being suppressed. - Some documentation tweaks. @@ -252,7 +360,7 @@ v11.3.1 ======= - Declare support for new versions: pdfminer.six 20201018 and pikepdf 2.x -- Fix warning related to ``--pdfa-image-compression`` that appears at the wrong +- Fixed warning related to ``--pdfa-image-compression`` that appears at the wrong time. v11.3.0 @@ -272,7 +380,7 @@ v11.3.0 macOS and Windows only where the parent process is not forked. - Fixed the hookspec of rasterize_pdf_page to remove default parameters that were not handled in an expected way by pluggy. -- Fixed another issue with automatic page rotation (#658) due to the issue above. +- Fixed another issue with automatic page rotation (:issue:`658`) due to the issue above. v11.2.1 ======= @@ -294,7 +402,7 @@ v11.1.2 - Fixed hOCR renderer writing the text in roughly reverse order. This should not affect reasonably smart PDF readers that properly locate the position of all text, but may confuse those that rely on the order of objects in the content - stream. (#642) + stream. (:issue:`642`) v11.1.1 ======= @@ -307,9 +415,9 @@ v11.1.1 v11.1.0 ======= -- Fixed page rotation issues: #634, #589. +- Fixed page rotation issues: :issue:`634,589`. - Fixed some cases where optimization created an invalid image such as a - 1-bit "RGB" image: #629, #620. + 1-bit "RGB" image: :issue:`629,620`. - Page numbers are now displayed in debug logs when pages are being grafted. - ocrmypdf.optimize.rewrite_png and ocrmypdf.optimize.rewrite_png_as_g4 were marked deprecated. Strictly speaking these should have been internal APIs, @@ -322,7 +430,7 @@ v11.1.0 v11.0.2 ======= -- Fixed issue #612, TypeError exception. Fixed by eliminating unnecessary repair of +- Fixed :issue:`612`, TypeError exception. Fixed by eliminating unnecessary repair of input PDF metadata in memory. v11.0.1 @@ -339,7 +447,7 @@ v11.0.0 - Project license changed to Mozilla Public License 2.0. Some miscellaneous code is now under MIT license and non-code content/media remains under CC-BY-SA 4.0. License changed with approval of all people who were found - to have contributed to GPLv3 licensed sections of the project. (#600) + to have contributed to GPLv3 licensed sections of the project. (:issue:`600`) - Because the license changed, this is being treated as a major version number change; however, there are no known breaking changes in functional behavior or API compared to v10.x. @@ -348,7 +456,7 @@ v10.3.3 ======= - Fixed a "KeyError: 'dpi'" error message when using ``--threshold`` on an image. - (#607) + (:issue:`607`) v10.3.2 ======= @@ -391,16 +499,16 @@ v10.2.0 ======= - Update Docker image to use Ubuntu 20.04. -- Fixed issue PDF/A acquires title "Untitled" after conversion. (#582) +- Fixed issue PDF/A acquires title "Untitled" after conversion. (:issue:`582`) - Fixed a problem where, when using ``--pdf-renderer hocr``, some text would be missing from the output when using a more recent version of Tesseract. Tesseract began adding more detailed markup about the semantics of text that our HOCR transform did not recognize, so it ignored them. This option is not the default. If necessary ``--redo-ocr`` also redoing OCR to fix such issues. - Fixed an error in Python 3.9 beta, due to removal of deprecated - ``Element.getchildren()``. (#584) + ``Element.getchildren()``. (:issue:`584`) - Implemented support using the API with ``BytesIO`` and other file stream objects. - (#545) + (:issue:`545`) v10.1.1 ======= @@ -493,7 +601,7 @@ v9.8.0 - Fixed issue where only the first PNG (FlateDecode) image in a file would be considered for optimization. File sizes should be improved from here on. -- Fixed a startup crash when the chosen language was Japanese (#543). +- Fixed a startup crash when the chosen language was Japanese (:issue:`543`). - Added options to configure polling and log level to watcher.py. v9.7.2 @@ -540,11 +648,11 @@ v9.6.1 they can be copied out as whole files, and to ensure syntax checking is maintained. -- Fixed an error that caused bash completions to fail on macOS. (#502, #504; +- Fixed an error that caused bash completions to fail on macOS. (:issue:`502,504`; @AlexanderWillner) - Fixed a rare case where OCRmyPDF threw an exception while processing a PDF with the wrong object type in its ``/Trailer /Info``. The error is now logged - and incorrect object is ignored. (#497) + and incorrect object is ignored. (:issue:`497`) - Removed potentially non-free file ``enron1.pdf`` and simplified the test that used it. - Removed potentially non-free file ``misc/media/logo.afdesign``. @@ -746,7 +854,7 @@ v8.3.1 ====== - Fixed an issue where PDFs with malformed metadata would be rendered as - blank pages. `#398 `_. + blank pages. :issue:`398`. v8.3.0 ====== @@ -829,7 +937,7 @@ v8.2.0 designed. However, quality would not be impacted. Lossless JBIG2 was entirely unaffected. - Updated dependencies, including pikepdf to 1.1.0. This fixes - `#358 `__. + :issue:`358`. - The install-time version checks for certain external programs have been removed from setup.py. These tests are now performed at run-time. @@ -850,7 +958,7 @@ v8.1.0 (specifically, those with invalid destination objects) - Fixed an issue when using ``--tesseract-timeout`` and image processing features on a file with more than 100 pages. - `#347 `__ + :issue:`347` - OCRmyPDF now always calls ``os.nice(5)`` to signal to operating systems that it is a background process. @@ -858,7 +966,7 @@ v8.0.1 ====== - Fixed an exception when parsing PDFs that are missing a required - field. `#325 `__ + field. :issue:`325` - pikepdf 1.0.5 is now required, to address some other PDF parsing issues. @@ -879,7 +987,7 @@ older versions of certain dependencies. **Other changes** - Fixed an unhandled exception when attempting to mask barcodes. - `#322 `__ + :issue:`322` - It is now possible to use ocrmypdf without pdfminer.six, to support distributions that do not have it or cannot currently use it (e.g. Homebrew). Downstream maintainers should include pdfminer.six if @@ -906,13 +1014,13 @@ v7.4.0 - chardet >= 3.0.4 is temporarily listed as required. pdfminer.six depends on it, but the most recent release does not specify this requirement. - (`#326 `__) + (:issue:`326`) - python-xmp-toolkit and libexempi are no longer required. - A new Docker image is now being provided for users who wish to access OCRmyPDF over a simple HTTP interface, instead of the command line. - Increase tolerance of PDFs that overflow or underflow the PDF graphics stack. - (`#325 `__) + (:issue:`325`) v7.3.1 ====== @@ -986,7 +1094,7 @@ v7.3.0 v7.2.1 ====== -- Fix compatibility with an API change in pikepdf 0.3.5. +- Fixed compatibility with an API change in pikepdf 0.3.5. - A kludge to support Leptonica versions older than 1.72 in the test suite was dropped. Older versions of Leptonica are likely still compatible. The only impact is that a portion of the test suite will @@ -1023,7 +1131,7 @@ Users who did not install an optional JBIG2 encoder are unaffected. will now attempt to further optimize that image as CCITT or JBIG2, instead of keeping it in the "flate" encoding which is not efficient for 1 bpp images. - (`#297 `__) + (:issue:`297`) - Images in PDFs that are used as soft masks (i.e. transparency masks or alpha channels) are now excluded from optimization. - Fixed handling of Tesseract 4.0-rc1 which now accepts invalid @@ -1035,15 +1143,14 @@ v7.1.0 - Improve the performance of initial text extraction, which is done to determine if a file contains existing text of some kind or not. On large files, this initial processing is now about 20x times faster. - (`#299 `__) + (:issue:`299`) - pikepdf 0.3.3 is now required. -- Fixed issue - `#231 `__, a +- Fixed :issue:`231`, a problem with JPEG2000 images where image metadata was only available inside the JPEG2000 file. - Fixed some additional Ghostscript 9.25 compatibility issues. - Improved handling of KeyboardInterrupt error messages. - (`#301 `__) + (:issue:`301`) - README.md is now served in GitHub markdown instead of reStructuredText. @@ -1075,35 +1182,34 @@ v7.0.5 v7.0.4 ====== -- Fix exception thrown when trying to optimize a certain type of PNG +- Fixed exception thrown when trying to optimize a certain type of PNG embedded in a PDF with the ``-O2`` - Update to pikepdf 0.3.2, to gain support for optimizing some additional image types that were previously excluded from optimization (CMYK and grayscale). Fixes - `#285 `__. + :issue:`285`. v7.0.3 ====== -- Fix issue - `#284 `__, an error +- Fixed :issue:`284`, an error when parsing inline images that have are also image masks, by upgrading pikepdf to 0.3.1 v7.0.2 ====== -- Fix a regression with ``--rotate-pages`` on pages that already had +- Fixed a regression with ``--rotate-pages`` on pages that already had rotations applied. - (`#279 `__) + (:issue:`279`) - Improve quality of page rotation in some cases by rasterizing a higher quality preview image. - (`#281 `__) + (:issue:`281`) v7.0.1 ====== -- Fix compatibility with img2pdf >= 0.3.0 by rejecting input images +- Fixed compatibility with img2pdf >= 0.3.0 by rejecting input images that have an alpha channel - Add forward compatibility for pikepdf 0.3.0 (unrelated to img2pdf) - Various documentation updates for v7.0.0 changes @@ -1206,7 +1312,7 @@ v6.2.4 v6.2.3 ====== -- Fix compatibility with img2pdf >= 0.3.0 by rejecting input images +- Fixed compatibility with img2pdf >= 0.3.0 by rejecting input images that have an alpha channel - This version will be included in Ubuntu 18.10 @@ -1223,9 +1329,8 @@ v6.2.2 v6.2.1 ====== -- Fix recent versions of Tesseract (after 4.0.0-beta1) not being - detected as supporting the ``sandwich`` renderer - (`#271 `__). +- Fixed recent versions of Tesseract (after 4.0.0-beta1) not being + detected as supporting the ``sandwich`` renderer (:issue:`271`). v6.2.0 ====== @@ -1238,21 +1343,19 @@ v6.2.0 - Creation of PDF/A-3 is now supported. However, there is no ability to attach files to PDF/A-3. - Lists more reasons why the file size might grow. -- Fix issue - `#262 `__, +- Fixed :issue:`262`, ``--remove-background`` error on PDFs contained colormapped (paletted) images. -- Fix another XMP metadata validation issue, in cases where the input +- Fixed another XMP metadata validation issue, in cases where the input file's creation date has no timezone and the creation date is not overridden. v6.1.5 ====== -- Fix issue - `#253 `__, a +- Fixed :issue:`253`, a possible division by zero when using the ``hocr`` renderer. -- Fix incorrectly formatted ```` field inside XMP +- Fixed incorrectly formatted ```` field inside XMP metadata for PDF/As. veraPDF flags this as a PDF/A validation failure. The error is caused the timezone and final digit of the seconds of modified time to be omitted, so at worst the modification @@ -1261,7 +1364,7 @@ v6.1.5 v6.1.4 ====== -- Fix issue `#248 `__ +- Fixed :issue:`248` ``--clean`` argument may remove OCR from left column of text on certain documents. We now set ``--layout none`` to suppress this. - The test cache was updated to reflect the change above. @@ -1286,8 +1389,7 @@ Notes v6.1.3 ====== -- Fix issue - `#247 `__, +- Fixed :issue:`247`, ``/CreationDate`` metadata not copied from input to output. - A warning is now issued when Python 3.5 is used on files with a large page count, as this case is known to regress to single core @@ -1297,13 +1399,13 @@ v6.1.2 ====== - Upgrade to PyMuPDF v1.12.5 which includes a more complete fix to - `#239 `__. + :issue:`239`. - Add ``defusedxml`` dependency. v6.1.1 ====== -- Fix text being reported as found on all pages if PyMuPDF is not +- Fixed text being reported as found on all pages if PyMuPDF is not installed. v6.1.0 @@ -1314,15 +1416,15 @@ v6.1.0 PyMuPDF than the author anticipated. (For version 6.x only) install OCRmyPDF with ``pip install ocrmypdf[fitz]`` to use it to its full potential. -- Fix ``FileExistsError`` that could occur if OCR timed out while it +- Fixed ``FileExistsError`` that could occur if OCR timed out while it was generating the output file. - (`#218 `__) -- Fix table of contents/bookmarks all being redirected to page 1 when + (:issue:`218`) +- Fixed table of contents/bookmarks all being redirected to page 1 when generating a PDF/A (with PyMuPDF). (Without PyMuPDF the table of contents is removed in PDF/A mode.) -- Fix "RuntimeError: invalid key in dict" when table of +- Fixed "RuntimeError: invalid key in dict" when table of contents/bookmarks titles contained the character ``)``. - (`#239 `__) + (:issue:`239`) - Added a new argument ``--skip-repair`` to skip the initial PDF repair step if the PDF is already well-formed (because another program repaired it). @@ -1349,35 +1451,29 @@ v6.0.0 - Fixed an issue where OCRmyPDF failed to detect existing text on pages, depending on how the text and fonts were encoded within the - PDF. (`#233 `__, - `#232 `__) + PDF. (:issue:`233,232`) - Fixed an issue that caused dramatic inflation of file sizes when ``--skip-text --output-type pdf`` was used. OCRmyPDF now removes duplicate resources such as fonts, images and other objects that it - generates. - (`#237 `__) + generates. (:issue:`237`) - Improved performance of the initial page splitting step. Originally this step was not believed to be expensive and ran in a process. Large file testing revealed it to be a bottleneck, so it is now parallelized. On a 700 page file with quad core machine, this change - saves about 2 minutes. - (`#234 `__) + saves about 2 minutes. (:issue:`234`) - The test suite now includes a cache that can be used to speed up test runs across platforms. This also does not require computing - checksums, so it's faster. - (`#217 `__) + checksums, so it's faster. (:issue:`217`) v5.7.0 ====== - Fixed an issue that caused poor CPU utilization on machines with more - than 4 cores when running Tesseract 4. (Related to issue - `#217 `__.) + than 4 cores when running Tesseract 4. (Related to :issue:`217`.) - The 'hocr' renderer has been improved. The 'sandwich' and 'tesseract' renderers are still better for most use cases, but 'hocr' may be useful for people who work with the PDF.js renderer in English/ASCII - languages. - (`#225 `__) + languages. (:issue:`225`) - It now formats text in a matter that is easier for certain PDF viewers to select and extract copy and paste text. This should @@ -1405,11 +1501,10 @@ v5.6.2 v5.6.1 ====== -- Fix issue - `#219 `__: change +- Fixed :issue:`219`: change how the final output file is created to avoid triggering permission errors when the output is a special file such as ``/dev/null`` -- Fix test suite failures due to a qpdf 8.0.0 regression and Python +- Fixed test suite failures due to a qpdf 8.0.0 regression and Python 3.5's handling of symlink - The "encrypted PDF" error message was different depending on the type of PDF encryption. Now a single clear message appears for all types @@ -1422,8 +1517,7 @@ v5.6.1 v5.6.0 ====== -- Fix issue - `#216 `__: preserve +- Fixed :issue:`216`: preserve "text as curves" PDFs without rasterizing file - Related to the above, messages about rasterizing are more consistent - For consistency versions minor releases will now get the trailing .0 @@ -1435,34 +1529,32 @@ v5.5 - Add new argument ``--max-image-mpixels``. Pillow 5.0 now raises an exception when images may be decompression bombs. This argument can be used to override the limit Pillow sets. -- Fix output page cropped when using the sandwich renderer and OCR is +- Fixed output page cropped when using the sandwich renderer and OCR is skipped on a rotated and image-processed page - A warning is now issued when old versions of Ghostscript are used in cases known to cause issues with non-Latin characters -- Fix a few parameter validation checks for ``-output-type pdfa-1`` and +- Fixed a few parameter validation checks for ``-output-type pdfa-1`` and ``pdfa-2`` v5.4.4 ====== -- Fix issue - `#181 `__: fix +- Fixed :issue:`181`: fix final merge failure for PDFs with more pages than the system file handle limit (``ulimit -n``) -- Fix issue - `#200 `__: an +- Fixed :issue:`200`: an uncommon syntax for formatting decimal numbers in a PDF would cause qpdf to issue a warning, which ocrmypdf treated as an error. Now this the warning is relayed. -- Fix an issue where intermediate PDFs would be created at version 1.3 +- Fixed an issue where intermediate PDFs would be created at version 1.3 instead of the version of the original file. It's possible but unlikely this had side effects. - A warning is now issued when older versions of qpdf are used since issues like - `#200 `__ cause + :issue:`200` cause qpdf to infinite-loop - Address issue - `#140 `__: if + :issue:`140`: if Tesseract outputs invalid UTF-8, escape it and print its message instead of aborting with a Unicode error - Adding previously unlisted setup requirement, pytest-runner @@ -1528,13 +1620,13 @@ v5.3 forwarded to Tesseract OCR as words and regular expressions respective to use to guide OCR. Supplying a list of subject-domain words should assist Tesseract with resolving words. - (`#165 `__) + (:issue:`165`) - Using a non Latin-1 language with the "hocr" renderer now warns about possible OCR quality and recommends workarounds - (`#176 `__) + (:issue:`176`) - Output file path added to error message when that location is not writable - (`#175 `__) + (:issue:`175`) - Otherwise valid PDFs with leading whitespace at the beginning of the file are now accepted @@ -1562,8 +1654,7 @@ v5.1 v5.0.1 ====== -- Fixed issue - `#169 `__, +- Fixed :issue:`169`, exception due to failure to create sidecar text files on some versions of Tesseract 3.04, including the jbarlow83/ocrmypdf Docker image @@ -1581,19 +1672,17 @@ v5.0 - Add a new feature, ``--sidecar``, which allows creating "sidecar" text files which contain the OCR results in plain text. These OCR text is more reliable than extracting text from PDFs. Closes - `#126 `__. + :issue:`126`. - New feature: ``--pdfa-image-compression``, which allows overriding Ghostscript's lossy-or-lossless image encoding heuristic and making all images JPEG encoded or lossless encoded as desired. Fixes - `#163 `__. + :issue:`163`. -- Fixed issue - `#143 `__, added +- Fixed :issue:`143`, added ``--quiet`` to suppress "INFO" messages -- Fixed issue - `#164 `__, a typo +- Fixed :issue:`164`, a typo - Removed the command line parameters ``-n`` and ``--just-print`` since they have not worked for some time (reported as Ubuntu bug @@ -1602,17 +1691,14 @@ v5.0 v4.5.6 ====== -- Fixed issue - `#156 `__, +- Fixed :issue:`156`, 'NoneType' object has no attribute 'getObject' on pages with no optional /Contents record. This should resolve all issues related to pages with no /Contents record. -- Fixed issue - `#158 `__, ocrmypdf +- Fixed :issue:`158`, ocrmypdf now stops and terminates if Ghostscript fails on an intermediate step, as it is not possible to proceed. -- Fixed issue - `#160 `__, +- Fixed :issue:`160`, exception thrown on certain invalid arguments instead of error message @@ -1620,20 +1706,19 @@ v4.5.5 ====== - Automated update of macOS homebrew tap -- Fixed issue - `#154 `__, KeyError +- Fixed :issue:`154`, KeyError '/Contents' when searching for text on blank pages that have no /Contents record. Note: incomplete fix for this issue. v4.5.4 ====== -- Fix ``--skip-big`` raising an exception if a page contains no images - (`#152 `__) (thanks +- Fixed ``--skip-big`` raising an exception if a page contains no images + (:issue:`152`) (thanks to @TomRaz) -- Fix an issue where pages with no images might trigger "cannot write +- Fixed an issue where pages with no images might trigger "cannot write mode P as JPEG" - (`#151 `__) + (:issue:`151`) v4.5.3 ====== @@ -1652,8 +1737,7 @@ v4.5.3 v4.5.2 ====== -- Fix issue - `#147 `__. +- Fixed :issue:`147`, ``--pdf-renderer tess4 --clean`` will produce an oversized page containing the original image in the bottom left corner, due to loss DPI information. @@ -1663,8 +1747,7 @@ v4.5.2 v4.5.1 ====== -- Fix issue - `#137 `__, +- Fixed :issue:`137`, proportions of images with a non-square pixel aspect ratio would be distorted in output for ``--force-ocr`` and some other combinations of flags @@ -1673,7 +1756,7 @@ v4.5 ==== - PDFs containing "Form XObjects" are now supported (issue - `#134 `__; PDF + :issue:`134`; PDF reference manual 8.10), and images they contain are taken into account when determining the resolution for rasterizing - The Tesseract 4 Docker image no longer includes all languages, @@ -1802,7 +1885,7 @@ v4.2.5 ====== - Fixed an issue - (`#100 `__) with + (:issue:`100`) with PDFs that omit the optional /BitsPerComponent parameter on images - Removed non-free file milk.pdf @@ -1810,7 +1893,7 @@ v4.2.4 ====== - Fixed an error - (`#90 `__) caused by + (:issue:`90`) caused by PDFs that use stencil masks properly - Fixed handling of PDFs that try to draw images or stencil masks without properly setting up the graphics state (such images are now @@ -1848,7 +1931,7 @@ v4.2 - ocrmypdf will now try to convert single image files to PDFs if they are provided as input - (`#15 `__) + (:issue:`15`) - This is a basic convenience feature. It only supports a single image and always makes the image fill the whole page. @@ -1876,11 +1959,11 @@ v4.2 - supports reinterpreting PDFs where text was rendered as curves for printing, and text needs to be recovered - fixes issue - `#82 `__ + :issue:`82` - Fixes an issue where, with certain settings, monochrome images in PDFs would be converted to 8-bit grayscale, increasing file size - (`#79 `__) + (:issue:`79`) - Support for Ubuntu 12.04 LTS "precise" has been dropped in favor of (roughly) Ubuntu 14.04 LTS "trusty" @@ -1908,7 +1991,7 @@ v4.1.3 - More helpful error message for PDFs with version 4 security handler - Update usage instructions for Windows/Docker users -- Fix order of operations for matrix multiplication (no effect on most +- Fixed order of operations for matrix multiplication (no effect on most users) - Add a few leptonica wrapper functions (no effect on most users) @@ -2001,7 +2084,7 @@ New features dominant orientation of detectable text. This feature is fairly reliable but some false positives occur especially if there is not much text to work with. - (`#4 `__) + (:issue:`4`) - Deskewing is now performed using Leptonica instead of unpaper. Leptonica is faster and more reliable at image deskewing than unpaper. @@ -2014,13 +2097,13 @@ Fixes - Fixed an issue where lossless reconstruction could misalign the graphics layer with respect to text layer if the page had been cropped such that its origin is not (0, 0) - (`#49 `__) + (:issue:`49`) Changes - Logging output is now much easier to read - ``--deskew`` is now performed by Leptonica instead of unpaper - (`#25 `__) + (:issue:`25`) - libffi is now required - Some changes were made to the Docker and Travis build environments to support libffi @@ -2035,7 +2118,7 @@ v3.2.1 Changes -- Fixed issue `#47 `__ +- Fixed :issue:`47` "convert() got and unexpected keyword argument 'dpi'" by upgrading to img2pdf 0.2 - Tweaked the Dockerfiles @@ -2080,8 +2163,7 @@ Changes - Python 3.5 and macOS El Capitan are now supported platforms - no changes were needed to implement support - Improved some error messages related to missing input files -- Fixed issue `#20 `__ - - uppercase .PDF extension not accepted +- Fixed :issue:`20`: uppercase .PDF extension not accepted - Fixed an issue where OCRmyPDF failed to text that certain pages contained previously OCR'ed text, such as OCR text produced by Tesseract 3.04 @@ -2158,19 +2240,19 @@ Release candidates^ - rc9: - - fix issue - `#118 `__: + - Fix + :issue:`118`: report error if ghostscript iccprofiles are missing - fixed another issue related to - `#111 `__: PDF + :issue:`111`: PDF rasterized to palette file - add support image files with a palette - don't try to validate PDF file after an exception occurs - rc8: - - fix issue - `#111 `__: + - Fix + :issue:`111`: exception thrown if PDF is missing DocumentInfo dictionary - rc7: diff --git a/misc/batch.py b/misc/batch.py index fcd0e5cf..0b3793b2 100644 --- a/misc/batch.py +++ b/misc/batch.py @@ -52,7 +52,7 @@ logging.basicConfig( ocrmypdf.configure_logging(ocrmypdf.Verbosity.default) -for dir_name, subdirs, file_list in os.walk(start_dir): +for dir_name, _subdirs, file_list in os.walk(start_dir): logging.info(dir_name + '\n') os.chdir(dir_name) for filename in file_list: diff --git a/misc/completion/ocrmypdf.fish b/misc/completion/ocrmypdf.fish index d085acdd..d4bb76d3 100644 --- a/misc/completion/ocrmypdf.fish +++ b/misc/completion/ocrmypdf.fish @@ -54,6 +54,7 @@ function __fish_ocrmypdf_output_type echo -e "pdfa-1\t"(_ "output a PDF/A-1b") echo -e "pdfa-2\t"(_ "output a PDF/A-2b") echo -e "pdfa-3\t"(_ "output a PDF/A-3b") + echo -e "none\t"(_ "do not produce an output PDF (for example, if you only care about --sidecar)") end complete -c ocrmypdf -x -l output-type -a '(__fish_ocrmypdf_output_type)' -d "select PDF output options" diff --git a/misc/synology.py b/misc/synology.py index 6e294ce1..7e243229 100644 --- a/misc/synology.py +++ b/misc/synology.py @@ -46,7 +46,7 @@ if len(sys.argv) > 1: else: start_dir = '.' -for dir_name, subdirs, file_list in os.walk(start_dir): +for dir_name, _subdirs, file_list in os.walk(start_dir): logging.info(dir_name) os.chdir(dir_name) for filename in file_list: diff --git a/misc/watcher.py b/misc/watcher.py index 68437878..193b485a 100644 --- a/misc/watcher.py +++ b/misc/watcher.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Copyright (C) 2019 Ian Alexander: https://github.com/ianalexander # Copyright (C) 2020 James R Barlow: https://github.com/jbarlow83 # @@ -36,14 +37,19 @@ import ocrmypdf # pylint: disable=logging-format-interpolation + +def getenv_bool(name: str, default: str = 'False'): + return os.getenv(name, default).lower() in ('true', 'yes', 'y', '1') + + INPUT_DIRECTORY = os.getenv('OCR_INPUT_DIRECTORY', '/input') OUTPUT_DIRECTORY = os.getenv('OCR_OUTPUT_DIRECTORY', '/output') -OUTPUT_DIRECTORY_YEAR_MONTH = bool(os.getenv('OCR_OUTPUT_DIRECTORY_YEAR_MONTH', '')) -ON_SUCCESS_DELETE = bool(os.getenv('OCR_ON_SUCCESS_DELETE', '')) -DESKEW = bool(os.getenv('OCR_DESKEW', '')) +OUTPUT_DIRECTORY_YEAR_MONTH = getenv_bool('OCR_OUTPUT_DIRECTORY_YEAR_MONTH') +ON_SUCCESS_DELETE = getenv_bool('OCR_ON_SUCCESS_DELETE') +DESKEW = getenv_bool('OCR_DESKEW') OCR_JSON_SETTINGS = json.loads(os.getenv('OCR_JSON_SETTINGS', '{}')) POLL_NEW_FILE_SECONDS = int(os.getenv('OCR_POLL_NEW_FILE_SECONDS', '1')) -USE_POLLING = bool(os.getenv('OCR_USE_POLLING', '')) +USE_POLLING = getenv_bool('OCR_USE_POLLING') LOGLEVEL = os.getenv('OCR_LOGLEVEL', 'INFO') PATTERNS = ['*.pdf', '*.PDF'] diff --git a/misc/webservice.py b/misc/webservice.py index ed2a0374..e005b7c2 100644 --- a/misc/webservice.py +++ b/misc/webservice.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # webservice.py wrapper for OCRmyPDF # Copyright (C) 2019 James R. Barlow: github.com/jbarlow83 # @@ -28,16 +29,7 @@ import shlex from subprocess import PIPE, run from tempfile import TemporaryDirectory -from flask import ( - Flask, - Response, - abort, - flash, - redirect, - request, - send_from_directory, - url_for, -) +from flask import Flask, Response, request, send_from_directory from werkzeug.utils import secure_filename app = Flask(__name__) @@ -45,7 +37,7 @@ app.secret_key = "secret" app.config['MAX_CONTENT_LENGTH'] = 50_000_000 app.config.from_envvar("OCRMYPDF_WEBSERVICE_SETTINGS", silent=True) -ALLOWED_EXTENSIONS = set(["pdf"]) +ALLOWED_EXTENSIONS = {"pdf"} def allowed_file(filename): diff --git a/pyproject.toml b/pyproject.toml index a28f55c0..49d9b760 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,11 +3,14 @@ requires = [ "setuptools >= 30.3.0", "wheel", "cffi", - "setuptools_scm", + "setuptools_scm[toml] >= 3.4", "setuptools_scm_git_archive" ] build-backend = "setuptools.build_meta" +[tool.setuptools_scm] +version_scheme = "post-release" + [tool.black] line-length = 88 target-version = ["py36", "py37", "py38"] @@ -31,3 +34,49 @@ exclude = ''' | src/ocrmypdf/lib/_leptonica.py )/ ''' + +[tool.coverage.run] +branch = true +parallel = true +concurrency = ["multiprocessing"] + +[tool.coverage.paths] +source = ["src/ocrmypdf"] + +[tool.coverage.report] +# Regexes for lines to exclude from consideration +exclude_lines = [ + # Have to re-enable the standard pragma + "pragma: no cover", + + # Don't complain if tests don't hit defensive assertion code: + "raise AssertionError", + "raise NotImplementedError", + + # Don't complain if non-runnable code isn't run: + "if 0:", + "if False:", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:" +] + +[tool.isort] +profile = "black" +known_first_party = "ocrmypdf" +known_third_party = ["PIL", "_cffi_backend", "cffi", "flask", "img2pdf", "ocrmypdf", "pdfminer", "pikepdf", "pkg_resources", "pluggy", "pytest", "reportlab", "setuptools", "sphinx_rtd_theme", "tqdm", "watchdog", "werkzeug"] + +[tool.pytest.ini_options] +minversion = "6.0" +norecursedirs = ["lib", ".pc", ".git", "venv", "output", "cache", "resources"] +testpaths = ["tests"] +addopts = "-n auto" +markers = ["slow"] +filterwarnings = ["ignore:.*XMLParser.*:DeprecationWarning"] + +[tool.mypy] + +[[tool.mypy.overrides]] +module = [ + 'pluggy', 'tqdm', 'coloredlogs', 'img2pdf', 'cffi', '_cffi_backend', 'pdfminer.*', 'reportlab.*' +] +ignore_missing_imports = true diff --git a/requirements/main.txt b/requirements/main.txt index a49cf9fa..f4a7865e 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1,12 +1,10 @@ -# requirements.txt can be used to replicate the developer's build environment -# setup.py lists a separate set of requirements that are looser to simplify -# installation +# Deprecated and not maintained; use "pip install ocrmypdf" instead cffi == 1.14.5 coloredlogs == 15.0 # technically optional img2pdf == 0.4.0 pdfminer.six == 20201018 pikepdf == 2.10.0 pluggy == 0.13.1 -Pillow == 8.1.2 +Pillow == 8.3.2 reportlab == 3.5.66 tqdm == 4.59.0 diff --git a/requirements/test.txt b/requirements/test.txt index 72225e2d..db7cb14e 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,3 +1,4 @@ +# Deprecated and not maintained; use "pip install ocrmypdf[test]" instead pytest >= 6.0.0 pytest-xdist >= 2.2.0 pytest-cov >= 2.11.1 diff --git a/requirements/watcher.txt b/requirements/watcher.txt index 660d7af4..b4a2b744 100644 --- a/requirements/watcher.txt +++ b/requirements/watcher.txt @@ -1 +1,2 @@ +# Deprecated and not maintained; use "pip install ocrmypdf[watcher]" instead watchdog == 1.0.2 diff --git a/requirements/webservice.txt b/requirements/webservice.txt index f6e3c4e6..7f8e08de 100644 --- a/requirements/webservice.txt +++ b/requirements/webservice.txt @@ -1 +1,2 @@ +# Deprecated and not maintained; use "pip install ocrmypdf[webservice]" instead Flask >= 1, < 2 diff --git a/setup.cfg b/setup.cfg index 36545d66..b5aa691e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,56 +1,115 @@ +[metadata] +name = ocrmypdf +description = OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/jbarlow83/OCRmyPDF +author = James R. Barlow +author_email = james@purplerock.ca +license = MPL-2.0 +license_file = LICENSE +license_files = + LICENSE +classifiers = + Development Status :: 5 - Production/Stable + Environment :: Console + Intended Audience :: End Users/Desktop + Intended Audience :: Science/Research + Intended Audience :: System Administrators + License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0) + Operating System :: MacOS :: MacOS X + Operating System :: Microsoft :: Windows :: Windows 10 + Operating System :: POSIX + Operating System :: POSIX :: BSD + Operating System :: POSIX :: Linux + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Topic :: Scientific/Engineering :: Image Recognition + Topic :: Text Processing :: Indexing + Topic :: Text Processing :: Linguistic +keywords = + PDF + OCR + optical character recognition + PDF/A + scanning +project_urls = + Documentation = https://ocrmypdf.readthedocs.io/ + Source = https://github.com/jbarlow83/ocrmypdf + Tracker = https://github.com/jbarlow83/ocrmypdf/issues + +[options] +packages = find: +install_requires = + Pillow>=8.2.0 + cffi>=1.9.1 # must be a setup and install requirement + coloredlogs>=14.0 # strictly optional + img2pdf>=0.3.0,<0.5 # pure Python + importlib-metadata>=4;python_version<'3.8' # until Python 3.8 + importlib-resources>=5;python_version<'3.9' # until Python 3.9 + pdfminer.six!=20200720,>=20191110,<=20201018 + pikepdf>=2.10.0 + pikepdf<3;implementation_name=="pypy" and python_version=='3.6' + pluggy>=0.13.0,<2 + reportlab>=3.5.66 + tqdm>=4 +python_requires = >=3.6 +include_package_data = True +package_dir = + =src +platforms = any +setup_requires = + cffi>=1.9.1 # to build the leptonica module + setuptools_scm + setuptools_scm_git_archive +zip_safe = False + +[options.packages.find] +where = src + +[options.entry_points] +console_scripts = + ocrmypdf = ocrmypdf.__main__:run + +[options.extras_require] +docs = + sphinx + sphinx-issues + sphinx-rtd-theme +extended_test = + PyMuPDF==1.13.4 +test = + coverage[toml]>=5 + pytest>=6.0.0 + pytest-cov>=2.11.1 + pytest-xdist>=2.2.0 + python-xmp-toolkit==2.0.1 # also requires apt-get install libexempi3 +watcher = + watchdog>=1.0.2,<3 +webservice = + Flask>=1,<3 + +[options.package_data] +ocrmypdf = + data/sRGB.icc + py.typed + [bdist_wheel] python-tag = py36 [aliases] -test=pytest +test = pytest [check-manifest] ignore = - .github + .github -[tool:pytest] -norecursedirs = lib .pc .git output cache resources -testpaths = tests -filterwarnings = - ignore:.*XMLParser.*:DeprecationWarning -markers = - slow -addopts = - -n auto - -[isort] -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -line_length=88 -known_first_party = ocrmypdf -known_third_party = PIL,_cffi_backend,cffi,flask,img2pdf,pdfminer,pikepdf,pkg_resources,pluggy,pytest,reportlab,setuptools,sphinx_rtd_theme,tqdm,watchdog,werkzeug - -[metadata] -license_file = LICENSE - -[coverage:paths] -source = - src/ocrmypdf - -[coverage:run] -branch = true -parallel = true -concurrency = multiprocessing - -[coverage:report] -# Regexes for lines to exclude from consideration -exclude_lines = - # Have to re-enable the standard pragma - pragma: no cover - - # Don't complain if tests don't hit defensive assertion code: - raise AssertionError - raise NotImplementedError - - # Don't complain if non-runnable code isn't run: - if 0: - if False: - if __name__ == .__main__.: - if TYPE_CHECKING: +[flake8] +ignore = D203,F401,W503,E501,E203,F841 +exclude = .git,__pycache__,docs/conf.py,build,dist,.venv,.venvpp,.eggs,tmp,src/ocrmypdf/lib/ +max-complexity = 10 +max-line-length = 100 diff --git a/setup.py b/setup.py index e2342651..a3dd6458 100644 --- a/setup.py +++ b/setup.py @@ -1,62 +1,14 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# © 2015 James R. Barlow: github.com/jbarlow83 +# © 2021 James R. Barlow: github.com/jbarlow83 # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import print_function, unicode_literals - -import sys - -from setuptools import find_packages, setup - -if sys.version_info < (3, 6): - print("Python 3.6 or newer is required", file=sys.stderr) - sys.exit(1) - -tests_require = open('requirements/test.txt', encoding='utf-8').read().splitlines() - - -def readme(): - with open('README.md', encoding='utf-8') as f: - return f.read() - +from setuptools import setup +# Minimal setup to support older setuptools/setuptools_scm setup( - name='ocrmypdf', - description='OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched', - long_description=readme(), - long_description_content_type='text/markdown', - url='https://github.com/jbarlow83/OCRmyPDF', - author='James R. Barlow', - author_email='james@purplerock.ca', - packages=find_packages('src', exclude=["tests", "tests.*"]), - package_dir={'': 'src'}, - keywords=['PDF', 'OCR', 'optical character recognition', 'PDF/A', 'scanning'], - classifiers=[ - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: End Users/Desktop", - "Intended Audience :: Science/Research", - "Intended Audience :: System Administrators", - "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", - "Operating System :: MacOS :: MacOS X", - "Operating System :: Microsoft :: Windows :: Windows 10", - "Operating System :: POSIX", - "Operating System :: POSIX :: BSD", - "Operating System :: POSIX :: Linux", - "Topic :: Scientific/Engineering :: Image Recognition", - "Topic :: Text Processing :: Indexing", - "Topic :: Text Processing :: Linguistic", - ], - python_requires=' >= 3.6', setup_requires=[ # can be removed whenever we can drop pip 9 support 'cffi >= 1.9.1', # to build the leptonica module 'setuptools_scm', # so that version will work @@ -64,26 +16,4 @@ setup( ], use_scm_version={'version_scheme': 'post-release'}, cffi_modules=['src/ocrmypdf/lib/compile_leptonica.py:ffibuilder'], - install_requires=[ - 'cffi >= 1.9.1', # must be a setup and install requirement - 'coloredlogs >= 14.0', # strictly optional - 'img2pdf >= 0.3.0, < 0.5', # pure Python, so track HEAD closely - 'pdfminer.six >= 20191110, != 20200720, <= 20201018', - "pikepdf >= 2.10.0", - 'Pillow >= 8.1.2', - 'pluggy >= 0.13.0, < 1.0', - 'reportlab >= 3.5.66', - 'setuptools', - 'tqdm >= 4', - ], - tests_require=tests_require, - entry_points={'console_scripts': ['ocrmypdf = ocrmypdf.__main__:run']}, - package_data={'ocrmypdf': ['data/sRGB.icc', 'py.typed']}, - include_package_data=True, - zip_safe=False, - project_urls={ - 'Documentation': 'https://ocrmypdf.readthedocs.io/', - 'Source': 'https://github.com/jbarlow83/ocrmypdf', - 'Tracker': 'https://github.com/jbarlow83/ocrmypdf/issues', - }, ) diff --git a/src/ocrmypdf/RELEASE.md b/src/ocrmypdf/RELEASE.md index 41a40e97..2cb89b78 100644 --- a/src/ocrmypdf/RELEASE.md +++ b/src/ocrmypdf/RELEASE.md @@ -28,7 +28,7 @@ - Search for deprecation: search all files for deprec*, etc. -- Check requirements/* +- Check requirements in setup.cfg - Delete `tests/cache`, do `pytest --runslow`, and update cache. diff --git a/src/ocrmypdf/_exec/tesseract.py b/src/ocrmypdf/_exec/tesseract.py index 119771ec..79374670 100644 --- a/src/ocrmypdf/_exec/tesseract.py +++ b/src/ocrmypdf/_exec/tesseract.py @@ -56,11 +56,12 @@ class TesseractLoggerAdapter(logging.LoggerAdapter): class TesseractVersion(StrictVersion): + version_re = re.compile( r''' ^(\d+) \. (\d+) (\. (\d+))? # groups: 1/major, 2/minor, 3/[skip], 4/patch [-]? # optional hyphen separator - (?:(alpha|beta|rc|dev)[.\-\ ]?(\d+)?)? # 5/prerelease, 6/prerelease_num + (?:(alpha|beta|rc|dev)?[.\-\ ]?(\d+)?)? # 5/prerelease, 6/prerelease_num (?:-(\d+)-g[0-9a-f]+)? # untagged git version $ ''', @@ -116,7 +117,7 @@ def get_languages(): if line.startswith('Error'): raise MissingDependencyError(lang_error(output)) _header, *rest = output.splitlines() - return set(lang.strip() for lang in rest) + return {lang.strip() for lang in rest} def tess_base_args(langs: List[str], engine_mode: Optional[int]) -> List[str]: @@ -249,7 +250,8 @@ def generate_hocr( # Reminder: test suite tesseract test plugins will break after any changes # to the number of order parameters here - args_tesseract.extend([input_file, prefix, 'hocr', 'txt'] + tessconfig) + args_tesseract.extend([os.fspath(input_file), os.fspath(prefix), 'hocr', 'txt']) + args_tesseract.extend(tessconfig) try: p = run(args_tesseract, stdout=PIPE, stderr=STDOUT, timeout=timeout, check=True) stdout = p.stdout @@ -323,7 +325,8 @@ def generate_pdf( # Reminder: test suite tesseract test plugins might break after any changes # to the number of order parameters here - args_tesseract.extend([input_file, prefix, 'pdf', 'txt'] + tessconfig) + args_tesseract.extend([os.fspath(input_file), os.fspath(prefix), 'pdf', 'txt']) + args_tesseract.extend(tessconfig) try: p = run(args_tesseract, stdout=PIPE, stderr=STDOUT, timeout=timeout, check=True) stdout = p.stdout diff --git a/src/ocrmypdf/_exec/unpaper.py b/src/ocrmypdf/_exec/unpaper.py index 5226326d..aec365c2 100644 --- a/src/ocrmypdf/_exec/unpaper.py +++ b/src/ocrmypdf/_exec/unpaper.py @@ -45,7 +45,7 @@ def _setup_unpaper_io(tmpdir: Path, input_file: Path) -> Tuple[Path, Path]: im = im.convert(mode='1') else: im = im.convert(mode='RGB') - except IOError as e: + except OSError as e: raise MissingDependencyError( "Could not convert image with type " + im.mode ) from e @@ -96,12 +96,12 @@ def run( try: with Image.open(output_pnm) as imout: imout.save(output_file, dpi=(dpi, dpi)) - except (FileNotFoundError, OSError): + except OSError as e: raise SubprocessOutputError( "unpaper: failed to produce the expected output file. " + " Called with: " + str(args_unpaper) - ) from None + ) from e def validate_custom_args(args: str) -> List[str]: diff --git a/src/ocrmypdf/_graft.py b/src/ocrmypdf/_graft.py index ba4b050f..a59513ab 100644 --- a/src/ocrmypdf/_graft.py +++ b/src/ocrmypdf/_graft.py @@ -11,8 +11,19 @@ from contextlib import suppress from pathlib import Path from typing import Optional -import pikepdf -from pikepdf.objects import Dictionary, Name +from pikepdf import ( + Dictionary, + Name, + Object, + Operator, + Page, + Pdf, + PdfError, + PdfMatrix, + Stream, + parse_content_stream, + unparse_content_stream, +) log = logging.getLogger(__name__) MAX_REPLACE_PAGES = 100 @@ -47,43 +58,28 @@ def strip_invisible_text(pdf, page): render_mode = 0 text_objects = [] - page.page_contents_coalesce() - for operands, operator in pikepdf.parse_content_stream(page, ''): + rich_page = Page(page) + rich_page.contents_coalesce() + for operands, operator in parse_content_stream(page, ''): if not in_text_obj: - if operator == pikepdf.Operator('BT'): + if operator == Operator('BT'): in_text_obj = True render_mode = 0 text_objects.append((operands, operator)) else: stream.append((operands, operator)) else: - if operator == pikepdf.Operator('Tr'): + if operator == Operator('Tr'): render_mode = operands[0] text_objects.append((operands, operator)) - if operator == pikepdf.Operator('ET'): + if operator == Operator('ET'): in_text_obj = False if render_mode != 3: stream.extend(text_objects) text_objects.clear() - def convert(op): - try: - return op.unparse() - except AttributeError: - return str(op).encode('ascii') - - lines = [] - - for operands, operator in stream: - if operator == pikepdf.Operator('INLINE IMAGE'): - iim = operands[0] - line = iim.unparse() - else: - line = b' '.join(convert(op) for op in operands) + b' ' + operator.unparse() - lines.append(line) - - content_stream = b'\n'.join(lines) - page.Contents = pikepdf.Stream(pdf, content_stream) + content_stream = unparse_content_stream(stream) + page.Contents = Stream(pdf, content_stream) class OcrGrafter: @@ -91,14 +87,14 @@ class OcrGrafter: self.context = context self.path_base = context.origin - self.pdf_base = pikepdf.open(self.path_base) + self.pdf_base = Pdf.open(self.path_base) self.font, self.font_key = None, None self.pdfinfo = context.pdfinfo self.output_file = context.get_path('graft_layers.pdf') self.procset = self.pdf_base.make_indirect( - pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]') + Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]') ) self.emplacements = 1 @@ -122,7 +118,7 @@ class OcrGrafter: # We are updating the old page with a rasterized PDF of the new # page (without changing objgen, to preserve references) log.debug("Emplacement update") - with pikepdf.open(image) as pdf_image: + with Pdf.open(path_image) as pdf_image: self.emplacements += 1 foreign_image_page = pdf_image.pages[0] self.pdf_base.pages.append(foreign_image_page) @@ -195,7 +191,7 @@ class OcrGrafter: self.pdf_base.save(next_file) self.pdf_base.close() - self.pdf_base = pikepdf.open(next_file) + self.pdf_base = Pdf.open(next_file) self.procset = self.pdf_base.pages[0].Resources.ProcSet self.font, self.font_key = None, None # Ensure we reacquire this information self.interim_count += 1 @@ -211,7 +207,7 @@ class OcrGrafter: font, font_key = None, None possible_font_names = ('/f-0-0', '/F1') try: - with pikepdf.open(text) as pdf_text: + with Pdf.open(text) as pdf_text: try: pdf_text_fonts = pdf_text.pages[0].Resources.get('/Font', {}) except (AttributeError, IndexError, KeyError): @@ -225,7 +221,7 @@ class OcrGrafter: if pdf_text_font: font = self.pdf_base.copy_foreign(pdf_text_font) return font, font_key - except (FileNotFoundError, pikepdf.PdfError): + except (FileNotFoundError, PdfError): # PdfError occurs if a 0-length file is written e.g. due to OCR timeout return None, None @@ -234,9 +230,9 @@ class OcrGrafter: *, page_num: int, textpdf: Path, - font: pikepdf.Object, - font_key: pikepdf.Object, - procset: pikepdf.Object, + font: Object, + font_key: Object, + procset: Object, text_rotation: int, strip_old_text: bool, ): @@ -247,7 +243,7 @@ class OcrGrafter: return # This is a pointer indicating a specific page in the base file - with pikepdf.open(textpdf) as pdf_text: + with Pdf.open(textpdf) as pdf_text: pdf_text_contents = pdf_text.pages[0].Contents.read_bytes() base_page = self.pdf_base.pages.p(page_num) @@ -262,13 +258,13 @@ class OcrGrafter: mediabox = [float(base_page.MediaBox[v]) for v in range(4)] wp, hp = mediabox[2] - mediabox[0], mediabox[3] - mediabox[1] - translate = pikepdf.PdfMatrix().translated(-wt / 2, -ht / 2) - untranslate = pikepdf.PdfMatrix().translated(wp / 2, hp / 2) - corner = pikepdf.PdfMatrix().translated(mediabox[0], mediabox[1]) + translate = PdfMatrix().translated(-wt / 2, -ht / 2) + untranslate = PdfMatrix().translated(wp / 2, hp / 2) + corner = PdfMatrix().translated(mediabox[0], mediabox[1]) # -rotation because the input is a clockwise angle and this formula # uses CCW text_rotation = -text_rotation % 360 - rotate = pikepdf.PdfMatrix().rotated(text_rotation) + rotate = PdfMatrix().rotated(text_rotation) # Because of rounding of DPI, we might get a text layer that is not # identically sized to the target page. Scale to adjust. Normally this @@ -279,7 +275,7 @@ class OcrGrafter: scale_y = hp / ht # log.debug('%r', scale_x, scale_y) - scale = pikepdf.PdfMatrix().scaled(scale_x, scale_y) + scale = PdfMatrix().scaled(scale_x, scale_y) # Translate the text so it is centered at (0, 0), rotate it there, adjust # for a size different between initial and text PDF, then untranslate, and @@ -302,12 +298,19 @@ class OcrGrafter: pdf_draw_xobj = ( (b'q %s cm\n' % ctm.encode()) + (b'%s Do\n' % text_xobj_name) + b'\nQ\n' ) - new_text_layer = pikepdf.Stream(self.pdf_base, pdf_draw_xobj) + new_text_layer = Stream(self.pdf_base, pdf_draw_xobj) if strip_old_text: strip_invisible_text(self.pdf_base, base_page) - base_page.page_contents_add(new_text_layer, prepend=True) + if hasattr(Page, 'contents_add'): + # pikepdf >= 2.14 adds this method and deprecates the one below + Page(base_page).contents_add(new_text_layer, prepend=True) + else: + # pikepdf < 2.14 + base_page.page_contents_add( + new_text_layer, prepend=True + ) # pragma: no cover _update_resources( obj=base_page, font=font, font_key=font_key, procset=procset diff --git a/src/ocrmypdf/_pipeline.py b/src/ocrmypdf/_pipeline.py index ce3b8d88..be1b935d 100644 --- a/src/ocrmypdf/_pipeline.py +++ b/src/ocrmypdf/_pipeline.py @@ -48,7 +48,7 @@ def triage_image_file(input_file, output_file, options): log.info("Input file is not a PDF, checking if it is an image...") try: im = Image.open(input_file) - except EnvironmentError as e: + except OSError as e: # Recover the original filename log.error(str(e).replace(str(input_file), str(options.input_file))) raise UnsupportedImageFormatError() from e @@ -135,7 +135,7 @@ def triage(original_filename, input_file, output_file, options): # Origin file is a pdf create a symlink with pdf extension safe_symlink(input_file, output_file) return output_file - except EnvironmentError as e: + except OSError as e: log.debug(f"Temporary file was at: {input_file}") msg = str(e).replace(str(input_file), original_filename) raise InputFileError(msg) from e @@ -521,13 +521,12 @@ def create_ocr_image(image: Path, page_context: PageContext): # be None) bbox = [float(v) for v in textarea] xyscale = tuple(float(coord) / 72.0 for coord in im.info['dpi']) - pixcoords = [ + pixcoords = ( bbox[0] * xyscale[0], im.height - bbox[3] * xyscale[1], bbox[2] * xyscale[0], im.height - bbox[1] * xyscale[1], - ] - pixcoords = [int(round(c)) for c in pixcoords] + ) log.debug('blanking %r', pixcoords) draw.rectangle(pixcoords, fill=white) # draw.rectangle(pixcoords, outline=pink) @@ -856,7 +855,7 @@ def merge_sidecars(txt_files: Iterable[Optional[Path]], context: PdfContext): if frm != 1: stream.write('\f') # Form feed between pages if txt_file: - with open(txt_file, 'r', encoding="utf-8") as in_: + with open(txt_file, encoding="utf-8") as in_: txt = in_.read() # Some OCR engines (e.g. Tesseract v4 alpha) add form feeds # between pages, and some do not. For consistency, we ignore diff --git a/src/ocrmypdf/_sync.py b/src/ocrmypdf/_sync.py index d9432cf4..ea76dabb 100644 --- a/src/ocrmypdf/_sync.py +++ b/src/ocrmypdf/_sync.py @@ -290,15 +290,16 @@ def exec_concurrent(context: PdfContext, executor: Executor): # Copy text file to destination copy_final(text, options.sidecar, context) - # Merge layers to one single pdf - pdf = ocrgraft.finalize() + if options.output_type != 'none': + # Merge layers to one single pdf + pdf = ocrgraft.finalize() - # PDF/A and metadata - log.info("Postprocessing...") - pdf = post_process(pdf, context, executor) + # PDF/A and metadata + log.info("Postprocessing...") + pdf = post_process(pdf, context, executor) - # Copy PDF file to destination - copy_final(pdf, options.output_file, context) + # Copy PDF file to destination + copy_final(pdf, options.output_file, context) def configure_debug_logging(log_filename: Path, prefix: str = ''): @@ -399,19 +400,28 @@ def run_pipeline(options, *, plugin_manager, api=False): return ExitCode.invalid_output_pdf report_output_file_size(options, start_input_file, options.output_file) - except (KeyboardInterrupt if not api else NeverRaise) as e: + except (KeyboardInterrupt if not api else NeverRaise): if options.verbose >= 1: log.exception("KeyboardInterrupt") else: log.error("KeyboardInterrupt") return ExitCode.ctrl_c except (ExitCodeException if not api else NeverRaise) as e: - if str(e): + if options.verbose >= 1: + log.exception("ExitCodeException") + elif str(e): log.error("%s: %s", type(e).__name__, str(e)) else: log.error(type(e).__name__) return e.exit_code - except (Exception if not api else NeverRaise) as e: # pylint: disable=broad-except + except (PIL.Image.DecompressionBombError if not api else NeverRaise) as e: + log.exception( + "A decompression bomb error was encountered while executing the " + "pipeline. Use the argument --max-image-mpixels to raise the maximum " + "image pixel limit." + ) + return ExitCode.other_error + except (Exception if not api else NeverRaise): # pylint: disable=broad-except log.exception("An exception occurred while executing the pipeline") return ExitCode.other_error finally: @@ -419,7 +429,7 @@ def run_pipeline(options, *, plugin_manager, api=False): try: debug_log_handler.close() log.removeHandler(debug_log_handler) - except EnvironmentError as e: + except OSError as e: print(e, file=sys.stderr) cleanup_working_files(work_folder, options) diff --git a/src/ocrmypdf/_validation.py b/src/ocrmypdf/_validation.py index ab48195e..15c45226 100644 --- a/src/ocrmypdf/_validation.py +++ b/src/ocrmypdf/_validation.py @@ -13,7 +13,7 @@ import sys import unicodedata from pathlib import Path from shutil import copyfileobj -from typing import List, Set, Tuple, Union +from typing import List, Set, Tuple import pikepdf import PIL @@ -26,18 +26,13 @@ from ocrmypdf.exceptions import ( MissingDependencyError, OutputFileAccessError, ) -from ocrmypdf.helpers import ( - is_file_writable, - is_iterable_notstr, - monotonic, - safe_symlink, -) +from ocrmypdf.helpers import is_file_writable, monotonic, safe_symlink, samefile +from ocrmypdf.hocrtransform import HOCR_OK_LANGS from ocrmypdf.subprocess import check_external_program # ------------- # External dependencies -HOCR_OK_LANGS = frozenset(['eng', 'deu', 'spa', 'ita', 'por']) DEFAULT_LANGUAGE = 'eng' # Enforce English hegemony log = logging.getLogger(__name__) @@ -65,13 +60,14 @@ def check_options_languages(options, ocr_engine_languages): log.debug("No language specified; assuming --language %s", DEFAULT_LANGUAGE) if not ocr_engine_languages: return - if not options.languages.issubset(ocr_engine_languages): + missing_languages = options.languages - ocr_engine_languages + if missing_languages: msg = ( - f"OCR engine does not have language data for the following " + "OCR engine does not have language data for the following " "requested languages: \n" ) - for lang in options.languages - ocr_engine_languages: - msg += lang + '\n' + msg += '\n'.join(lang for lang in missing_languages) + msg += '\nNote: most languages are identified by a 3-digit ISO 639-2 Code' raise MissingDependencyError(msg) @@ -79,12 +75,18 @@ def check_options_output(options): is_latin = options.languages.issubset(HOCR_OK_LANGS) if options.pdf_renderer.startswith('hocr') and not is_latin: - msg = ( + log.warning( "The 'hocr' PDF renderer is known to cause problems with one " "or more of the languages in your document. Use " - "--pdf-renderer auto (the default) to avoid this issue." + "`--pdf-renderer auto` (the default) to avoid this issue." + ) + + if options.output_type == 'none' and options.output_file != os.devnull: + raise BadArgsError( + "Since you specified `--pdf-renderer none`, the output file " + f"{options.output_file} cannot be produced. Set the output file to " + f"{os.devnull} to suppress this message." ) - log.warning(msg) lossless_reconstruction = False if not any( @@ -111,6 +113,10 @@ def check_options_sidecar(options): raise BadArgsError( "--sidecar filename must be specified when output file is stdout." ) + elif options.output_file == os.devnull: + raise BadArgsError( + "--sidecar filename must be specified when output file is /dev/null or NUL." + ) options.sidecar = options.output_file + '.txt' if options.sidecar == options.input_file or options.sidecar == options.output_file: raise BadArgsError( @@ -141,8 +147,6 @@ def check_options_preprocessing(options): def _pages_from_ranges(ranges: str) -> Set[int]: - if is_iterable_notstr(ranges): - return set(ranges) pages: List[int] = [] page_groups = ranges.replace(' ', '').split(',') for g in page_groups: @@ -181,10 +185,8 @@ def _pages_from_ranges(ranges: str) -> Set[int]: def check_options_ocr_behavior(options): exclusive_options = sum( - [ - (1 if opt else 0) - for opt in (options.force_ocr, options.skip_text, options.redo_ocr) - ] + (1 if opt else 0) + for opt in (options.force_ocr, options.skip_text, options.redo_ocr) ) if exclusive_options >= 2: raise BadArgsError("Choose only one of --force-ocr, --skip-text, --redo-ocr.") @@ -279,7 +281,7 @@ def check_closed_streams(options): # pragma: no cover Attempting to a fork/exec a new Python process when any of std{in,out,err} are closed or not flushable for some reason may raise an exception. Fix this by opening devnull if the handle seems to be closed. Do this - globally to avoid tracking places all places that fork. + globally to avoid tracking all places that fork. Seems to be specific to multiprocessing.Process not all Python process forkers. @@ -301,7 +303,7 @@ def check_closed_streams(options): # pragma: no cover if options.input_file == '-': log.error("Trying to read from stdin but stdin seems closed") return False - sys.stdin = open(os.devnull, 'r') + sys.stdin = open(os.devnull) if sys.stdout is None: if options.output_file == '-': diff --git a/src/ocrmypdf/_version.py b/src/ocrmypdf/_version.py index 6751fede..3cf5ff8b 100644 --- a/src/ocrmypdf/_version.py +++ b/src/ocrmypdf/_version.py @@ -5,9 +5,12 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. -import pkg_resources +try: + from importlib_metadata import version as _package_version +except ImportError: + from importlib.metadata import version as _package_version PROGRAM_NAME = 'ocrmypdf' # Official PEP 396 -__version__ = pkg_resources.get_distribution('ocrmypdf').version +__version__ = _package_version('ocrmypdf') diff --git a/src/ocrmypdf/api.py b/src/ocrmypdf/api.py index 65ab910b..8f731fa3 100644 --- a/src/ocrmypdf/api.py +++ b/src/ocrmypdf/api.py @@ -15,10 +15,7 @@ from pathlib import Path from typing import AnyStr, BinaryIO, Iterable, Optional, Union from warnings import warn -from ocrmypdf._logging import ( # pylint: disable=unused-import - PageNumberFilter, - TqdmConsole, -) +from ocrmypdf._logging import PageNumberFilter, TqdmConsole from ocrmypdf._plugin_manager import get_plugin_manager from ocrmypdf._sync import run_pipeline from ocrmypdf._validation import check_options @@ -31,7 +28,7 @@ except ModuleNotFoundError: coloredlogs = None -StrPath = Union[os.PathLike, AnyStr] +StrPath = Union[Path, AnyStr] PathOrIO = Union[BinaryIO, StrPath] _api_lock = threading.Lock() @@ -338,3 +335,17 @@ def ocr( # pylint: disable=unused-argument options = create_options(**create_options_kwargs) check_options(options, plugin_manager) return run_pipeline(options=options, plugin_manager=plugin_manager, api=True) + + +__all__ = [ + 'PageNumberFilter', + 'TqdmConsole', + 'Verbosity', + 'check_options', + 'configure_logging', + 'create_options', + 'get_parser', + 'get_plugin_manager', + 'ocr', + 'run_pipeline', +] diff --git a/src/ocrmypdf/builtin_plugins/concurrency.py b/src/ocrmypdf/builtin_plugins/concurrency.py index 797087ae..74e6b504 100644 --- a/src/ocrmypdf/builtin_plugins/concurrency.py +++ b/src/ocrmypdf/builtin_plugins/concurrency.py @@ -20,9 +20,8 @@ import signal import sys import threading from contextlib import suppress -from multiprocessing import Pool as ProcessPool -from multiprocessing.pool import ThreadPool -from typing import Callable, Iterable, Union +from multiprocessing.pool import Pool, ThreadPool +from typing import Callable, Iterable, Type, Union from tqdm import tqdm @@ -31,7 +30,10 @@ from ocrmypdf._logging import TqdmConsole from ocrmypdf.exceptions import InputFileError from ocrmypdf.helpers import remove_all_log_handlers +ProcessPool = Pool Queue = Union[multiprocessing.Queue, queue.Queue] +UserInit = Callable[[], None] +WorkerInit = Callable[[Queue, UserInit, int], None] def log_listener(q: Queue): @@ -62,7 +64,7 @@ def process_sigbus(*args): raise InputFileError("A worker process lost access to an input file") -def process_init(q: Queue, user_init: Callable[[], None], loglevel): +def process_init(q: Queue, user_init: UserInit, loglevel) -> None: """Initialize a process pool worker""" # Ignore SIGINT (our parent process will kill us gracefully) @@ -85,7 +87,7 @@ def process_init(q: Queue, user_init: Callable[[], None], loglevel): return -def thread_init(_queue: Queue, user_init: Callable[[], None], _loglevel): +def thread_init(q: Queue, user_init: UserInit, loglevel) -> None: # As a thread, block SIGBUS so the main thread deals with it... with suppress(AttributeError): signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGBUS}) @@ -107,9 +109,9 @@ class StandardExecutor(Executor): task_finished: Callable, ): if use_threads: - log_queue = queue.Queue(-1) - pool_class = ThreadPool - initializer = thread_init + log_queue: Queue = queue.Queue(-1) + pool_class: Type[Pool] = ThreadPool + initializer: WorkerInit = thread_init else: log_queue = multiprocessing.Queue(-1) pool_class = ProcessPool diff --git a/src/ocrmypdf/builtin_plugins/tesseract_ocr.py b/src/ocrmypdf/builtin_plugins/tesseract_ocr.py index a6973b01..f4f45639 100644 --- a/src/ocrmypdf/builtin_plugins/tesseract_ocr.py +++ b/src/ocrmypdf/builtin_plugins/tesseract_ocr.py @@ -11,7 +11,6 @@ import os from ocrmypdf import hookimpl from ocrmypdf._exec import tesseract from ocrmypdf.cli import numeric -from ocrmypdf.exceptions import MissingDependencyError from ocrmypdf.helpers import clamp from ocrmypdf.pluginspec import OcrEngine from ocrmypdf.subprocess import check_external_program diff --git a/src/ocrmypdf/cli.py b/src/ocrmypdf/cli.py index 204b7b31..87012321 100644 --- a/src/ocrmypdf/cli.py +++ b/src/ocrmypdf/cli.py @@ -6,7 +6,7 @@ import argparse -from typing import Optional, Type, TypeVar +from typing import Any, Callable, Optional, TypeVar from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME from ocrmypdf._version import __version__ as _VERSION @@ -14,7 +14,9 @@ from ocrmypdf._version import __version__ as _VERSION T = TypeVar('T') -def numeric(basetype: Type[T], min_: Optional[T] = None, max_: Optional[T] = None): +def numeric( + basetype: Callable[[Any], T], min_: Optional[T] = None, max_: Optional[T] = None +): """Validator for numeric params""" min_ = basetype(min_) if min_ is not None else None max_ = basetype(max_) if max_ is not None else None @@ -22,7 +24,7 @@ def numeric(basetype: Type[T], min_: Optional[T] = None, max_: Optional[T] = Non def _numeric(string): value = basetype(string) if (min_ is not None and value < min_) or (max_ is not None and value > max_): - msg = "%r not in valid range %r" % (string, (min_, max_)) + msg = f"{string!r} not in valid range {(min_, max_)!r}" raise argparse.ArgumentTypeError(msg) return value @@ -145,7 +147,7 @@ Online documentation is located at: ) parser.add_argument( '--output-type', - choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'], + choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3', 'none'], default='pdfa', help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for " "long term archiving (default, recommended) but may not suitable " @@ -153,7 +155,8 @@ Online documentation is located at: "also has problems with full Unicode text. 'pdf' attempts to " "preserve file contents as much as possible. 'pdf-a1' creates a " "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'. 'pdf-a3' creates a " - "PDF/A3-b file.", + "PDF/A3-b file. 'none' will produce no output, which may be helpful if " + "only the --sidecar is desired.", ) # Use null string '\0' as sentinel to indicate the user supplied no argument, @@ -338,8 +341,9 @@ Online documentation is located at: "Control how PDF is optimized after processing:" "0 - do not optimize; " "1 - do safe, lossless optimizations (default); " - "2 - do some lossy optimizations; " - "3 - do aggressive lossy optimizations (including lossy JBIG2)" + "2 - do lossy JPEG and JPEG2000 optimizations; " + "3 - do more aggressive lossy JPEG and JPEG2000 optimizations. " + "To enable lossy JBIG2, see --jbig2-lossy." ), ) optimizing.add_argument( @@ -377,7 +381,8 @@ Online documentation is located at: action='store_true', help=( "Enable JBIG2 lossy mode (better compression, not suitable for some " - "use cases - see documentation)." + "use cases - see documentation). Only takes effect if --optimize 1 or " + "higher is also enabled." ), ) optimizing.add_argument( diff --git a/src/ocrmypdf/data/__init__.py b/src/ocrmypdf/data/__init__.py new file mode 100644 index 00000000..ad56a522 --- /dev/null +++ b/src/ocrmypdf/data/__init__.py @@ -0,0 +1,8 @@ +# © 2021 James R. Barlow: github.com/jbarlow83 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +"""Data files used to generate certain PDFs.""" diff --git a/src/ocrmypdf/extra_plugins/semfree.py b/src/ocrmypdf/extra_plugins/semfree.py index c84b1b2d..186206d9 100644 --- a/src/ocrmypdf/extra_plugins/semfree.py +++ b/src/ocrmypdf/extra_plugins/semfree.py @@ -28,7 +28,7 @@ from enum import Enum, auto from itertools import islice, repeat, takewhile, zip_longest from multiprocessing import Pipe, Process from multiprocessing.connection import Connection, wait -from typing import Callable, Iterable, Iterator +from typing import Callable, Iterable, Iterator, List from ocrmypdf import Executor, hookimpl from ocrmypdf._concurrent import NullProgressBar @@ -60,7 +60,9 @@ def process_sigbus(*args): class ConnectionLogHandler(logging.handlers.QueueHandler): def __init__(self, conn: Connection) -> None: - super().__init__(None) + # sets the parent's queue to None - parent only touches queue + # in enqueue() which we override + super().__init__(None) # type: ignore self.conn = conn def enqueue(self, record): @@ -126,8 +128,8 @@ class LambdaExecutor(Executor): if not grouped_args: return - processes = [] - connections = [] + processes: List[Process] = [] + connections: List[Connection] = [] for chunk in grouped_args: parent_conn, child_conn = Pipe() @@ -152,6 +154,8 @@ class LambdaExecutor(Executor): with self.pbar_class(**tqdm_kwargs) as pbar: while connections: for r in wait(connections): + if not isinstance(r, Connection): + raise NotImplementedError("We only support Connection()") try: msg_type, msg = r.recv() except EOFError: diff --git a/src/ocrmypdf/helpers.py b/src/ocrmypdf/helpers.py index d7080f34..7ff2ea71 100644 --- a/src/ocrmypdf/helpers.py +++ b/src/ocrmypdf/helpers.py @@ -25,19 +25,31 @@ log = logging.getLogger(__name__) class Resolution(namedtuple('Resolution', ('x', 'y'))): - """The number of pixels per inch in each 2D direction.""" + """The number of pixels per inch in each 2D direction. + + Resolution objects are considered "equal" for == purposes if they are + equal to a reasonable tolerance. + """ __slots__ = () + # rel_tol after converting from dpi to pixels per meter and saving + # as integer with rounding, as many file formats + CONVERSION_ERROR = 0.002 + def round(self, ndigits: int): return Resolution(round(self.x, ndigits), round(self.y, ndigits)) def to_int(self): return Resolution(int(round(self.x)), int(round(self.y))) + @classmethod + def _isclose(cls, a, b): + return isclose(a, b, rel_tol=cls.CONVERSION_ERROR) + @property def is_square(self) -> bool: - return isclose(self.x, self.y, rel_tol=1e-3) + return self._isclose(self.x, self.y) @property def is_finite(self) -> bool: @@ -61,6 +73,13 @@ class Resolution(namedtuple('Resolution', ('x', 'y'))): def __repr__(self): # pragma: no cover return f"Resolution({self.x}x{self.y} dpi)" + def __eq__(self, other): + if isinstance(other, tuple) and len(other) == 2: + other = Resolution(*other) + if not isinstance(other, Resolution): + return NotImplemented + return self._isclose(self.x, other.x) and self._isclose(self.y, other.y) + class NeverRaise(Exception): """An exception that is never raised""" @@ -170,7 +189,7 @@ def is_file_writable(test_file: os.PathLike) -> bool: with suppress(OSError): p.unlink() return True - except (EnvironmentError, RuntimeError) as e: + except (OSError, RuntimeError) as e: log.debug(e) log.error(str(e)) return False @@ -207,7 +226,7 @@ def check_pdf(input_file: Path) -> bool: except ( # Workaround for a problematic pikepdf version # pragma: no cover - getattr(pikepdf, 'ForeignObjectError') + pikepdf.ForeignObjectError if pikepdf.__version__ == '2.1.0' else NeverRaise ): @@ -254,7 +273,7 @@ def deprecated(func): def new_func(*args, **kwargs): warnings.simplefilter('always', DeprecationWarning) # turn off filter warnings.warn( - "Call to deprecated function {}.".format(func.__name__), + f"Call to deprecated function {func.__name__}.", category=DeprecationWarning, stacklevel=2, ) diff --git a/src/ocrmypdf/hocrtransform.py b/src/ocrmypdf/hocrtransform.py index 6c64bcff..90895b4b 100755 --- a/src/ocrmypdf/hocrtransform.py +++ b/src/ocrmypdf/hocrtransform.py @@ -31,7 +31,6 @@ import argparse import os import re -from itertools import chain from math import atan, cos, sin from pathlib import Path from typing import Any, NamedTuple, Optional, Tuple, Union @@ -41,6 +40,62 @@ from reportlab.lib.colors import black, cyan, magenta, red from reportlab.lib.units import inch from reportlab.pdfgen.canvas import Canvas +# According to Wikipedia these languages are supported in the ISO-8859-1 character +# set, meaning reportlab can generate them and they are compatible with hocr, +# assuming Tesseract has the necessary languages installed. Note that there may +# not be language packs for them. +HOCR_OK_LANGS = frozenset( + [ + # Languages fully covered by Latin-1: + 'afr', # Afrikaans + 'alb', # Albanian + 'ast', # Leonese + 'baq', # Basque + 'bre', # Breton + 'cos', # Corsican + 'eng', # English + 'eus', # Basque + 'fao', # Faoese + 'gla', # Scottish Gaelic + 'glg', # Galician + 'glv', # Manx + 'ice', # Icelandic + 'ind', # Indonesian + 'isl', # Icelandic + 'ita', # Italian + 'ltz', # Luxembourgish + 'mal', # Malay Rumi + 'mga', # Irish + 'nor', # Norwegian + 'oci', # Occitan + 'por', # Portugeuse + 'roh', # Romansh + 'sco', # Scots + 'sma', # Sami + 'spa', # Spanish + 'sqi', # Albanian + 'swa', # Swahili + 'swe', # Swedish + 'tgl', # Tagalog + 'wln', # Walloon + # Languages supported by Latin-1 except for a few rare characters that OCR + # is probably not trained to recognize anyway: + 'cat', # Catalan + 'cym', # Welsh + 'dan', # Danish + 'deu', # German + 'dut', # Dutch + 'est', # Estonian + 'fin', # Finnish + 'fra', # French + 'hun', # Hungarian + 'kur', # Kurdish + 'nld', # Dutch + 'wel', # Welsh + ] +) + + Element = ElementTree.Element @@ -241,13 +296,11 @@ class HocrTransform: ) found_lines = False - for line in sorted( - chain( - self.hocr.iterfind(self._child_xpath('span', 'ocr_header')), - self.hocr.iterfind(self._child_xpath('span', 'ocr_line')), - self.hocr.iterfind(self._child_xpath('span', 'ocr_textfloat')), - ), - key=self.topdown_position, + for line in ( + element + for element in self.hocr.iterfind(self._child_xpath('span')) + if 'class' in element.attrib + and element.attrib['class'] in {'ocr_header', 'ocr_line', 'ocr_textfloat'} ): found_lines = True self._do_line( @@ -296,7 +349,7 @@ class HocrTransform: interword_spaces: bool, show_bounding_boxes: bool, ): - if not line: + if line is None: return pxl_line_coords = self.element_coordinates(line) line_box = self.pt_from_pixel(pxl_line_coords) diff --git a/src/ocrmypdf/leptonica.py b/src/ocrmypdf/leptonica.py index 807560ff..e4814f1a 100644 --- a/src/ocrmypdf/leptonica.py +++ b/src/ocrmypdf/leptonica.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # © 2013-16: jbarlow83 from Github (https://github.com/jbarlow83) # @@ -13,6 +12,7 @@ import argparse import logging import os +import platform import sys import threading from collections import deque @@ -23,6 +23,7 @@ from functools import lru_cache from io import BytesIO, UnsupportedOperation from os import fspath from tempfile import TemporaryFile +from typing import ContextManager, Type from warnings import warn from ocrmypdf.exceptions import MissingDependencyError @@ -67,7 +68,7 @@ if os.name == 'nt': # Loading zlib from other places could cause a version mismatch _zlib_path = os.path.join(os.path.dirname(_libpath), 'zlib1.dll') if not os.path.exists(_zlib_path): - _zlib_path = find_library('zlib') + _zlib_path = find_library('zlib') or '' try: zlib = ffi.dlopen(_zlib_path) except ffi.error as e: @@ -86,7 +87,7 @@ except ffi.error as e: ) from e -class _LeptonicaErrorTrap_Redirect: +class _LeptonicaErrorTrap_Redirect(ContextManager): """ Context manager to trap errors reported by Leptonica < 1.79 or on Apple Silicon. @@ -132,7 +133,7 @@ class _LeptonicaErrorTrap_Redirect: except Exception: self.leptonica_lock.release() raise - return self + return def __exit__(self, exc_type, exc_value, traceback): # Restore old stderr @@ -172,7 +173,7 @@ tls = threading.local() tls.trap = None -class _LeptonicaErrorTrap_Queue: +class _LeptonicaErrorTrap_Queue(ContextManager): def __init__(self): self.queue = deque() @@ -226,7 +227,7 @@ except (ffi.error, MemoryError): # Pre-1.79 Leptonica does not have leptSetStderrHandler # And some platforms, notably Apple ARM 64, do not allow the write+execute # memory needed to set up the callback function. - _LeptonicaErrorTrap = _LeptonicaErrorTrap_Redirect + _LeptonicaErrorTrap: Type[ContextManager] = _LeptonicaErrorTrap_Redirect else: # 1.79 have this new symbol _LeptonicaErrorTrap = _LeptonicaErrorTrap_Queue @@ -272,7 +273,7 @@ class LeptonicaObject: # Leptonica API uses double-pointers for its destroy APIs to prevent # dangling pointers. This means we need to put our single pointer, # cdata, in a temporary CDATA**. - pp = ffi.new('{} **'.format(cls.LEPTONICA_TYPENAME), cdata) + pp = ffi.new(f'{cls.LEPTONICA_TYPENAME} **', cdata) cls.cdata_destroy(pp) @@ -439,6 +440,9 @@ class Pix(LeptonicaObject): bio = BytesIO() pillow_image.save(bio, format='png', compress_level=1) py_buffer = bio.getbuffer() + if platform.python_implementation() == 'PyPy': + # PyPy complains that it cannot do from_buffer(memoryview) + py_buffer = bytes(py_buffer) c_buffer = ffi.from_buffer(py_buffer) with _LeptonicaErrorTrap(): pix = Pix(lept.pixReadMem(c_buffer, len(c_buffer))) @@ -844,7 +848,7 @@ class Box(LeptonicaObject): def __repr__(self): if self._cdata: - return ''.format( + return ''.format( self.x, self.y, self.w, self.h ) return '' @@ -916,7 +920,7 @@ class Sel(LeptonicaObject): lines = [line.strip() for line in selstr.split('\n') if line.strip()] h = len(lines) w = len(lines[0]) - lengths = set(len(line) for line in lines) + lengths = {len(line) for line in lines} if len(lengths) != 1: raise ValueError("All lines in selstr must be same length") diff --git a/src/ocrmypdf/optimize.py b/src/ocrmypdf/optimize.py index fce66464..0c6d3b77 100644 --- a/src/ocrmypdf/optimize.py +++ b/src/ocrmypdf/optimize.py @@ -25,8 +25,16 @@ from typing import ( ) import img2pdf -import pikepdf -from pikepdf import Dictionary, Name, Object, Pdf, PdfImage +from pikepdf import ( + Dictionary, + Name, + Object, + ObjectStreamMode, + Pdf, + PdfImage, + Stream, + UnsupportedImageTypeError, +) from PIL import Image from ocrmypdf import leptonica @@ -63,7 +71,7 @@ def jpg_name(root: Path, xref: Xref) -> Path: def extract_image_filter( - pike: Pdf, root: Path, image: Object, xref: Xref + pike: Pdf, root: Path, image: Stream, xref: Xref ) -> Optional[Tuple[PdfImage, Tuple[Name, Object]]]: del pike # unused args del root @@ -89,7 +97,7 @@ def extract_image_filter( return None # Don't mess with wide gamut images if filtdp[0] == Name.JPXDecode: - log.debug(f"Skipping JPEG2000 iamge, xref {xref}") + log.debug(f"Skipping JPEG2000 image, xref {xref}") return None # Don't do JPEG2000 if filtdp[0] == Name.CCITTFaxDecode and filtdp[1].get('/K', 0) >= 0: @@ -104,7 +112,7 @@ def extract_image_filter( def extract_image_jbig2( - *, pike: pikepdf.Pdf, root: Path, image: Object, xref: Xref, options + *, pike: Pdf, root: Path, image: Stream, xref: Xref, options ) -> Optional[XrefExt]: del options # unused arg @@ -123,16 +131,16 @@ def extract_image_jbig2( # Showing the palette or ICC to jbig2enc will cause it to perform # colorspace transform to 1bpp, which will conflict the palette or # ICC if it exists. - colorspace = pim.obj.get(pikepdf.Name.ColorSpace, None) + colorspace = pim.obj.get(Name.ColorSpace, None) if colorspace is not None or pim.image_mask: try: # Set to DeviceGray temporarily; we already in 1 bpc. - pim.obj.ColorSpace = pikepdf.Name.DeviceGray + pim.obj.ColorSpace = Name.DeviceGray imgname = root / f'{xref:08d}' with imgname.open('wb') as f: ext = pim.extract_to(stream=f) imgname.rename(imgname.with_suffix(ext)) - except pikepdf.UnsupportedImageTypeError: + except UnsupportedImageTypeError: return None finally: # Restore image colorspace after temporarily setting it to DeviceGray @@ -145,7 +153,7 @@ def extract_image_jbig2( def extract_image_generic( - *, pike: Pdf, root: Path, image: PdfImage, xref: Xref, options + *, pike: Pdf, root: Path, image: Stream, xref: Xref, options ) -> Optional[XrefExt]: result = extract_image_filter(pike, root, image, xref) if result is None: @@ -178,7 +186,7 @@ def extract_image_generic( with imgname.open('wb') as f: ext = pim.extract_to(stream=f) imgname.rename(imgname.with_suffix(ext)) - except pikepdf.UnsupportedImageTypeError: + except UnsupportedImageTypeError: return None return XrefExt(xref, ext) elif ( @@ -365,6 +373,7 @@ def convert_to_jbig2( When the JBIG2 symbolic coder is not used, each JBIG2 stands on its own and needs no dictionary. Currently this must be lossless JBIG2. """ + jbig2_globals_dict: Optional[Dictionary] _produce_jbig2_images(jbig2_groups, root, options, executor) @@ -373,7 +382,7 @@ def convert_to_jbig2( jbig2_symfile = root / (prefix + '.sym') if jbig2_symfile.exists(): jbig2_globals_data = jbig2_symfile.read_bytes() - jbig2_globals = pikepdf.Stream(pike, jbig2_globals_data) + jbig2_globals = Stream(pike, jbig2_globals_data) jbig2_globals_dict = Dictionary(JBIG2Globals=jbig2_globals) elif options.jbig2_page_group_size == 1: jbig2_globals_dict = None @@ -444,8 +453,8 @@ def _transcode_png(pike: Pdf, filename: Path, xref: Xref) -> bool: with output.open('wb') as f: img2pdf.convert(fspath(filename), outputstream=f) - with pikepdf.open(output) as pdf_image: - foreign_image = next(pdf_image.pages[0].images.values()) + with Pdf.open(output) as pdf_image: + foreign_image = next(iter(pdf_image.pages[0].images.values())) local_image = pike.copy_foreign(foreign_image) im_obj = pike.get_object(xref, 0) @@ -524,12 +533,15 @@ def transcode_pngs( _transcode_png(pike, filename, xref) +DEFAULT_EXECUTOR = SerialExecutor() + + def optimize( input_file: Path, output_file: Path, context, save_settings, - executor: Executor = SerialExecutor(), + executor: Executor = DEFAULT_EXECUTOR, ) -> None: options = context.options if options.optimize == 0: @@ -543,7 +555,7 @@ def optimize( if options.jbig2_page_group_size == 0: options.jbig2_page_group_size = 10 if options.jbig2_lossy else 1 - with pikepdf.Pdf.open(input_file) as pike: + with Pdf.open(input_file) as pike: root = output_file.parent / 'images' root.mkdir(exist_ok=True) @@ -575,7 +587,7 @@ def optimize( if savings < 0: log.info("Image optimization did not improve the file - discarded") # We still need to save the file - with pikepdf.open(input_file) as pike: + with Pdf.open(input_file) as pike: pike.remove_unreferenced_resources() pike.save(output_file, **save_settings) else: @@ -622,7 +634,7 @@ def main(infile, outfile, level, jobs=1): dict( compress_streams=True, preserve_pdfa=True, - object_stream_mode=pikepdf.ObjectStreamMode.generate, + object_stream_mode=ObjectStreamMode.generate, ), ) copy(fspath(tmpout), fspath(outfile)) diff --git a/src/ocrmypdf/pdfa.py b/src/ocrmypdf/pdfa.py index 4eaee8f1..b17eb213 100644 --- a/src/ocrmypdf/pdfa.py +++ b/src/ocrmypdf/pdfa.py @@ -13,13 +13,20 @@ import base64 from pathlib import Path from typing import Dict, Iterator, Union +try: + from importlib_resources import read_binary +except ImportError: + from importlib.resources import read_binary import pikepdf -import pkg_resources +import pkg_resources # deprecated +# Deprecated ICC_PROFILE_RELPATH = 'data/sRGB.icc' - +# Deprecated SRGB_ICC_PROFILE = pkg_resources.resource_filename('ocrmypdf', ICC_PROFILE_RELPATH) +SRGB_ICC_PROFILE_NAME = 'sRGB.icc' + def _postscript_objdef( alias: str, @@ -97,12 +104,12 @@ def generate_pdfa_ps(target_filename: Path, icc: str = 'sRGB'): References: Adobe PDFMARK Reference: https://www.adobe.com/content/dam/acom/en/devnet/acrobat/pdfs/pdfmark_reference.pdf """ - if icc == 'sRGB': - icc_profile = SRGB_ICC_PROFILE - else: + if icc != 'sRGB': raise NotImplementedError("Only supporting sRGB") - bytes_icc_profile = Path(icc_profile).read_bytes() + bytes_icc_profile = read_binary( + 'ocrmypdf.data', SRGB_ICC_PROFILE_NAME + ) ps = '\n'.join(_make_postscript(icc, bytes_icc_profile, 3)) # We should have encoded everything to pure ASCII by this point, and diff --git a/src/ocrmypdf/pdfinfo/info.py b/src/ocrmypdf/pdfinfo/info.py index 64b59b9c..b656e107 100644 --- a/src/ocrmypdf/pdfinfo/info.py +++ b/src/ocrmypdf/pdfinfo/info.py @@ -9,7 +9,7 @@ import atexit import logging import re -from collections import defaultdict, namedtuple +from collections import defaultdict from contextlib import ExitStack from decimal import Decimal from enum import Enum @@ -17,11 +17,27 @@ from functools import partial from math import hypot, inf, isclose from os import PathLike from pathlib import Path -from typing import Container, Iterator, Optional, Tuple, Union +from typing import ( + Container, + Dict, + Iterator, + List, + Mapping, + NamedTuple, + Optional, + Tuple, + Union, +) from warnings import warn -import pikepdf -from pikepdf import Object, Pdf, PdfMatrix +from pikepdf import ( + Object, + Pdf, + PdfImage, + PdfInlineImage, + PdfMatrix, + parse_content_stream, +) from ocrmypdf._concurrent import Executor, SerialExecutor from ocrmypdf.exceptions import EncryptedPdfError, InputFileError @@ -36,7 +52,7 @@ Encoding = Enum( 'Encoding', 'ccitt jpeg jpeg2000 jbig2 asciihex ascii85 lzw flate runlength' ) -FRIENDLY_COLORSPACE = { +FRIENDLY_COLORSPACE: Dict[str, Colorspace] = { '/DeviceGray': Colorspace.gray, '/CalGray': Colorspace.gray, '/DeviceRGB': Colorspace.rgb, @@ -54,7 +70,7 @@ FRIENDLY_COLORSPACE = { '/I': Colorspace.index, } -FRIENDLY_ENCODING = { +FRIENDLY_ENCODING: Dict[str, Encoding] = { '/CCITTFaxDecode': Encoding.ccitt, '/DCTDecode': Encoding.jpeg, '/JPXDecode': Encoding.jpeg2000, @@ -68,7 +84,7 @@ FRIENDLY_ENCODING = { '/RL': Encoding.runlength, } -FRIENDLY_COMP = { +FRIENDLY_COMP: Dict[Colorspace, int] = { Colorspace.gray: 1, Colorspace.rgb: 3, Colorspace.cmyk: 4, @@ -86,16 +102,30 @@ def _is_unit_square(shorthand): return all(isclose(a, b, rel_tol=1e-3) for a, b in pairwise) -XobjectSettings = namedtuple('XobjectSettings', ['name', 'shorthand', 'stack_depth']) +class XobjectSettings(NamedTuple): + name: str + shorthand: Tuple[float, float, float, float, float, float] + stack_depth: int -InlineSettings = namedtuple('InlineSettings', ['iimage', 'shorthand', 'stack_depth']) -ContentsInfo = namedtuple( - 'ContentsInfo', - ['xobject_settings', 'inline_images', 'found_vector', 'found_text', 'name_index'], -) +class InlineSettings(NamedTuple): + iimage: PdfInlineImage + shorthand: Tuple[float, float, float, float, float, float] + stack_depth: int -TextboxInfo = namedtuple('TextboxInfo', ['bbox', 'is_visible', 'is_corrupt']) + +class ContentsInfo(NamedTuple): + xobject_settings: List[XobjectSettings] + inline_images: List[InlineSettings] + found_vector: bool + found_text: bool + name_index: Mapping[str, List[XobjectSettings]] + + +class TextboxInfo(NamedTuple): + bbox: Tuple[float, float, float, float] + is_visible: bool + is_corrupt: bool class VectorMarker: @@ -146,8 +176,8 @@ def _interpret_contents(contentstream: Object, initial_shorthand=UNIT_SQUARE): stack = [] ctm = PdfMatrix(initial_shorthand) - xobject_settings = [] - inline_images = [] + xobject_settings: List[XobjectSettings] = [] + inline_images: List[InlineSettings] = [] name_index = defaultdict(lambda: []) found_vector = False found_text = False @@ -157,9 +187,7 @@ def _interpret_contents(contentstream: Object, initial_shorthand=UNIT_SQUARE): operator_whitelist = ' '.join(vector_ops | text_showing_ops | image_ops) for n, graphobj in enumerate( - _normalize_stack( - pikepdf.parse_content_stream(contentstream, operator_whitelist) - ) + _normalize_stack(parse_content_stream(contentstream, operator_whitelist)) ): operands, operator = graphobj if operator == 'q': @@ -185,7 +213,7 @@ def _interpret_contents(contentstream: Object, initial_shorthand=UNIT_SQUARE): name=image_name, shorthand=ctm.shorthand, stack_depth=len(stack) ) xobject_settings.append(settings) - name_index[image_name].append(settings) + name_index[str(image_name)].append(settings) elif operator == 'INLINE IMAGE': # BI/ID/EI are grouped into this iimage = operands[0] inline = InlineSettings( @@ -271,23 +299,28 @@ def _get_dpi(ctm_shorthand, image_size) -> Resolution: class ImageInfo: DPI_PREC = Decimal('1.000') + _comp: Optional[int] + _name: str + def __init__( self, *, name='', pdfimage: Optional[Object] = None, - inline: Optional[Object] = None, + inline: Optional[PdfInlineImage] = None, shorthand=None, ): self._name = str(name) self._shorthand = shorthand + pim: Union[PdfInlineImage, PdfImage] + if inline is not None: self._origin = 'inline' - pim = inline.iimage + pim = inline elif pdfimage is not None: self._origin = 'xobject' - pim = pikepdf.PdfImage(pdfimage) + pim = PdfImage(pdfimage) else: raise ValueError("Either pdfimage or inline must be set") self._width = pim.width @@ -303,14 +336,14 @@ class ImageInfo: self._bpc = int(pim.bits_per_component) try: - self._enc = FRIENDLY_ENCODING.get(pim.filters[0], 'image') + self._enc = FRIENDLY_ENCODING.get(pim.filters[0]) except IndexError: - self._enc = '?' + self._enc = None try: - self._color = FRIENDLY_COLORSPACE.get(pim.colorspace, '?') + self._color = FRIENDLY_COLORSPACE.get(pim.colorspace or '') except NotImplementedError: - self._color = '?' + self._color = None if self._enc == Encoding.jpeg2000: self._color = Colorspace.jpeg2000 @@ -324,11 +357,14 @@ class ImageInfo: else: self._comp = 3 else: - self._comp = FRIENDLY_COMP.get(self._color, '?') + if isinstance(self._color, Colorspace): + self._comp = FRIENDLY_COMP.get(self._color) + else: + self._comp = None # Bit of a hack... infer grayscale if component count is uncertain # but encoding only supports monochrome. - if self._comp == '?' and self._enc in (Encoding.ccitt, Encoding.jbig2): + if self._comp is None and self._enc in (Encoding.ccitt, Encoding.jbig2): self._comp = FRIENDLY_COMP[Colorspace.gray] @property @@ -353,15 +389,15 @@ class ImageInfo: @property def color(self): - return self._color + return self._color if self._color is not None else '?' @property def comp(self): - return self._comp + return self._comp if self._comp is not None else '?' @property def enc(self): - return self._enc + return self._enc if self._enc is not None else 'image' @property def renderable(self): @@ -388,7 +424,7 @@ def _find_inline_images(contentsinfo: ContentsInfo) -> Iterator[ImageInfo]: for n, inline in enumerate(contentsinfo.inline_images): yield ImageInfo( - name='inline-%02d' % n, shorthand=inline.shorthand, inline=inline + name='inline-%02d' % n, shorthand=inline.shorthand, inline=inline.iimage ) @@ -413,7 +449,7 @@ def _image_xobjects(container) -> Iterator[Tuple[Object, str]]: xobjs = resources['/XObject'].as_dict() for xobj in xobjs: candidate: Object = xobjs[xobj] - if not '/Subtype' in candidate: + if '/Subtype' not in candidate: continue if candidate['/Subtype'] == '/Image': pdfimage = candidate @@ -583,7 +619,7 @@ def _pdf_pageinfo_sync_init(pdf: Pdf, infile: Path, pdfminer_loglevel): # If the pdf is not opened, open a copy for our worker process to use if pdf is None: - worker_pdf = pikepdf.open(infile) + worker_pdf = Pdf.open(infile) def on_process_close(): worker_pdf.close() @@ -597,7 +633,7 @@ def _pdf_pageinfo_sync(args): pdf = thread_pdf if thread_pdf is not None else worker_pdf with ExitStack() as stack: if not pdf: # When called with SerialExecutor - pdf = stack.enter_context(pikepdf.open(infile)) + pdf = stack.enter_context(Pdf.open(infile)) page = PageInfo(pdf, pageno, infile, check_pages, detailed_analysis) return page @@ -661,6 +697,10 @@ def _pdf_pageinfo_concurrent( class PageInfo: + _has_text: Optional[bool] + _has_vector: Optional[bool] + _images: List[ImageInfo] + def __init__( self, pdf: Pdf, @@ -732,7 +772,7 @@ class PageInfo: else: self._has_vector = None # i.e. "no information" self._has_text = None - self._images = None + self._images = [] self._dpi = None if self._images: @@ -749,7 +789,7 @@ class PageInfo: @property def has_text(self) -> bool: - return self._has_text + return bool(self._has_text) @property def has_corrupt_text(self) -> bool: @@ -759,7 +799,7 @@ class PageInfo: @property def has_vector(self) -> bool: - return self._has_vector + return bool(self._has_vector) @property def width_inches(self) -> Decimal: @@ -837,6 +877,9 @@ class PageInfo: ) +DEFAULT_EXECUTOR = SerialExecutor() + + class PdfInfo: """Get summary information about a PDF""" @@ -848,13 +891,13 @@ class PdfInfo: progbar: bool = False, max_workers: int = None, check_pages=None, - executor: Executor = SerialExecutor(), + executor: Executor = DEFAULT_EXECUTOR, ): self._infile = infile if check_pages is None: check_pages = range(0, 1_000_000_000) - with pikepdf.open(infile) as pdf: + with Pdf.open(infile) as pdf: if pdf.is_encrypted: raise EncryptedPdfError() # Triggered by encryption with empty passwd self._pages = _pdf_pageinfo_concurrent( diff --git a/src/ocrmypdf/pdfinfo/layout.py b/src/ocrmypdf/pdfinfo/layout.py index 4159a1cb..bf8d7c30 100644 --- a/src/ocrmypdf/pdfinfo/layout.py +++ b/src/ocrmypdf/pdfinfo/layout.py @@ -135,7 +135,7 @@ class LTStateAwareChar(LTChar): return self._text def __repr__(self): - return '<%s %s matrix=%s rendermode=%r font=%r adv=%s text=%r>' % ( + return '<{} {} matrix={} rendermode={!r} font={!r} adv={} text={!r}>'.format( self.__class__.__name__, bbox2str(self.bbox), matrix2str(self.matrix), diff --git a/src/ocrmypdf/pluginspec.py b/src/ocrmypdf/pluginspec.py index 47de64b6..8d485bf1 100644 --- a/src/ocrmypdf/pluginspec.py +++ b/src/ocrmypdf/pluginspec.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from abc import ABC, abstractmethod, abstractstaticmethod +from abc import ABC, abstractmethod from argparse import ArgumentParser, Namespace from collections import namedtuple from logging import Handler @@ -197,7 +197,7 @@ def rasterize_pdf_page( @hookspec(firstresult=True) -def filter_ocr_image(page: 'PageContext', image: 'Image') -> 'Image': +def filter_ocr_image(page: 'PageContext', image: 'Image.Image') -> 'Image.Image': """Called to filter the image before it is sent to OCR. This is the image that OCR sees, not what the user sees when they view the @@ -325,11 +325,13 @@ class OcrEngine(ABC): Tesseract OCR. """ - @abstractstaticmethod + @staticmethod + @abstractmethod def version() -> str: """Returns the version of the OCR engine.""" - @abstractstaticmethod + @staticmethod + @abstractmethod def creator_tag(options: Namespace) -> str: """Returns the creator tag to identify this software's role in creating the PDF. @@ -349,24 +351,28 @@ class OcrEngine(ABC): to the user, usually in an error message. """ - @abstractstaticmethod + @staticmethod + @abstractmethod def languages(options: Namespace) -> AbstractSet[str]: """Returns the set of all languages that are supported by the engine. Languages are typically given in 3-letter ISO 3166-1 codes, but actually can be any value understood by the OCR engine.""" - @abstractstaticmethod + @staticmethod + @abstractmethod def get_orientation(input_file: Path, options: Namespace) -> OrientationConfidence: """Returns the orientation of the image.""" - @abstractstaticmethod + @staticmethod + @abstractmethod def generate_hocr( input_file: Path, output_hocr: Path, output_text: Path, options: Namespace ) -> None: """Called to produce a hOCR file and sidecar text file.""" - @abstractstaticmethod + @staticmethod + @abstractmethod def generate_pdf( input_file: Path, output_pdf: Path, output_text: Path, options: Namespace ) -> None: diff --git a/src/ocrmypdf/subprocess/__init__.py b/src/ocrmypdf/subprocess/__init__.py index 99f052c6..7cc153b5 100644 --- a/src/ocrmypdf/subprocess/__init__.py +++ b/src/ocrmypdf/subprocess/__init__.py @@ -15,7 +15,6 @@ from collections.abc import Mapping from contextlib import suppress from distutils.version import LooseVersion, Version from functools import lru_cache -from pathlib import Path from subprocess import PIPE, STDOUT, CalledProcessError, CompletedProcess, Popen from subprocess import run as subprocess_run from typing import Callable, Optional, Type, Union @@ -27,7 +26,9 @@ from ocrmypdf.exceptions import MissingDependencyError log = logging.getLogger(__name__) -def run(args, *, env=None, logs_errors_to_stdout=False, **kwargs): +def run( + args, *, env=None, logs_errors_to_stdout: bool = False, **kwargs +) -> CompletedProcess: """Wrapper around :py:func:`subprocess.run` The main purpose of this wrapper is to log subprocess output in an orderly @@ -65,7 +66,9 @@ def run(args, *, env=None, logs_errors_to_stdout=False, **kwargs): return proc -def run_polling_stderr(args, *, callback, check=False, env=None, **kwargs): +def run_polling_stderr( + args, *, callback: Callable[[str], None], check: bool = False, env=None, **kwargs +) -> CompletedProcess: """Run a process like ``ocrmypdf.subprocess.run``, and poll stderr. Every line of produced by stderr will be forwarded to the callback function. @@ -83,6 +86,8 @@ def run_polling_stderr(args, *, callback, check=False, env=None, **kwargs): with Popen(args, env=env, **kwargs) as proc: lines = [] while proc.poll() is None: + if proc.stderr is None: + continue for msg in iter(proc.stderr.readline, ''): if process_log.isEnabledFor(logging.DEBUG): process_log.debug(msg.strip()) @@ -102,7 +107,7 @@ def _fix_process_args(args, env, kwargs): env = os.environ # Search in spoof path if necessary - program = args[0] + program = str(args[0]) if os.name == 'nt': from ocrmypdf.subprocess._windows import fix_windows_args diff --git a/src/ocrmypdf/subprocess/_windows.py b/src/ocrmypdf/subprocess/_windows.py index aec082b7..c3130a7d 100644 --- a/src/ocrmypdf/subprocess/_windows.py +++ b/src/ocrmypdf/subprocess/_windows.py @@ -9,9 +9,9 @@ import os import shutil import sys from distutils.version import LooseVersion -from itertools import chain, filterfalse +from itertools import chain from pathlib import Path -from typing import Any, Callable, Iterator, Optional, Tuple, TypeVar, cast +from typing import Any, Callable, Iterable, Iterator, Set, Tuple, TypeVar try: import winreg @@ -113,7 +113,7 @@ SHIMS = [ ] -def fix_windows_args(program, args, env): +def fix_windows_args(program: str, args, env): """Adjust our desired program and command line arguments for use on Windows""" if sys.version_info < (3, 8): @@ -137,14 +137,12 @@ def fix_windows_args(program, args, env): return args -def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." +def unique_everseen(iterable: Iterable[T], key: Callable[[T], T]) -> Iterator[T]: + "List unique elements, preserving order." # unique_everseen('AAAABBBCCDAABBB') --> A B C D # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen = set() + seen: Set[T] = set() seen_add = seen.add - if key is None: - key = lambda x: x for element in iterable: k = key(element) if k not in seen: diff --git a/tests/cache/2400dpi/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/2400dpi/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin index 023d341f..48b6ae30 100644 Binary files a/tests/cache/2400dpi/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/2400dpi/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/3small/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/3small/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin index edfe4b01..c73a8b8b 100644 Binary files a/tests/cache/3small/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/3small/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/3small/__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt/pdf.bin b/tests/cache/3small/__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt/pdf.bin index 22b7068b..3b2321db 100644 Binary files a/tests/cache/3small/__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/3small/__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/3small/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/pdf.bin b/tests/cache/3small/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/pdf.bin index 0bb07c30..cda54f9e 100644 Binary files a/tests/cache/3small/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/3small/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/aspect/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/aspect/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin index 4a5241fc..e82cafcc 100644 --- a/tests/cache/aspect/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/aspect/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin @@ -9,7 +9,7 @@ -
+

diff --git a/tests/cache/aspect/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/aspect/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin index a0e93b41..ad465b3b 100644 Binary files a/tests/cache/aspect/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/aspect/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/cardinal/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/cardinal/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin index 51333eae..5b72892d 100644 --- a/tests/cache/cardinal/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/cardinal/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin @@ -9,7 +9,7 @@ -

+

diff --git a/tests/cache/cardinal/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/cardinal/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin index 394e7bbd..2f4f13b8 100644 Binary files a/tests/cache/cardinal/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/cardinal/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt/hocr.bin index 23a18626..bcfa42c9 100644 --- a/tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt/hocr.bin @@ -9,7 +9,7 @@ -

+

diff --git a/tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt/pdf.bin b/tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt/pdf.bin index dd362045..2f4f13b8 100644 Binary files a/tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt/txt.bin b/tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt/txt.bin index 686fd1ac..d3c2e860 100644 --- a/tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt/txt.bin +++ b/tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt/txt.bin @@ -103,7 +103,7 @@ ERASE, REPEAT, PLAY/STOP, or LOCATE. © Will sync to standard LinnDrum or Linn 9000 sync tone. -® Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation. +© Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation. * TEMPO may be specified in BEATS-PER-MINUTE or FRAMES-PER-BEAT at 24, 25, or 30 frames per second, (even drop frame!) @@ -115,9 +115,9 @@ on the TAP TEMPO button. ¢ TEMPO CHANGES may be programmed into a sequence, with smooth transitions if desired. ¢ Any TIME SIGNATURE may be used, and may be changed within a song. -nn - +linn Linn Electronics, Inc. + 18720 Oxnard Street, Tarzana, CA 91356 (818) 708-8131 TELEX #298949 LINN UR \ No newline at end of file diff --git a/tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt/hocr.bin index 48bd2afa..9d1f3184 100644 --- a/tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt/hocr.bin @@ -9,7 +9,7 @@ -

+

diff --git a/tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/pdf.bin b/tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/pdf.bin index a9c86e15..2f4f13b8 100644 Binary files a/tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/txt.bin b/tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/txt.bin index d80b111f..d3c2e860 100644 --- a/tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/txt.bin +++ b/tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/txt.bin @@ -1,128 +1,123 @@ -2A NNI‘I 6F6867# XATALL IE18-80L (818) +The LinnSequencer +32 Track MIDI Sequence Recorder -9SEI6 VO “BUBZIRY, “J0aNS PIPUXO OZLEI -“Uy ‘soTUOMOI,q UUrT +The LinnSequencer is a state-of-the-art composition and performance tool for the professional musician. It is -uut] +extremely powerful, yet amazingly simple to learn and use. It’s many remarkable features include: -“‘SUOS B UIJIM pasueyo oq ABU pue ‘posn oq AWW AYN IVNOIS AWLL AUV -“parlsop Jr SUOTIISUBI} YIOOUIS YIM “BoueNbas eB OJUI pourtueIZOId 9q ABU SFONWHO OdINAL e +¢ Operation is similar to multi-track tape recorder with PLAY, STOP, RECORD, FAST +FORWARD, REWIND, and LOCATE controls. -‘uonng OdNAL dV L 9) uO +e Each of the 100 sequences contains 32 simultaneous, polyphonic tracks. Each track may +be assigned to one of 16 MIDI channels. Simultaneously plays up to 16 polyphonic -sojou Jayienb Suiddy} Aq 10 ‘syUSTIOIOUI oINUTIAI-J8g-Jesg & JO sys} UL ofquisn(pe ‘ATTeouIAUINU paiajus oq ABU OdINALL e +synthesizers! -(jouer doup u3a9) +¢ Ultra-fast 3%” disk drive stores complex songs in seconds and holds over 110,000 notes -“puooes Jed souely O€ 10 “SZ “pz 18 [LVAG-MAd-SHN VU 10 ALOANIWAAd-SLVAd U! patyoeds aq kewl OAL « -‘uoTe1odo [SVx JO} Aj[eusoyUT JoyndUIOd 11g 9] 98108 ZHI 8g ‘poeds-ysry Bann soz] e +per disk! -"9U0} DUAS 0006 UUL] Jo wNIqUUr] prepue}s 0} OUAS [ITAA © +¢ One or all tracks may be TRANSPOSED at the touch of a key. +e Exclusive real-time ERASE function makes editing FAST. +* Exclusive REPEAT function automatically repeats any held notes at a pre-selected -“ONYBA 9}OU poloapes Aue Je sas—nd jndyno 07 pewureigold 3q ACW SL Ad LNO YADONAL OML +rhythmic value. -"ALVOOT 10 GOLS/AV 1d ‘LWddad “ASV +¢ TIMING CORRECTION works during playback and operates without ‘chopping’ notes. -SUIpNpoUr ‘suOTIOUN] posn A[UOUILUOS 94] JO AUBUT [O1]UOD AJ9]OWIAI 0} PousIsse oq ACUI ST AdNI HOLIMSLOO OME « -“SUIPIONAI I[IYM P2sesd JOU Iv $3}OU BUTISIXO—ZUIPIOIA SATON.ASOP-UON +¢ Optional SMPTE time code synchronization. -‘suoneurldxa peuoyippe sdeydsip uowng g1TqH +© Optional remote control. -oy] ‘pepsau JI ‘suoneiodo [ye yYsnosy] NOA sapins ApIespo Avfdsip QO] Joey Z7¢ 9y3—uoeIodo Urea] 0} Ased ‘aus « +Recording a Sequence -jUorel]suowtap & IO} Aepol Jayeap uur’] INOA dag ‘dISHUL -INOA 0} UONUS}]¥ PaplAIPUN INOA SUTJOASp ITY ps pue -p1osai ‘asoduod no Jay 0} pausisap st 1s0uenbesuur’] oy) -Aum Aposiooid $,Jeu], ‘SS9d0Id SATTBS1D OY} YIM SOIOJIOIUT -yey) xo]dwWI0d Os dq JOA9U P[NoUsS osn NOA AZopOuYdE} oy +To record a sequence, simply press RECORD and PLAY, +then play your MIDI keyboard in time to the Sequencer’s +click track. When the sequence loops back around to bar 1, +you’ ll hear what you played—only all timing errors will be -ISTUMOIAUIO?) NOAA UOHISOdWIO) +corrected! (Timing correction may be adjusted or defeated). -"NOSpr] B Oy ‘AONUTJUT yada 0} seq Maz Se] BY] Jas UdAd -uvd NOA ‘palisap JJ ‘souanbes Mou ¥B OVUT sjied ou] [Te Adoo -ATesrewO Ne WI) [IM ONOS ALVAAO JeyIe80} wey} -,deyd,, 0} UOTOUNJ ONOS ALVA ou] asn usy] ‘saouanbes -JENPIAIpUt UI (“949 ‘snJOYD ‘aS1OA) UOTIDIS JIseq Yes -Pl0da1 OF ST ABM JOuIOUY “(812g 666 01 dn) ysnory) ABM +Any additional notes played will be added into the track +— existing notes are not erased while recording! -dU} [fe YORI] YORs p10991 0} ST SUOS B 9789I9 0} ABM SUG, +FAST FORWARD, REWIND, and LOCATE controls +may be used at any time to quickly access any location in +your sequence for spot-recording. To overdub a new part, +select a different track and start recording—while you +record, the first track will play in perfect sync (unless you +MUTE it, or SOLO another track). In this way, up to 32 +tracks may be overdubbed! All MIDI effects are recorded +including pitch bend, modulation, velocity, aftertouch, +sustain pedal, and program changes! -SUOS & SUTVAID +Editing -*suoT}oes poJUBMUN +To erase a wrong note, simply hold ERASE and press +the note to be erased just before it plays in the sequence— +when played back, it will be gone. Notes may also be -SAOUIOI 0} ABM SWS dU} SoyeIodo SUV ALATAaG +added, erased, or changed using the SINGLE STEP func- +tion. To overdub notes at specific points within a sequence, -“OBPLIq dy} PUB SNIOY PUOdAS dT]] Ud9MIAQ SIDA ISI +Additional Features -ay) Jo Adoo B JJasuT WYSE NOAA ‘afdwexs 10.f ‘UO JUSIN]JIP +simply use LOCATE, FAST FORWARD, or REWIND to +find the desired bar number, then start recording. -B IO aouaNbas sues OY} UI—JOY OUP 0} UOTIEIO] 9UO WOT] -$1Bq JAOUI OF NOA sMOTIe WOTIOUNS AdOO/IMASNI OULL +The INSERT/COPY function allows you to move bars +from one location to another—in the same sequence or a +different one. For example, you might insert a copy of the +first verse between the second chorus and the bridge. +DELETE BARS operates the same way to remove +unwanted sections, -‘SUIPIONAI JIVIS Udy) “OQuINU eq porisop ay} puy +Creating a Song -0} CNIMAY 10 ‘CYVM Od LSWA “AEVOOT esn Apduns +One way to create a song is to record each track all the +way through (up to 999 bars). Another way is to record +each basic section (verse, chorus, etc.) in individual +sequences, then use the CREATE SONG function to “chain” +them together. CREATE SONG will then automatically +copy all the parts into a new sequence. If desired, you can +even set the last few bars to repeat infinitely, for a fadeout. -sainjeay [PUOHIPPY +Composition Without Compromise -‘gouanbas & UTYIIM s]UTOd a1y1dads 3¥ $9100 QnPIOAO OL "UOT} --ouns dALLS ATONIS 24) Suisn pasueyo Jo ‘pasesa ‘pappe -aq osye ABUT S9]ON ‘U0 9q ]IIM 1 “yoeq podeyd uayM -—aouanbas oy] ul skeyd 71 a10J9q Isnf posers oq 0} d]0U ayy -ssaid pue ASvwug ploy Aydunis ‘jou Suomm & aseso OL +The technology you use should never be so complex that +it interferes with the creative process. That’s precisely why +the LinnSequencer is designed to let you compose, record +and edit while devoting your undivided attention to your +music. See your Linn dealer today for a demonstration! -sunipa +* Simple, easy to learn operation—the 32 character LCD display clearly guides you through all operations. If needed, the -jsesdueyo ureisoid pue ‘fepod ureysns -‘yonoplalje ‘AWOOTOA ‘UOTyeTNpow ‘pusg youd Surpnyour -pep10del are $199JJ2 TCTIN [WV iPeqqnpseao aq Aeur syoen -Ze 07 dn ‘Kem sie Uy *(foeI} JOyOUR OJOS 10 ALLAN -NOA ssofum) duAS yOaysod ul Avy [[IM Yow] ISI 93 “prooar -NOA 3[IYM—SUIPIOIA LIBIS PU YORI) TUdIOTJIP B JOaTas -*y1ed MOU B QNPIsA0 OL, “SuIps0daJ-jods 10} aouanbes mno0k -UI UOHBIO] Aue ssad0e ATYOIND 0} owt} Aue ye pasn aq AvUE -SJONUOD FLIVOOT pur ‘ANIMA ‘CYVMaYOd LSVd -{SUIPIOSAI {IY posesa JOU se So]OU SuTsTXO— -yous} 3U} OUT poppe aq JIM poteyd sajou yeuonippe Auy +HELP button displays additional explanations. -*(povesjap 10 poysn{pe oq ABW UOTIIII0D BUTUTT]) j{paqoeLI09 +* Non-destructive recording—existing notes are not erased while recording. +¢ Two FOOTSWITCH INPUTS may be assigned to remotely control many of the commonly used functions, including -2q ][IM S1OLIe Sur [fe ATUO—patey]d nod Jey Jedy ]],NOA +ERASE, REPEAT, PLAY/STOP, or LOCATE. -‘] req 0] punose yoeq sdoo] sduanbas ay] Udy AA “YOu Yor +¢ Iwo TRIGGER OUTPUTS may be programmed to output pulses at any selected note value. -§,sa0uaNbas at} O] SUIT) UI preogday [IW] INO Avy usy3 -AV'1d pue (YOON ssoid Ayduus ‘aousnbes & p1o09es OF, +© Will sync to standard LinnDrum or Linn 9000 sync tone. -g0uaNbas & SUIP10I0y] +© Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation. +* TEMPO may be specified in BEATS-PER-MINUTE or FRAMES-PER-BEAT at 24, 25, or 30 frames per second, -‘JONWOD s}JouNaI TeuONdGO e +(even drop frame!) -"UOTJEZIUOIYUAS OPOS UIT} FLAWS [euondo e +¢ TEMPO may be entered numerically, adjustable in tenths of a Beat-Per-Minute increments, or by tapping quarter notes -‘sou .sulddoys, noyyM sayelodo pue yoegdvyd ZuLINp S¥IOM NOLLOANNYOO ONIWILL e +on the TAP TEMPO button. -‘onqea ory AY +¢ TEMPO CHANGES may be programmed into a sequence, with smooth transitions if desired. +¢ Any TIME SIGNATURE may be used, and may be changed within a song. -pojoojes-oid & ye sajou pyoy Aue syeadas ATTeONewWO Ne UOTOUNS [WAdAY OAISNOX e -‘LSVJ SUnIpS soyeu UOTOUN ASV UA OUlN-[eal SAISNIOXY e -‘Koy B JO YONO} 941 12 CASOdSNVALL 0g ABU Syde] [Te 10 9UC e +linn +Linn Electronics, Inc. -i ASIP Jed - -S9}0U OOO‘OTT JOA SpfOy puv SpUOdeS UT SBUOS Xa[AUIOD So10}S DALIP YSIP , 74 € ISCJ-CNIN - -jSIOZISOUJUAS - -stuoydAjod of 0} dn skeyd A[snoourynuls ‘spouueYd [IW 9T JO duo 0} pousisse oq -ABUL YORI] YOR ‘syous) oruoydAjod ‘snoouelnurs 7¢ SuTeJUOS ssouUaNbas QO] OY} JO YORA e - -‘SJONUOS ATWOOT pur ‘GNIMAY ‘GaVM OA -LSVd ‘GYOOde AOLS ‘AV Td YIM Jopsocas ade} Yowsj-N[NU O} eps st UOTLISdO @ -LOPNOUT SaINjeoy s[quyIeUlss AUB S.JJ ‘OSN pue UIes] 0} o[duns A[suIzeUe JOA ‘PnJsomod APOUIOITXO -St 1] “UeIOIsNUL feUOIssajoid oY} 10 JOO} soUBULIOJIJAd pue UOTIsOduIOS 11e-dY1-JO-9}e)s B SI IONUANbDaguUT] ay - -JOps1odady soUINbIS [GTI YVAL ZE -Jgouanbaguury oy +18720 Oxnard Street, Tarzana, CA 91356 +(818) 708-8131 TELEX #298949 LINN UR \ No newline at end of file diff --git a/tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt/hocr.bin index 518ac636..14333b45 100644 --- a/tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt/hocr.bin @@ -9,7 +9,7 @@ -

+

diff --git a/tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/pdf.bin b/tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/pdf.bin index aa26441b..2f4f13b8 100644 Binary files a/tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/txt.bin b/tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/txt.bin index 137fef56..d3c2e860 100644 --- a/tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/txt.bin +++ b/tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/txt.bin @@ -1,124 +1,123 @@ -2A NNI‘I 6F6867# XATALL IE18-80L (818) +The LinnSequencer +32 Track MIDI Sequence Recorder -9SEI6 VO “BUBZIRY, “J0aNS PIPUXO OZLEI -“Uy ‘soTUOMOI,q UUrT +The LinnSequencer is a state-of-the-art composition and performance tool for the professional musician. It is -uu +extremely powerful, yet amazingly simple to learn and use. It’s many remarkable features include: -“‘SUOS B UIJIM pasueyo oq ABU pue ‘posn oq AWW AYN IVNOIS AWLL AUV -“parlsop Jr SUOTIISUBI} YIOOUIS YIM “BoueNbas eB OJUI pourtueIZOId 9q ABU SFONWHO OdINAL e +¢ Operation is similar to multi-track tape recorder with PLAY, STOP, RECORD, FAST +FORWARD, REWIND, and LOCATE controls. -‘uonng OdNAL dV L 9) uO +e Each of the 100 sequences contains 32 simultaneous, polyphonic tracks. Each track may +be assigned to one of 16 MIDI channels. Simultaneously plays up to 16 polyphonic -sojou Jayienb Suiddy} Aq 10 ‘syUSTIOIOUI oINUTIAI-J8g-Jesg & JO sys} UL ofquisn(pe ‘ATTeouIAUINU paiajus oq ABU OdINALL e +synthesizers! -(jouer doup u3a9) +¢ Ultra-fast 3%” disk drive stores complex songs in seconds and holds over 110,000 notes -“puooes Jed souely O€ 10 “SZ “pz 18 [LVAG-MAd-SHN VU 10 ALOANIWAAd-SLVAd U! patyoeds aq kewl OAL « -‘uoTe1odo [SVx JO} Aj[eusoyUT JoyndUIOd 11g 9] 98108 ZHI 8g ‘poeds-ysry Bann soz] e +per disk! -"9U0} DUAS 0006 UUL] Jo wNIqUUr] prepue}s 0} OUAS [ITAA © +¢ One or all tracks may be TRANSPOSED at the touch of a key. +e Exclusive real-time ERASE function makes editing FAST. +* Exclusive REPEAT function automatically repeats any held notes at a pre-selected -“ONYBA 9}OU poloapes Aue Je sas—nd jndyno 07 pewureigold 3q ACW SL Ad LNO YADONAL OML +rhythmic value. -"ALVOOT 10 GOLS/AV 1d ‘LWddad “ASV +¢ TIMING CORRECTION works during playback and operates without ‘chopping’ notes. -SUIpNpoUr ‘suOTIOUN] posn A[UOUILUOS 94] JO AUBUT [O1]UOD AJ9]OWIAI 0} PousIsse oq ACUI ST AdNI HOLIMSLOO OME « -“SUIPIONAI I[IYM P2sesd JOU Iv $3}OU BUTISIXO—ZUIPIOIA SATON.ASOP-UON +¢ Optional SMPTE time code synchronization. -‘suoneurldxa peuoyippe sdeydsip uowng g1TqH +© Optional remote control. -oy] ‘pepsau JI ‘suoneiodo [ye yYsnosy] NOA sapins ApIespo Avfdsip QO] Joey Z7¢ 9y3—uoeIodo Urea] 0} Ased ‘aus « +Recording a Sequence -jUorel]suowtap & IO} Aepol Jayeap uur’] INOA dag ‘dISHUL -INOA 0} UONUS}]¥ PaplAIPUN INOA SUTJOASp ITY ps pue -p1osai ‘asoduod no Jay 0} pausisap st 1s0uenbesuur’] oy) -Aum Aposiooid $,Jeu], ‘SS9d0Id SATTBS1D OY} YIM SOIOJIOIUT -yey) xo]dwWI0d Os dq JOA9U P[NoUsS osn NOA AZopOuYdE} oy +To record a sequence, simply press RECORD and PLAY, +then play your MIDI keyboard in time to the Sequencer’s +click track. When the sequence loops back around to bar 1, +you’ ll hear what you played—only all timing errors will be -ISTUMOIAUIO?) NOAA UOHISOdWIO) +corrected! (Timing correction may be adjusted or defeated). -"NOSpr] B Oy ‘AONUTJUT yada 0} seq Maz Se] BY] Jas UdAd -uvd NOA ‘palisap JJ ‘souanbes Mou ¥B OVUT sjied ou] [Te Adoo -ATesrewO Ne WI) [IM ONOS ALVAAO JeyIe80} wey} -,deyd,, 0} UOTOUNJ ONOS ALVA ou] asn usy] ‘saouanbes -JENPIAIpUt UI (“949 ‘snJOYD ‘aS1OA) UOTIDIS JIseq Yes -Pl0da1 OF ST ABM JOuIOUY “(812g 666 01 dn) ysnory) ABM +Any additional notes played will be added into the track +— existing notes are not erased while recording! -dU} [fe YORI] YORs p10991 0} ST SUOS B 9789I9 0} ABM SUG, +FAST FORWARD, REWIND, and LOCATE controls +may be used at any time to quickly access any location in +your sequence for spot-recording. To overdub a new part, +select a different track and start recording—while you +record, the first track will play in perfect sync (unless you +MUTE it, or SOLO another track). In this way, up to 32 +tracks may be overdubbed! All MIDI effects are recorded +including pitch bend, modulation, velocity, aftertouch, +sustain pedal, and program changes! -SUOS & SUTVAID +Editing -*suoT}oes poJUBMUN +To erase a wrong note, simply hold ERASE and press +the note to be erased just before it plays in the sequence— +when played back, it will be gone. Notes may also be -SAOUIOI 0} ABM SWS dU} SoyeIodo SUV ALATAaG +added, erased, or changed using the SINGLE STEP func- +tion. To overdub notes at specific points within a sequence, -“OBPLIq dy} PUB SNIOY PUOdAS dT]] Ud9MIAQ SIDA ISI +Additional Features -ay) Jo Adoo B JJasuT WYSE NOAA ‘afdwexs 10.f ‘UO JUSIN]JIP +simply use LOCATE, FAST FORWARD, or REWIND to +find the desired bar number, then start recording. -B IO aouaNbas sues OY} UI—JOY OUP 0} UOTIEIO] 9UO WOT] -$1Bq JAOUI OF NOA sMOTIe WOTIOUNS AdOO/IMASNI OULL +The INSERT/COPY function allows you to move bars +from one location to another—in the same sequence or a +different one. For example, you might insert a copy of the +first verse between the second chorus and the bridge. +DELETE BARS operates the same way to remove +unwanted sections, -‘SUIPIONAI JIVIS Udy) “OQuINU eq porisop ay} puy +Creating a Song -0} CNIMAY 10 ‘CYVM Od LSWA “AEVOOT esn Apduns +One way to create a song is to record each track all the +way through (up to 999 bars). Another way is to record +each basic section (verse, chorus, etc.) in individual +sequences, then use the CREATE SONG function to “chain” +them together. CREATE SONG will then automatically +copy all the parts into a new sequence. If desired, you can +even set the last few bars to repeat infinitely, for a fadeout. -sainjeay [PUOHIPPY +Composition Without Compromise -‘gouanbas & UTYIIM s]UTOd a1y1dads 3¥ $9100 QnPIOAO OL "UOT} --ouns dALLS ATONIS 24) Suisn pasueyo Jo ‘pasesa ‘pappe -aq osye ABUT S9]ON ‘U0 9q ]IIM 1 “yoeq podeyd uayM -—aouanbas oy] ul skeyd 71 a10J9q Isnf posers oq 0} d]0U ayy -ssaid pue ASvwug ploy Aydunis ‘jou Suomm & aseso OL +The technology you use should never be so complex that +it interferes with the creative process. That’s precisely why +the LinnSequencer is designed to let you compose, record +and edit while devoting your undivided attention to your +music. See your Linn dealer today for a demonstration! -sunipa +* Simple, easy to learn operation—the 32 character LCD display clearly guides you through all operations. If needed, the -jsesdueyo ureisoid pue ‘fepod ureysns -‘yonoplalje ‘AWOOTOA ‘UOTyeTNpow ‘pusg youd Surpnyour -pep10del are $199JJ2 TCTIN [WV iPeqqnpseao aq Aeur syoen -Ze 07 dn ‘Kem sie Uy *(foeI} JOyOUR OJOS 10 ALLAN -NOA ssofum) duAS yOaysod ul Avy [[IM Yow] ISI 93 “prooar -NOA 3[IYM—SUIPIOIA LIBIS PU YORI) TUdIOTJIP B JOaTas -*y1ed MOU B QNPIsA0 OL, “SuIps0daJ-jods 10} aouanbes mno0k -UI UOHBIO] Aue ssad0e ATYOIND 0} owt} Aue ye pasn aq AvUE -SJONUOD FLIVOOT pur ‘ANIMA ‘CYVMaYOd LSVd -{SUIPIOSAI {IY posesa JOU se So]OU SuTsTXO— -yous} 3U} OUT poppe aq JIM poteyd sajou yeuonippe Auy -*(povesjap 10 poysn{pe oq ABW UOTIIII0D BUTUTT]) j{paqoeLI09 -2q ][IM S1OLIe Sur [fe ATUO—patey]d nod Jey Jedy ]],NOA -‘] req 0] punose yoeq sdoo] sduanbas ay] Udy AA “YOu Yor -§,sa0uaNbas at} O] SUIT) UI preogday [IW] INO Avy usy3 -AV'1d pue (YOON ssoid Ayduus ‘aousnbes & p1o09es OF, +HELP button displays additional explanations. -g0uaNbas & SUIP10I0y] +* Non-destructive recording—existing notes are not erased while recording. +¢ Two FOOTSWITCH INPUTS may be assigned to remotely control many of the commonly used functions, including -‘JONWOD s}JouNaI TeuONdGO e +ERASE, REPEAT, PLAY/STOP, or LOCATE. -"UOTJEZIUOIYUAS OPOS UIT} FLAWS [euondo e +¢ Iwo TRIGGER OUTPUTS may be programmed to output pulses at any selected note value. -‘sou .sulddoys, noyyM sayelodo pue yoegdvyd ZuLINp S¥IOM NOLLOANNYOO ONIWILL e +© Will sync to standard LinnDrum or Linn 9000 sync tone. -‘onqea ory AY +© Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation. +* TEMPO may be specified in BEATS-PER-MINUTE or FRAMES-PER-BEAT at 24, 25, or 30 frames per second, -pojoojes-oid & ye sajou pyoy Aue syeadas ATTeONewWO Ne UOTOUNS [WAdAY OAISNOX e -‘LSVJ SUnIpS soyeu UOTOUN ASV UA OUlN-[eal SAISNIOXY e -‘Koy B JO YONO} 941 12 CASOdSNVALL 0g ABU Syde] [Te 10 9UC e +(even drop frame!) -i ASIP Jed +¢ TEMPO may be entered numerically, adjustable in tenths of a Beat-Per-Minute increments, or by tapping quarter notes -S9}0U OOO‘OTT JOA SpfOy puv SpUOdeS UT SBUOS Xa[AUIOD So10}S DALIP YSIP , 74 € ISCJ-CNIN +on the TAP TEMPO button. -jSIOZISOUJUAS +¢ TEMPO CHANGES may be programmed into a sequence, with smooth transitions if desired. +¢ Any TIME SIGNATURE may be used, and may be changed within a song. -stuoydAjod of 0} dn skeyd A[snoourynuls ‘spouueYd [IW 9T JO duo 0} pousisse oq -ABUL YORI] YOR ‘syous) oruoydAjod ‘snoouelnurs 7¢ SuTeJUOS ssouUaNbas QO] OY} JO YORA e +linn +Linn Electronics, Inc. -‘SJONUOS ATWOOT pur ‘GNIMAY ‘GaVM OA -LSVd ‘GYOOde AOLS ‘AV Td YIM Jopsocas ade} Yowsj-N[NU O} eps st UOTLISdO @ -LOPNOUT SaINjeoy s[quyIeUlss AUB S.JJ ‘OSN pue UIes] 0} o[duns A[suIzeUe JOA ‘PnJsomod APOUIOITXO -St 1] “UeIOIsNUL feUOIssajoid oY} 10 JOO} soUBULIOJIJAd pue UOTIsOduIOS 11e-dY1-JO-9}e)s B SI IONUANbDaguUT] ay - -JOps1odady soUINbIS [GTI YVAL ZE -Jgouanbaguury oy +18720 Oxnard Street, Tarzana, CA 91356 +(818) 708-8131 TELEX #298949 LINN UR \ No newline at end of file diff --git a/tests/cache/ccitt/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/ccitt/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin index d06920de..e7d83825 100644 --- a/tests/cache/ccitt/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/ccitt/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin @@ -9,7 +9,7 @@ -

+

diff --git a/tests/cache/ccitt/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/ccitt/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin index 88bb9fc9..a064b5ab 100644 Binary files a/tests/cache/ccitt/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/ccitt/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin deleted file mode 100644 index 750ad232..00000000 Binary files a/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin and /dev/null differ diff --git a/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin b/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin deleted file mode 100644 index 16b617e5..00000000 --- a/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin +++ /dev/null @@ -1 +0,0 @@ -Tesseract Open Source OCR Engine v4.1.1 with Leptonica diff --git a/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stdout.bin b/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stdout.bin deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin b/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin deleted file mode 100644 index 25fdded2..00000000 --- a/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin +++ /dev/null @@ -1,13 +0,0 @@ -Portez ce vieux whisky au juge -blond qui fume sur son Ile -interieure, a cöte de l'alcöve -ovoide, oU les büches se -consument dans l'ätre, ce qui -lui permet de penser & la -caenogenese de |'etre dont il -est question dans la cause -ambigu& entendue a MoY, dans -un capharnaüm qui, pense-t-il, -diminue ca et la la qualite de son -ceuvre. - \ No newline at end of file diff --git a/tests/cache/graph_ocred/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/graph_ocred/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin index 74e98d6d..3597925e 100644 Binary files a/tests/cache/graph_ocred/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/graph_ocred/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/graph_ocred/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin b/tests/cache/graph_ocred/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin index d951032a..12fa5df9 100644 --- a/tests/cache/graph_ocred/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin +++ b/tests/cache/graph_ocred/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin @@ -1,20 +1,38 @@ Replacement of "creationism" with "intelligent design" + + + + + + + + + + + + + + + 120 - - - -—@— "Creation" and "creationist" -—®@-— "Intelligent design" -and "design proponent" - -Word count - - - - - - - - +100 - +Cc 80 += +© +oO 60 —@— "Creation" and "creationist" +5 —@— "Intelligent design" += and "design proponent" +40 4 +20 - +—@ © +0 e- T T T | rE © +3) 6) Ay AN y 9) oN +a i +\ oe a) \ YL S S +3 ad Ss x x x. s? +3 Ri oe a? ace Nc) os +J we % si we oe e +eS SS S S oe eS is” +oe sO o Q Q Q Q \ No newline at end of file diff --git a/tests/cache/jbig2/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/jbig2/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin index 946585c5..fe88dea6 100644 --- a/tests/cache/jbig2/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/jbig2/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin @@ -9,467 +9,459 @@ -

-
-

- - 4ist - ConGREss, - } - SENATE. - { - Ex. - Doc, +

+
+

+ + 41st + CONGRESS; + | + SENATE. - - 3d - Session. - No. - 25. + + 3d + Session. + }

-
-

- - MESSAGE +

+

+ + MESSAGE

-
-

- - OF - THE +

+

+ + OF + THE

-
-

- - PRESIDENT - OF - THE - UNITED - STATES, +

+

+ + PRESIDENT + OF + THE + UNITED + STATES,

-
-

- - COMMUNICATING +

+

+ + COMMUNICATING

-
-

- - A - copy - of - regulations - for - the - consular - courts - of - the - United - States - in - Japan, +

+

+ + A + copy + of + regulations + for + the + consular + courts + of + the + United + States + in + Japan, - - decreed - and - issued - by - the - minister - of - the - United - States - in - that - country. + + decreed + and + issued + by + the + minister + of + the + United + States + in + that + country.

-
-

- - JANUARY - 27, - 1871,—Read, - referred - to - the - Committee - on - Commerce, - and - ordered - to - be +

+

+ + January + 27, + 1871,—Read, + referred + to + the + Committee + on + Commerce, + and + ordered + to + be - - printed. + + printed.

-
-

- - To - the - Senate - and - House - of - Representatives - : +

+

+ + To + the + Senate + and + House + of + Representatives + :

-

- - I - transmit - herewith, - for - the - consideration - of - Congress, - a - report - from +

+ + I + transmit + herewith, + for + the + consideration + of + Congress, + a + report + from - - the - Secretary - of - State, - and - the - papers - which - accompanied - it, - concern- + + the + Secretary + of + State, + and + the + papers + which + accompanied + it, + concern- - - ing - regulations - for - the - consular - courts - of - the - United - States - in - Japan. + + ing + regulations + for + the + consular + courts + of + the + United + States + in + Japan.

-

- - U. - 8. - GRANT. +

+ + U. + 8. + GRANT.

-

- - ‘WASHINGTON, - January - 27, - 1871. +

+ + ‘WASHINGTON, + January + 27, + 1871.

-
-

- - DEPARTMENT - OF - STATE, +

+

+ + DEPARTMENT + OF + STATE, - - Washington, - January - 26, - 1870, + + . + Washington, + January + 26, + 1870,

-

- - The - Secretary - of - State - has - the - honor - to - submit - herewith, - for - revision +

+ + The + Secretary + of + State + has + the + honor + to + submit + herewith, + for + revision - - by - Congress, - in - conformity - with - the - provisions - of - section - 6 - of - the - act + + by + Congress, + in + conformity + with + the + provisions + of + section + 6 + of + the + act - - approved - 22d - of - June, - 1860, - a - copy - of - “regulations - for - the - consular + + approved + 22d + of + June, + 1860, + a + copy + of + “regulations + for + the + consular - - courts - of - the - United - States - in - Japan,” - decreed - and - issued - by - C. - BE. + + courts + of + the + United + States + in + Japan,” + decreed + and + issued + by + C. + E. - - De - Long, - the - minister - of - the - United - States - in - that - country, - in - Septem- + + De + Long, + the + miniater + of + the + United + States + in + that + country, + in + Septem- - - ber, - 1870; - and - also - the - papers - mentioned - in - the - subjoined - list, - which, + + ber, + 1870; + and + also + the + papers + mentioned + in + the + subjoined + list, + which, - - contain - suggestions - on - the - subject - thereof. + + contain + suggestions + on + the + subject + thereof.

-

- - A - copy - of - Article - XXVI - of - the - consular - regulations - is - also - submitted, +

+ + A + copy + of + Article + XXVI + of + the + consular + regulations + is + also + submitted, - - and - the - Secretary - of - State - respectfully - suggests, - for - the - consideration + + and + the + Secretary + of + State + respectfully + suggests, + for + the + consideration - - of - Congress, - the - propriety - of - limiting - the - power - of - ministers - to - make + + of + Congress, + the + propriety + of + limiting + the + power + of + ministers + to + make - - decrees - and - regulation, - in - the - sense - in - which - it - is - limited - by - paragraph + + decrees + and + regulation, + in + the + sense + in + which + it + is + limited + by + paragraph - - 431 - of - the - article - before - named—that - is, - “to - acts - necessary - to - organize + + 431 + of + the + article + before + named—that + is, + “to + acts + necessary + to + organize - - and - give - efficiency - to - the - courts - created - by - the - act.” + + and + give + efficiency + to + the + courts + created + by + the + act.”

-

- - Respectfully - submitted. +

+ + Respectfully + submitted. + :

-

- - HAMILTON - FISH. +

+ + HAMILTON + FISH.

-
-

- - The - PRESIDENT, +

+

+ + The + PRESIDENT,

-
-

- - List - of - accompanying - papers. +

+

+ + List + of + accompanying + papers.

-
-

- - 1, - Regulations - for - the - consular - courts - of - the - United - States - in - Japan. +

+

+ + 1, + Regulations + for + the + consular + courts + of + the + United + States + in + Japan. - - 2, - Mr. - Fish - to - Mr. - De - Long, - September - 10, - 1870, + + 2. + Mr. + Fish + to + Mr. + De + Long, + September + 10, + 1870. + .

- - + +

-
-

- - - -

-
-
-

- - +

+

+ +

diff --git a/tests/cache/jbig2/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/txt.bin b/tests/cache/jbig2/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/txt.bin index a450f78e..c8e99baa 100644 --- a/tests/cache/jbig2/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/txt.bin +++ b/tests/cache/jbig2/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/txt.bin @@ -1,5 +1,5 @@ -4ist ConGREss, } SENATE. { Ex. Doc, -3d Session. No. 25. +41st CONGRESS; | SENATE. +3d Session. } MESSAGE @@ -12,7 +12,7 @@ COMMUNICATING A copy of regulations for the consular courts of the United States in Japan, decreed and issued by the minister of the United States in that country. -JANUARY 27, 1871,—Read, referred to the Committee on Commerce, and ordered to be +January 27, 1871,—Read, referred to the Committee on Commerce, and ordered to be printed. To the Senate and House of Representatives : @@ -26,13 +26,13 @@ U. 8. GRANT. ‘WASHINGTON, January 27, 1871. DEPARTMENT OF STATE, -Washington, January 26, 1870, +. Washington, January 26, 1870, The Secretary of State has the honor to submit herewith, for revision by Congress, in conformity with the provisions of section 6 of the act approved 22d of June, 1860, a copy of “regulations for the consular -courts of the United States in Japan,” decreed and issued by C. BE. -De Long, the minister of the United States in that country, in Septem- +courts of the United States in Japan,” decreed and issued by C. E. +De Long, the miniater of the United States in that country, in Septem- ber, 1870; and also the papers mentioned in the subjoined list, which, contain suggestions on the subject thereof. @@ -43,7 +43,7 @@ decrees and regulation, in the sense in which it is limited by paragraph 431 of the article before named—that is, “to acts necessary to organize and give efficiency to the courts created by the act.” -Respectfully submitted. +Respectfully submitted. : HAMILTON FISH. @@ -52,9 +52,7 @@ The PRESIDENT, List of accompanying papers. 1, Regulations for the consular courts of the United States in Japan. -2, Mr. Fish to Mr. De Long, September 10, 1870, - - +2. Mr. Fish to Mr. De Long, September 10, 1870. . diff --git a/tests/cache/jbig2/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/jbig2/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin index 96c1415a..0e48a8e7 100644 Binary files a/tests/cache/jbig2/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/jbig2/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/lichtenstein/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/lichtenstein/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin index fc7f0c29..278faafb 100644 --- a/tests/cache/lichtenstein/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/lichtenstein/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin @@ -9,7 +9,7 @@ -
+

diff --git a/tests/cache/lichtenstein/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/lichtenstein/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin index 2987783d..4a0061fc 100644 Binary files a/tests/cache/lichtenstein/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/lichtenstein/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/manifest.jsonl b/tests/cache/manifest.jsonl index f482aaaa..a0124d50 100644 --- a/tests/cache/manifest.jsonl +++ b/tests/cache/manifest.jsonl @@ -1,44 +1,43 @@ -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/francais.pdf", "args": ["-l", "deu", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__--psm__7__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "--psm", "7", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__--psm__7__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "--psm", "7", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/aspect.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/ccitt.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/jbig2.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/lichtenstein.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/palette.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/2400dpi.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/aspect.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/ccitt.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/graph_ocred.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/jbig2.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/lichtenstein.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/palette.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/poster.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000005_ocr.png__000005_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000005_ocr.png", "$TMPDIR/000005_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000005_ocr.png__000005_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000005_ocr.png", "$TMPDIR/000005_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000006_ocr.png__000006_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000006_ocr.png", "$TMPDIR/000006_ocr_hocr", "hocr", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__eng__000006_ocr.png__000006_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000006_ocr.png", "$TMPDIR/000006_ocr_tess", "pdf", "txt"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__osd__--psm__0__000001_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000001_rasterize_preview.jpg", "stdout"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__osd__--psm__0__000001_rasterize_preview.jpg__stdout", "sourcefile": "resources/poster.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000001_rasterize_preview.jpg", "stdout"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__osd__--psm__0__000002_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000002_rasterize_preview.jpg", "stdout"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__osd__--psm__0__000003_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000003_rasterize_preview.jpg", "stdout"]} -{"tesseract_version": "4.1.1", "platform": "macOS-10.14.6-x86_64-i386-64bit", "python": "3.9.0", "argv_slug": "__-l__osd__--psm__0__000004_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000004_rasterize_preview.jpg", "stdout"]} \ No newline at end of file +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/ccitt.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/graph_ocred.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000005_ocr.png__000005_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000005_ocr.png", "$TMPDIR/000005_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000006_ocr.png__000006_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000006_ocr.png", "$TMPDIR/000006_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000005_ocr.png__000005_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000005_ocr.png", "$TMPDIR/000005_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000006_ocr.png__000006_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000006_ocr.png", "$TMPDIR/000006_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__--psm__7__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "--psm", "7", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__--psm__7__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "--psm", "7", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/jbig2.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/2400dpi.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/palette.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/jbig2.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/lichtenstein.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/palette.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/ccitt.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/lichtenstein.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/aspect.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/aspect.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__osd__--psm__0__000004_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000004_rasterize_preview.jpg", "stdout"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__osd__--psm__0__000001_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000001_rasterize_preview.jpg", "stdout"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__osd__--psm__0__000002_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000002_rasterize_preview.jpg", "stdout"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__osd__--psm__0__000003_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000003_rasterize_preview.jpg", "stdout"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/poster.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__osd__--psm__0__000001_rasterize_preview.jpg__stdout", "sourcefile": "resources/poster.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000001_rasterize_preview.jpg", "stdout"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_hocr", "hocr", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-25-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_hocr", "hocr", "txt"]} diff --git a/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin index 38166827..59efd297 100644 --- a/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin @@ -9,7 +9,7 @@ -

+

@@ -19,22 +19,22 @@

- + THEY - TIP-TOED - ALONG. + TIP-TOED + ALONG.

-
+

- - ee + + al . - - Se - We + + SE + We went tip-toeing along @@ -46,23 +46,23 @@

- + the trees back towards the - end + end of the - + widow’s garden, stooping down so - as + as the @@ -70,82 +70,82 @@ wouldn’t scrape our - heads. + heads. When we - was - passing + was + passing by the kitchen - + I fell over - a - root - and - made - a + a + root + and + made + a noise. - - We - scrouched + + We + crouched down - and - laid - still. + and + laid + still. - Miss - Watson’s - big - nigger, + Miss + Watson’s + big + nigger, named - + Jim, was setting in the - kitchen - door - ; + kitchen + door + ; we - could + could see him pretty clear, because - + there was - a - light + a + light behind - him. - He + him. + He - got + got up - and - stretched + and + stretched his neck out - about + about a minute, listening. @@ -158,8 +158,8 @@

- - “Who + + “Who dah?”

@@ -170,15 +170,15 @@ listened some more; - then - he + then + he - come - tip-toeing - down - and_ - stood + come + tip-toeing + down + and_ + stood right @@ -195,10 +195,10 @@ Well, likely it - was - min- + was + min- - + utes and minutes @@ -207,22 +207,22 @@ warn’t a - - sound, + + sound, and we - all - there - so - close + all + there + so + close - together. - There + together. + There was - a + a place - on + on my @@ -240,14 +240,14 @@

dasn’t - scratch + scratch it; and then - my - ear + my + ear begun - to + to itch; and next @@ -262,14 +262,14 @@ shoulders. Seemed like - I’d + I’d die if - I + I couldn’t scratch. Well, - I’ve + I’ve

@@ -289,13 +289,13 @@

- Tf + If you are with the - quality, - or + quality, + or at a @@ -317,7 +317,7 @@ sleepy—if you are - anywheres + anywheres

diff --git a/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/stderr.bin b/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/stderr.bin index 25cb8633..e2b14846 100644 --- a/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/stderr.bin +++ b/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/stderr.bin @@ -1,2 +1,2 @@ Tesseract Open Source OCR Engine v4.1.1 with Leptonica -Detected 60 diacritics +Detected 98 diacritics diff --git a/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/txt.bin b/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/txt.bin index 1f008c8e..90514d84 100644 --- a/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/txt.bin +++ b/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/txt.bin @@ -2,15 +2,15 @@ THEY TIP-TOED ALONG. -ee . -Se We went tip-toeing along a path amongst +al . +SE We went tip-toeing along a path amongst the trees back towards the end of the widow’s garden, stooping down so as the branches wouldn’t scrape our heads. When we was passing by the kitchen I fell over a root and made a noise. -We scrouched down and laid still. +We crouched down and laid still. Miss Watson’s big nigger, named Jim, was setting in the kitchen door ; we could see him pretty clear, because @@ -35,7 +35,7 @@ tween my shoulders. Seemed like I’d die if I couldn’t scratch. Well, I’ve noticed that thing plenty of times since. -Tf you are with the quality, or at a +If you are with the quality, or at a funeral, or trying to go to sleep when you ain’t sleepy—if you are anywheres \ No newline at end of file diff --git a/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin index 66a30325..e5061977 100644 Binary files a/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin b/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin index 25cb8633..e2b14846 100644 --- a/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin +++ b/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin @@ -1,2 +1,2 @@ Tesseract Open Source OCR Engine v4.1.1 with Leptonica -Detected 60 diacritics +Detected 98 diacritics diff --git a/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin b/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin index 1f008c8e..90514d84 100644 --- a/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin +++ b/tests/cache/multipage/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin @@ -2,15 +2,15 @@ THEY TIP-TOED ALONG. -ee . -Se We went tip-toeing along a path amongst +al . +SE We went tip-toeing along a path amongst the trees back towards the end of the widow’s garden, stooping down so as the branches wouldn’t scrape our heads. When we was passing by the kitchen I fell over a root and made a noise. -We scrouched down and laid still. +We crouched down and laid still. Miss Watson’s big nigger, named Jim, was setting in the kitchen door ; we could see him pretty clear, because @@ -35,7 +35,7 @@ tween my shoulders. Seemed like I’d die if I couldn’t scratch. Well, I’ve noticed that thing plenty of times since. -Tf you are with the quality, or at a +If you are with the quality, or at a funeral, or trying to go to sleep when you ain’t sleepy—if you are anywheres \ No newline at end of file diff --git a/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt/hocr.bin index 2c989a38..1a2376ad 100644 --- a/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt/hocr.bin @@ -9,16 +9,16 @@ -
+

- + Replacement - of - "creationism" + of + "creationism" with - "intelligent - design" + "intelligent + design"

@@ -76,21 +76,21 @@

- + 120 - + 100 - - + Cc 80 - + > - + 5 @@ -113,15 +113,15 @@ "design proponent" - + 40 - - + 20 - - + —@— —@® @@ -135,7 +135,7 @@ w ° - + gp) ee) 0 @@ -172,48 +172,49 @@ & s? - - ge - ee - Oo - ss - Ss - Ne - qs + + 30 + Ce + o + Ss + Ss + ° + se - - G - % - S - S - © - S + + © + os + S + S + 2 + S - - Ros - % - se - se - oe - AN + + Ro + % + & + NS + se + oe + AN - Ss - 3s - S - Ss - Ss - Ss - Ss + Ss + 3s + S + Ss + Ss + Ss + Ss - ow? - \O - Xo) - R - g - g - Q + ow? + \O + Xo) + R + g + g + Q

diff --git a/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt/txt.bin b/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt/txt.bin index 97c52199..c5445aab 100644 --- a/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt/txt.bin +++ b/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt/txt.bin @@ -31,9 +31,9 @@ gp) ee) 0 oN g\ gD) op) oO NC) NC) LN eo N N S os o* vs ws os os cs) Re ss & ow x & s? -ge ee Oo ss Ss Ne qs -G % S S © S -Ros % se se oe AN +30 Ce o Ss Ss ° se +© os S S 2 S +Ro % & NS se oe AN Ss 3s S Ss Ss Ss Ss ow? \O Xo) R g g Q \ No newline at end of file diff --git a/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/pdf.bin b/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/pdf.bin index 82d5f141..9d2babe6 100644 Binary files a/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/txt.bin b/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/txt.bin index 97c52199..c5445aab 100644 --- a/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/txt.bin +++ b/tests/cache/multipage/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/txt.bin @@ -31,9 +31,9 @@ gp) ee) 0 oN g\ gD) op) oO NC) NC) LN eo N N S os o* vs ws os os cs) Re ss & ow x & s? -ge ee Oo ss Ss Ne qs -G % S S © S -Ros % se se oe AN +30 Ce o Ss Ss ° se +© os S S 2 S +Ro % & NS se oe AN Ss 3s S Ss Ss Ss Ss ow? \O Xo) R g g Q \ No newline at end of file diff --git a/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt/hocr.bin index 182a705a..c4f23a68 100644 --- a/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt/hocr.bin @@ -9,189 +9,206 @@ -
-
-

- - Replacement - of - "creationism" - with - "intelligent - design" +

+
+

+ + Replacement + of + "creationism" + with + "intelligent + design"

-
-

- - +

+

+ +

-
-

- - +

+

+ + - - + +

-
-

- - +

+

+ +

-
-

- - +

+

+ +

-
-

- - +

+

+ +

-
-

- - +

+

+ +

-
-

- - +

+

+ +

-
-

- - 120 +

+

+ + 120 - - 100 - 4 + + 100 + - - - = - 80 + + Cc + 80 - - — + + — - - S + + © - - _ - 6047 - —@— - "Creation" - and - "creationist" + + oO + 607 + —@— + "Creation" + and + "creationist" - - 5 - —@— - "Intelligent - design" + + 5 + —@— + "Intelligent + design" - - and - "design - proponent" + + and + "design + proponent" - - S - «4 + + S + 40- - - 20 - - + + 20 + - - - 0 - oe - I - T - T - T - T - © + + 0 + e- + T + T + T + I + Tv + ® - - 3) - ©) - Ay - Ay - Ay - 9 - o>) + + 3) + SG) + AY + AY + Ay + 9) + 2) - - ee - ow - oe - oe - Cs - Cs - eS + + ee + oP + ce + oe + oP + oP + ” - - RQ - Q - R - R - XR - Q - R + + XR + XR + R + XR + XY + R + XR - - & - es - o - a - al - & - eo + + ~\ + oe + @ + \ + wl + ae + eo - - 3 - e - oe - we - a? - i) - as? + + oe + ee + ~) + se? + a? + N? + & - - 3 - 4? - 3 - & - oe - & - & + + & + cf + O + S + % + ed + ol - - oe - ww - 6 - e - Qe - Qe - Qe + + S + &- + % + os + es + se + ae + + + ~ + os + ss + Ss + eS + eS + Ss + + + cx? + sO + oS + Q + Q + Q

diff --git a/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt/txt.bin b/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt/txt.bin index 39cbe785..d74d5ad9 100644 --- a/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt/txt.bin +++ b/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt/txt.bin @@ -16,21 +16,23 @@ Replacement of "creationism" with "intelligent design" 120 -100 4 -= 80 +100 - +Cc 80 — -S -_ 6047 —@— "Creation" and "creationist" +© +oO 607 —@— "Creation" and "creationist" 5 —@— "Intelligent design" and "design proponent" -S «4 +S 40- 20 - -0 oe I T T T T © -3) ©) Ay Ay Ay 9 o>) -ee ow oe oe Cs Cs eS -RQ Q R R XR Q R -& es o a al & eo -3 e oe we a? i) as? -3 4? 3 & oe & & -oe ww 6 e Qe Qe Qe +0 e- T T T I Tv ® +3) SG) AY AY Ay 9) 2) +ee oP ce oe oP oP ” +XR XR R XR XY R XR +~\ oe @ \ wl ae eo +oe ee ~) se? a? N? & +& cf O S % ed ol +S &- % os es se ae +~ os ss Ss eS eS Ss +cx? sO oS Q Q Q \ No newline at end of file diff --git a/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/pdf.bin b/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/pdf.bin index a5591cce..46f15799 100644 Binary files a/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/txt.bin b/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/txt.bin index 39cbe785..d74d5ad9 100644 --- a/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/txt.bin +++ b/tests/cache/multipage/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/txt.bin @@ -16,21 +16,23 @@ Replacement of "creationism" with "intelligent design" 120 -100 4 -= 80 +100 - +Cc 80 — -S -_ 6047 —@— "Creation" and "creationist" +© +oO 607 —@— "Creation" and "creationist" 5 —@— "Intelligent design" and "design proponent" -S «4 +S 40- 20 - -0 oe I T T T T © -3) ©) Ay Ay Ay 9 o>) -ee ow oe oe Cs Cs eS -RQ Q R R XR Q R -& es o a al & eo -3 e oe we a? i) as? -3 4? 3 & oe & & -oe ww 6 e Qe Qe Qe +0 e- T T T I Tv ® +3) SG) AY AY Ay 9) 2) +ee oP ce oe oP oP ” +XR XR R XR XY R XR +~\ oe @ \ wl ae eo +oe ee ~) se? a? N? & +& cf O S % ed ol +S &- % os es se ae +~ os ss Ss eS eS Ss +cx? sO oS Q Q Q \ No newline at end of file diff --git a/tests/cache/multipage/__-l__eng__000005_ocr.png__000005_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/multipage/__-l__eng__000005_ocr.png__000005_ocr_hocr__hocr__txt/hocr.bin index 19e890e9..2850399b 100644 --- a/tests/cache/multipage/__-l__eng__000005_ocr.png__000005_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/multipage/__-l__eng__000005_ocr.png__000005_ocr_hocr__hocr__txt/hocr.bin @@ -9,7 +9,7 @@ -
+

diff --git a/tests/cache/multipage/__-l__eng__000005_ocr.png__000005_ocr_tess__pdf__txt/pdf.bin b/tests/cache/multipage/__-l__eng__000005_ocr.png__000005_ocr_tess__pdf__txt/pdf.bin index 99d3c4eb..aac063e6 100644 Binary files a/tests/cache/multipage/__-l__eng__000005_ocr.png__000005_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/multipage/__-l__eng__000005_ocr.png__000005_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/multipage/__-l__eng__000006_ocr.png__000006_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/multipage/__-l__eng__000006_ocr.png__000006_ocr_hocr__hocr__txt/hocr.bin index 3e3c4012..e47a20ce 100644 --- a/tests/cache/multipage/__-l__eng__000006_ocr.png__000006_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/multipage/__-l__eng__000006_ocr.png__000006_ocr_hocr__hocr__txt/hocr.bin @@ -9,7 +9,7 @@ -

+

diff --git a/tests/cache/multipage/__-l__eng__000006_ocr.png__000006_ocr_tess__pdf__txt/pdf.bin b/tests/cache/multipage/__-l__eng__000006_ocr.png__000006_ocr_tess__pdf__txt/pdf.bin index 4a339f17..5e624236 100644 Binary files a/tests/cache/multipage/__-l__eng__000006_ocr.png__000006_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/multipage/__-l__eng__000006_ocr.png__000006_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/palette/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/palette/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin index fd60c85f..ca3a7c38 100644 --- a/tests/cache/palette/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/palette/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin @@ -9,7 +9,7 @@ -

+

diff --git a/tests/cache/palette/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/palette/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin index 55a7fbdf..3d7abbbf 100644 Binary files a/tests/cache/palette/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/palette/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/poster/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/poster/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin index 33a54256..1b2b8e72 100644 Binary files a/tests/cache/poster/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/poster/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/poster/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin b/tests/cache/poster/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin index b3381922..8a68189b 100644 --- a/tests/cache/poster/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin +++ b/tests/cache/poster/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin @@ -97,7 +97,7 @@ ERASE, REPEAT, PLAY/STOP, or LOCATE. ¢ Two TRIGGER OUTPUTS may be programmed to output pulses at any selected note value. -e Will sync to standard LinnDrum or Linn 9000 sync tone. +@ Will sync to standard LinnDrum or Linn 9000 sync tone. © Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation. ¢ TEMPO may be specified in BEATS-PER-MINUTE or FRAMES-PER-BEAT at 24, 25, or 30 frames per second, diff --git a/tests/cache/skew/__-l__eng__--psm__7__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/skew/__-l__eng__--psm__7__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin index 43520b39..58eb4e7a 100644 --- a/tests/cache/skew/__-l__eng__--psm__7__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/skew/__-l__eng__--psm__7__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin @@ -9,7 +9,7 @@ -

+
diff --git a/tests/cache/skew/__-l__eng__--psm__7__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/skew/__-l__eng__--psm__7__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin index 31627deb..89aa8241 100644 Binary files a/tests/cache/skew/__-l__eng__--psm__7__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/skew/__-l__eng__--psm__7__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/skew/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin b/tests/cache/skew/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin index 17df1412..9e241235 100644 --- a/tests/cache/skew/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin +++ b/tests/cache/skew/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin @@ -9,7 +9,7 @@ -
+

diff --git a/tests/cache/skew/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/skew/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin index b143cc9b..1499489e 100644 Binary files a/tests/cache/skew/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin and b/tests/cache/skew/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/conftest.py b/tests/conftest.py index 9a059c83..f7874255 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -69,6 +69,11 @@ def outpdf(tmp_path): return tmp_path / 'out.pdf' +@pytest.fixture(scope="function") +def outtxt(tmp_path): + return tmp_path / 'out.txt' + + @pytest.fixture(scope="function") def no_outpdf(tmp_path): """This just documents the fact that a test is not expected to produce diff --git a/tests/test_ghostscript.py b/tests/test_ghostscript.py index 0907b819..28ecfe26 100644 --- a/tests/test_ghostscript.py +++ b/tests/test_ghostscript.py @@ -72,7 +72,7 @@ def test_rasterize_rotated(francais, outdir, caplog): with Image.open(outdir / 'out.png') as im: assert im.size == (target_size[1], target_size[0]) - assert im.info['dpi'] == (forced_dpi[1], forced_dpi[0]) + assert im.info['dpi'] == forced_dpi.flip_axis() def test_gs_render_failure(resources, outpdf): diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 5af7f610..9ab133be 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -16,6 +16,8 @@ from ocrmypdf import helpers from .conftest import running_in_docker +needs_symlink = pytest.mark.skipif(os.name == 'nt', reason='needs posix symlink') + class TestSafeSymlink: def test_safe_symlink_link_self(self, tmp_path, caplog): @@ -27,6 +29,7 @@ class TestSafeSymlink: with pytest.raises(FileExistsError): helpers.safe_symlink(tmp_path / 'input', tmp_path / 'regular_file') + @needs_symlink def test_safe_symlink_relink(self, tmp_path): (tmp_path / 'regular_file_a').touch() (tmp_path / 'regular_file_b').write_bytes(b'ABC') @@ -77,6 +80,7 @@ class TestFileIsWritable: def test_plain(self, non_existent): assert helpers.is_file_writable(non_existent) + @needs_symlink def test_symlink_loop(self, tmp_path): loop = tmp_path / 'loop' loop.symlink_to(loop) diff --git a/tests/test_image_input.py b/tests/test_image_input.py index 78ad5fd1..bbce6011 100644 --- a/tests/test_image_input.py +++ b/tests/test_image_input.py @@ -87,4 +87,4 @@ def test_jpeg_in_jpeg_out(resources, outpdf): 'tests/plugins/tesseract_noop.py', ) with pikepdf.open(outpdf) as pdf: - assert next(pdf.pages[0].images.values()).Filter == pikepdf.Name.DCTDecode + assert next(iter(pdf.pages[0].images.values())).Filter == pikepdf.Name.DCTDecode diff --git a/tests/test_main.py b/tests/test_main.py index 0d827fa3..5c326b79 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -701,7 +701,7 @@ def test_sidecar_pagecount(resources, outpdf): pdfinfo = PdfInfo(resources / '3small.pdf') num_pages = len(pdfinfo) - with open(sidecar, 'r', encoding='utf-8') as f: + with open(sidecar, encoding='utf-8') as f: ocr_text = f.read() # There should a formfeed between each pair of pages, so the count of @@ -722,7 +722,7 @@ def test_sidecar_nonempty(resources, outpdf): 'tests/plugins/tesseract_cache.py', ) - with open(sidecar, 'r', encoding='utf-8') as f: + with open(sidecar, encoding='utf-8') as f: ocr_text = f.read() assert 'the' in ocr_text @@ -745,14 +745,13 @@ def test_pdfa_n(pdfa_level, resources, outpdf): assert pdfa_info['conformance'] == f'PDF/A-{pdfa_level}B' -@pytest.mark.skipif( - PIL.__version__ < '5.0.0', reason="Pillow < 5.0.0 doesn't raise the exception" -) -@pytest.mark.slow -def test_decompression_bomb(resources, outpdf): +def test_decompression_bomb_error(resources, outpdf): p, _out, err = run_ocrmypdf(resources / 'hugemono.pdf', outpdf) - assert 'decompression bomb' in err + assert 'decompression bomb' in err and '--max-image-mpixels' in err + +@pytest.mark.slow +def test_decompression_bomb_succeeds(resources, outpdf): p, _out, err = run_ocrmypdf( resources / 'hugemono.pdf', outpdf, '--max-image-mpixels', '2000' ) @@ -881,3 +880,29 @@ def test_image_dpi_threshold(resources, outpdf): 'tests/plugins/tesseract_noop.py', ) assert outpdf.exists() + + +def test_outputtype_none_bad_setup(resources, outpdf): + p, _out, err = run_ocrmypdf( + resources / 'trivial.pdf', + outpdf, + '--output-type=none', + '--plugin', + 'tests/plugins/tesseract_noop.py', + ) + assert p.returncode == ExitCode.bad_args + assert 'Set the output file to' in err + + +def test_outputtype_none(resources, outtxt): + p, _out, err = run_ocrmypdf( + resources / 'trivial.pdf', + os.devnull, + '--output-type=none', + '--sidecar', + outtxt, + '--plugin', + 'tests/plugins/tesseract_noop.py', + ) + assert p.returncode == ExitCode.ok + assert outtxt.exists() diff --git a/tests/test_metadata.py b/tests/test_metadata.py index ebcbf031..cea73dd7 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -287,7 +287,7 @@ def test_srgb_in_unicode_path(tmp_path): def test_kodak_toc(resources, outpdf): - _output = check_ocrmypdf( + check_ocrmypdf( resources / 'kcs.pdf', outpdf, '--output-type', diff --git a/tests/test_page_numbers.py b/tests/test_page_numbers.py index 71f06a0d..d89a3bea 100644 --- a/tests/test_page_numbers.py +++ b/tests/test_page_numbers.py @@ -50,10 +50,6 @@ def test_nonmonotonic_warning(caplog): assert 'out of order' in caplog.text -def test_list_range(): - assert _pages_from_ranges([0, 1, 2]) == {0, 1, 2} - - def test_limited_pages(resources, outpdf): multi = resources / 'multipage.pdf' ocrmypdf.ocr( diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index becef2c4..41cb0f04 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -79,9 +79,7 @@ def test_dpi_needed(image, text, vector, result, rgb_image, outdir): # Input: ('', '', '', '', ''), # Output: - ( - ((1, 5), None), - ), + (((1, 5), None),), ), ( 'no_empty_values', @@ -147,4 +145,4 @@ def test_dpi_needed(image, text, vector, result, rgb_image, outdir): ), ) def test_enumerate_compress_ranges(name, input, output): - assert output == tuple(_pipeline.enumerate_compress_ranges(input)) \ No newline at end of file + assert output == tuple(_pipeline.enumerate_compress_ranges(input)) diff --git a/tests/test_validation.py b/tests/test_validation.py index fd4d6fc2..fdd5080d 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -6,6 +6,7 @@ import logging +import os from unittest.mock import patch import pikepdf @@ -237,6 +238,13 @@ def test_version_comparison(): need_version='4.0.0', version_parser=TesseractVersion, ) + vd.check_external_program( + program="tesseract", + package="tesseract", + version_checker=lambda: 'v4.0.0.20181030', # Some Windows builds use this format + need_version='4.0.0', + version_parser=TesseractVersion, + ) vd.check_external_program( program="tesseract", package="tesseract", @@ -291,3 +299,8 @@ def test_sidecar_equals_output(resources, no_outpdf): op = no_outpdf with pytest.raises(BadArgsError, match=r'--sidecar'): run_ocrmypdf_api(resources / 'trivial.pdf', op, '--sidecar', op) + + +def test_devnull_sidecar(resources): + with pytest.raises(BadArgsError, match=r'--sidecar.*NUL'): + run_ocrmypdf_api(resources / 'trivial.pdf', os.devnull, '--sidecar')