Refactor test suite to use fixtures to manage paths

This commit is contained in:
James R. Barlow
2017-01-26 16:38:59 -08:00
parent fb9e7c82f6
commit 02fba02d31
6 changed files with 325 additions and 362 deletions
+4 -1
View File
@@ -31,13 +31,16 @@ def version():
return qpdf_version
def check(input_file, log):
def check(input_file, log=None):
args_qpdf = [
get_program('qpdf'),
'--check',
input_file
]
if log is None:
import logging as log
try:
check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
except CalledProcessError as e:
+90
View File
@@ -8,6 +8,8 @@ import platform
pytest_plugins = ['helpers_namespace']
import pytest
from pathlib import Path
from subprocess import Popen, PIPE
if sys.version_info.major < 3:
@@ -24,3 +26,91 @@ def is_linux():
def running_in_docker():
# Docker creates a file named /.dockerinit
return os.path.exists('/.dockerinit')
TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
OCRMYPDF = [sys.executable, '-m', 'ocrmypdf']
@pytest.helpers.register
def spoof(**kwargs):
"""Modify environment variables to override subprocess executables
spoof(program1='replacement', ...)
Before running any executable, ocrmypdf checks the environment variable
OCRMYPDF_PROGRAMNAME to override default program name/location, e.g.
OCRMYPDF_GS redirects from the system path Ghostscript ("gs") to elsewhere.
"""
env = os.environ.copy()
for replace_program, with_spoof in kwargs.items():
spoofer = os.path.join(SPOOF_PATH, with_spoof)
if not os.access(spoofer, os.X_OK):
os.chmod(spoofer, 0o755)
env['OCRMYPDF_' + replace_program.upper()] = spoofer
return env
@pytest.fixture
def resources():
return Path(TESTS_ROOT) / 'resources'
@pytest.fixture
def ocrmypdf_exec():
return OCRMYPDF
@pytest.fixture(scope="function")
def outdir(tmpdir):
return Path(tmpdir)
@pytest.fixture(scope="function")
def outpdf(tmpdir):
return str(Path(tmpdir) / 'out.pdf')
@pytest.fixture(scope="function")
def no_outpdf(tmpdir):
"""This just documents the fact that a test is not expected to produce
output. Unfortunately an assertion failure inside a test fixture produces
an error rather than a test failure, so no testing is done. It's up to
the test to confirm that no output file was created."""
return str(Path(tmpdir) / 'no_output.pdf')
@pytest.helpers.register
def check_ocrmypdf(input_file, output_file, *args, env=None):
"Run ocrmypdf and confirmed that a valid file was created"
p, out, err = run_ocrmypdf(input_file, output_file, *args, env=env)
#print(err) # ensure py.test collects the output, use -s to view
assert p.returncode == 0
assert os.path.exists(str(output_file)), "Output file not created"
assert os.stat(str(output_file)).st_size > 100, "PDF too small or empty"
assert out == "", \
"The following was written to stdout and should not have been: \n" + \
"<stdout>\n" + out + "\n</stdout>"
return output_file
@pytest.helpers.register
def run_ocrmypdf(input_file, output_file, *args, env=None):
"Run ocrmypdf and let caller deal with results"
if env is None:
env = os.environ
p_args = OCRMYPDF + list(args) + [str(input_file), str(output_file)]
p = Popen(
p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
universal_newlines=True, env=env)
out, err = p.communicate()
#print(err)
return p, out, err
+13 -29
View File
@@ -3,10 +3,12 @@
from ocrmypdf import hocrtransform
from ocrmypdf.exec.tesseract import HOCR_TEMPLATE
from ocrmypdf.exec import qpdf
from reportlab.pdfgen.canvas import Canvas
from PIL import Image
from tempfile import NamedTemporaryFile
from contextlib import suppress
from pathlib import Path
import os
import shutil
import pytest
@@ -15,42 +17,24 @@ import pytest
import sys
TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
TEST_OUTPUT = os.environ.get(
'OCRMYPDF_TEST_OUTPUT',
default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'hocrtransform'))
def setup_module():
with suppress(FileNotFoundError):
shutil.rmtree(TEST_OUTPUT)
with suppress(FileExistsError):
os.makedirs(TEST_OUTPUT)
with open(_make_output('blank.hocr'), 'w') as f:
@pytest.fixture
def blank_hocr(tmpdir):
filename = Path(tmpdir) / "blank.hocr"
with open(filename, 'w') as f:
f.write(HOCR_TEMPLATE)
return filename
def _make_input(input_basename):
return os.path.join(TEST_RESOURCES, input_basename)
def _make_output(output_basename):
return os.path.join(TEST_OUTPUT, output_basename)
def test_mono_image():
def test_mono_image(blank_hocr, outdir):
im = Image.new('1', (8, 8), 0)
for n in range(8):
im.putpixel((n, n), 1)
im.save(_make_output('mono.tif'), format='TIFF')
hocr = hocrtransform.HocrTransform(_make_output('blank.hocr'), 300)
hocr.to_pdf(_make_output('mono.pdf'), imageFileName=_make_output('mono.tif'))
im.save(outdir / 'mono.tif', format='TIFF')
hocr = hocrtransform.HocrTransform(blank_hocr, 300)
hocr.to_pdf(str(outdir / 'mono.pdf'), imageFileName=outdir / 'mono.tif')
qpdf.check(str(outdir / 'mono.pdf'))
+187 -238
View File
@@ -14,84 +14,9 @@ from ocrmypdf import leptonica
from ocrmypdf.pdfa import file_claims_pdfa
TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
TEST_OUTPUT = os.environ.get(
'OCRMYPDF_TEST_OUTPUT',
default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'main'))
OCRMYPDF = [sys.executable, '-m', 'ocrmypdf']
def setup_module():
with suppress(FileNotFoundError):
shutil.rmtree(TEST_OUTPUT)
with suppress(FileExistsError):
os.makedirs(TEST_OUTPUT)
def _infile(input_basename):
return os.path.join(TEST_RESOURCES, input_basename)
def _outfile(output_basename):
return os.path.join(TEST_OUTPUT, os.path.basename(output_basename))
def check_ocrmypdf(input_basename, output_basename, *args, env=None):
"Run ocrmypdf and confirmed that a valid file was created"
input_file = _infile(input_basename)
output_file = _outfile(output_basename)
p, out, err = run_ocrmypdf(input_basename, output_basename, *args, env=env)
print(err) # ensure py.test collects the output, use -s to view
assert p.returncode == 0
assert os.path.exists(output_file), "Output file not created"
assert os.stat(output_file).st_size > 100, "PDF too small or empty"
assert out == "", \
"The following was written to stdout and should not have been: \n" + \
"<stdout>\n" + out + "\n</stdout>"
return output_file
def run_ocrmypdf(input_basename, output_basename, *args, env=None):
"Run ocrmypdf and let caller deal with results"
input_file = _infile(input_basename)
output_file = _outfile(output_basename)
if env is None:
env = os.environ
p_args = OCRMYPDF + list(args) + [input_file, output_file]
p = Popen(
p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
universal_newlines=True, env=env)
out, err = p.communicate()
print(err)
return p, out, err
def spoof(**kwargs):
"""Modify environment variables to override subprocess executables
spoof(program1='replacement', ...)
Before running any executable, ocrmypdf checks the environment variable
OCRMYPDF_PROGRAMNAME to override default program name/location, e.g.
OCRMYPDF_GS redirects from the system path Ghostscript ("gs") to elsewhere.
"""
env = os.environ.copy()
for replace_program, with_spoof in kwargs.items():
spoofer = os.path.join(SPOOF_PATH, with_spoof)
if not os.access(spoofer, os.X_OK):
os.chmod(spoofer, 0o755)
env['OCRMYPDF_' + replace_program.upper()] = spoofer
return env
check_ocrmypdf = pytest.helpers.check_ocrmypdf
run_ocrmypdf = pytest.helpers.run_ocrmypdf
spoof = pytest.helpers.spoof
@pytest.fixture
def spoof_tesseract_noop():
@@ -125,14 +50,20 @@ def spoof_no_tess_pdfa_warning():
return spoof(tesseract='tesseract_noop.py', gs='gs_feature_elision.py')
def test_quick(spoof_tesseract_cache):
check_ocrmypdf('ccitt.pdf', 'test_quick.pdf', env=spoof_tesseract_cache)
@pytest.fixture
def spoof_qpdf_always_error():
return spoof(qpdf='qpdf_dummy_return2.py')
def test_deskew(spoof_tesseract_noop):
def test_quick(spoof_tesseract_cache, resources, outpdf):
check_ocrmypdf(resources / 'ccitt.pdf', outpdf, env=spoof_tesseract_cache)
def test_deskew(spoof_tesseract_noop, resources, outdir):
# Run with deskew
deskewed_pdf = check_ocrmypdf(
'skew.pdf', 'test_deskew.pdf', '-d', '-v', '1', env=spoof_tesseract_noop)
resources / 'skew.pdf', outdir / 'skew.pdf', '-d', '-v', '1',
env=spoof_tesseract_noop)
# Now render as an image again and use Leptonica to find the skew angle
# to confirm that it was deskewed
@@ -140,38 +71,40 @@ def test_deskew(spoof_tesseract_noop):
import logging
log = logging.getLogger()
deskewed_png = _outfile('deskewed.png')
deskewed_png = outdir / 'deskewed.png'
rasterize_pdf(
deskewed_pdf,
deskewed_png,
str(deskewed_pdf),
str(deskewed_png),
xres=150,
yres=150,
raster_device='pngmono',
log=log)
from ocrmypdf.leptonica import Pix
pix = Pix.read(deskewed_png)
pix = Pix.read(str(deskewed_png))
skew_angle, skew_confidence = pix.find_skew()
print(skew_angle)
assert -0.5 < skew_angle < 0.5, "Deskewing failed"
def test_clean(spoof_tesseract_noop):
check_ocrmypdf('skew.pdf', 'test_clean.pdf', '-c',
def test_clean(spoof_tesseract_noop, resources, outpdf):
check_ocrmypdf(resources / 'skew.pdf', outpdf, '-c',
env=spoof_tesseract_noop)
def test_remove_background(spoof_tesseract_noop):
def test_remove_background(spoof_tesseract_noop, resources, outdir):
from PIL import Image
# Ensure the input image does not contain pure white/black
im = Image.open(_infile('congress.jpg'))
im = Image.open(resources / 'congress.jpg')
assert im.getextrema() != ((0, 255), (0, 255), (0, 255))
output_pdf = check_ocrmypdf(
'congress.jpg', 'test_remove_bg.pdf', '--remove-background',
resources / 'congress.jpg',
outdir / 'test_remove_bg.pdf',
'--remove-background',
'--image-dpi', '150',
env=spoof_tesseract_noop)
@@ -179,11 +112,11 @@ def test_remove_background(spoof_tesseract_noop):
import logging
log = logging.getLogger()
output_png = _outfile('remove_bg.png')
output_png = outdir / 'remove_bg.png'
rasterize_pdf(
output_pdf,
output_png,
str(output_pdf),
str(output_png),
xres=100,
yres=100,
raster_device='png16m',
@@ -201,10 +134,11 @@ def test_remove_background(spoof_tesseract_noop):
['palette.pdf', 'cmyk.pdf', 'ccitt.pdf', 'jbig2.pdf', 'lichtenstein.pdf'])
@pytest.mark.parametrize("renderer", ['hocr', 'tesseract'])
@pytest.mark.parametrize("output_type", ['pdf', 'pdfa'])
def test_exotic_image(spoof_tesseract_cache, pdf, renderer, output_type):
def test_exotic_image(spoof_tesseract_cache, pdf, renderer, output_type,
resources, outdir):
check_ocrmypdf(
pdf,
'test_{0}_{1}.pdf'.format(pdf, renderer),
resources / pdf,
outdir / 'test_{0}_{1}.pdf'.format(pdf, renderer),
'-dc',
'-v', '1',
'--output-type', output_type,
@@ -214,19 +148,21 @@ def test_exotic_image(spoof_tesseract_cache, pdf, renderer, output_type):
@pytest.mark.parametrize("output_type", [
'pdfa', 'pdf'
])
def test_preserve_metadata(spoof_tesseract_noop, output_type):
pdf_before = pypdf.PdfFileReader(_infile('graph.pdf'))
def test_preserve_metadata(spoof_tesseract_noop, output_type,
resources, outpdf):
pdf_before = pypdf.PdfFileReader(str(resources / 'graph.pdf'))
output = check_ocrmypdf('graph.pdf', 'test_metadata_preserve.pdf',
'--output-type', output_type,
env=spoof_tesseract_noop)
output = check_ocrmypdf(
resources / 'graph.pdf', outpdf,
'--output-type', output_type,
env=spoof_tesseract_noop)
pdf_after = pypdf.PdfFileReader(output)
pdf_after = pypdf.PdfFileReader(str(output))
for key in ('/Title', '/Author'):
assert pdf_before.documentInfo[key] == pdf_after.documentInfo[key]
pdfa_info = file_claims_pdfa(output)
pdfa_info = file_claims_pdfa(str(output))
assert pdfa_info['output'] == output_type
@@ -236,16 +172,16 @@ def test_preserve_metadata(spoof_tesseract_noop, output_type):
@pytest.mark.parametrize("output_type", [
'pdfa', 'pdf'
])
def test_override_metadata(spoof_tesseract_noop, output_type):
input_file = _infile('c02-22.pdf')
output_file = _outfile('test_override_metadata.pdf')
def test_override_metadata(spoof_tesseract_noop, output_type, resources,
outpdf):
input_file = resources / 'c02-22.pdf'
german = 'Du siehst den Wald vor lauter Bäumen nicht.'
chinese = '孔子'
high_unicode = 'U+1030C is: 𐌌'
p, out, err = run_ocrmypdf(
input_file, output_file,
input_file, outpdf,
'--title', german,
'--author', chinese,
'--subject', high_unicode,
@@ -254,7 +190,7 @@ def test_override_metadata(spoof_tesseract_noop, output_type):
assert p.returncode == ExitCode.ok
pdf = output_file
pdf = str(outpdf)
out_pdfinfo = check_output(['pdfinfo', pdf], universal_newlines=True)
lines_pdfinfo = out_pdfinfo.splitlines()
@@ -268,7 +204,7 @@ def test_override_metadata(spoof_tesseract_noop, output_type):
assert pdfinfo['Subject'] == high_unicode
assert pdfinfo.get('Keywords', '') == ''
pdfa_info = file_claims_pdfa(output_file)
pdfa_info = file_claims_pdfa(pdf)
assert pdfa_info['output'] == output_type
@@ -276,45 +212,46 @@ def test_override_metadata(spoof_tesseract_noop, output_type):
'hocr',
'tesseract',
])
def test_oversample(spoof_tesseract_cache, renderer):
def test_oversample(spoof_tesseract_cache, renderer, resources, outpdf):
oversampled_pdf = check_ocrmypdf(
'skew.pdf', 'test_oversample_%s.pdf' % renderer, '--oversample', '350',
resources / 'skew.pdf', outpdf, '--oversample', '350',
'-f',
'--pdf-renderer', renderer, env=spoof_tesseract_cache)
pdfinfo = pdf_get_all_pageinfo(oversampled_pdf)
pdfinfo = pdf_get_all_pageinfo(str(oversampled_pdf))
print(pdfinfo[0]['xres'])
assert abs(pdfinfo[0]['xres'] - 350) < 1
def test_repeat_ocr():
p, _, _ = run_ocrmypdf('graph_ocred.pdf', 'wontwork.pdf')
def test_repeat_ocr(resources, no_outpdf):
p, _, _ = run_ocrmypdf(resources / 'graph_ocred.pdf', no_outpdf)
assert p.returncode != 0
def test_force_ocr(spoof_tesseract_cache):
out = check_ocrmypdf('graph_ocred.pdf', 'test_force.pdf', '-f',
def test_force_ocr(spoof_tesseract_cache, resources, outpdf):
out = check_ocrmypdf(resources / 'graph_ocred.pdf', outpdf, '-f',
env=spoof_tesseract_cache)
pdfinfo = pdf_get_all_pageinfo(out)
assert pdfinfo[0]['has_text']
def test_skip_ocr(spoof_tesseract_cache):
check_ocrmypdf('graph_ocred.pdf', 'test_skip.pdf', '-s',
def test_skip_ocr(spoof_tesseract_cache, resources, outpdf):
check_ocrmypdf(resources / 'graph_ocred.pdf', outpdf, '-s',
env=spoof_tesseract_cache)
def test_argsfile(spoof_tesseract_noop):
with open(_outfile('test_argsfile.txt'), 'w') as argsfile:
def test_argsfile(spoof_tesseract_noop, resources, outdir):
with open(outdir / 'test_argsfile.txt', 'w') as argsfile:
print('--title', 'ArgsFile Test', '--author', 'Test Cases',
sep='\n', end='\n', file=argsfile)
check_ocrmypdf('graph.pdf', 'test_argsfile.pdf',
'@' + _outfile('test_argsfile.txt'),
check_ocrmypdf(resources / 'graph.pdf', outdir / 'args.pdf',
'@' + str(outdir / 'test_argsfile.txt'),
env=spoof_tesseract_noop)
def check_monochrome_correlation(
outdir,
reference_pdf, reference_pageno, test_pdf, test_pageno):
import ocrmypdf.exec.ghostscript as ghostscript
@@ -322,44 +259,46 @@ def check_monochrome_correlation(
gslog = logging.getLogger()
reference_png = _outfile('{}.ref{:04d}.png'.format(
reference_pdf, reference_pageno))
test_png = _outfile('{}.test{:04d}.png'.format(
test_pdf, test_pageno))
reference_png = outdir / '{}.ref{:04d}.png'.format(
reference_pdf, reference_pageno)
test_png = outdir / '{}.test{:04d}.png'.format(
test_pdf, test_pageno)
def rasterize(pdf, pageno, png):
if os.path.exists(png):
if png.exists():
print(png)
return
ghostscript.rasterize_pdf(
pdf,
png,
str(pdf),
str(png),
xres=100, yres=100,
raster_device='pngmono', log=gslog, pageno=pageno)
rasterize(reference_pdf, reference_pageno, reference_png)
rasterize(test_pdf, test_pageno, test_png)
pix_ref = leptonica.Pix.read(reference_png)
pix_test = leptonica.Pix.read(test_png)
pix_ref = leptonica.Pix.read(str(reference_png))
pix_test = leptonica.Pix.read(str(test_png))
return leptonica.Pix.correlation_binary(pix_ref, pix_test)
def test_monochrome_correlation():
def test_monochrome_correlation(resources, outdir):
# Verify leptonica: check that an incorrect rotated image has poor
# correlation with reference
corr = check_monochrome_correlation(
reference_pdf=_infile('cardinal.pdf'),
outdir,
reference_pdf=resources / 'cardinal.pdf',
reference_pageno=1, # north facing page
test_pdf=_infile('cardinal.pdf'),
test_pdf=resources / 'cardinal.pdf',
test_pageno=3, # south facing page
)
assert corr < 0.10
corr = check_monochrome_correlation(
reference_pdf=_infile('cardinal.pdf'),
outdir,
reference_pdf=resources / 'cardinal.pdf',
reference_pageno=2,
test_pdf=_infile('cardinal.pdf'),
test_pdf=resources / 'cardinal.pdf',
test_pageno=2,
)
assert corr > 0.90
@@ -369,44 +308,47 @@ def test_monochrome_correlation():
'hocr',
'tesseract',
])
def test_autorotate(spoof_tesseract_cache, renderer):
def test_autorotate(spoof_tesseract_cache, renderer, resources, outdir):
# cardinal.pdf contains four copies of an image rotated in each cardinal
# direction - these ones are "burned in" not tagged with /Rotate
out = check_ocrmypdf('cardinal.pdf', 'test_autorotate_%s.pdf' % renderer,
out = check_ocrmypdf(resources / 'cardinal.pdf', outdir / 'out.pdf',
'-r', '-v', '1', env=spoof_tesseract_cache)
for n in range(1, 4+1):
correlation = check_monochrome_correlation(
reference_pdf=_infile('cardinal.pdf'),
outdir,
reference_pdf=resources / 'cardinal.pdf',
reference_pageno=1,
test_pdf=out,
test_pdf=outdir / 'out.pdf',
test_pageno=n)
assert correlation > 0.80
def test_autorotate_threshold_low(spoof_tesseract_cache):
out = check_ocrmypdf('cardinal.pdf', 'test_autorotate_threshold_low.pdf',
def test_autorotate_threshold_low(spoof_tesseract_cache, resources, outdir):
out = check_ocrmypdf(resources / 'cardinal.pdf', outdir / 'out.pdf',
'--rotate-pages-threshold', '1',
'-r', '-v', '1', env=spoof_tesseract_cache)
# Low threshold -> always rotate -> expect high correlation between
# reference page and test page
correlation = check_monochrome_correlation(
reference_pdf=_infile('cardinal.pdf'),
outdir,
reference_pdf=resources / 'cardinal.pdf',
reference_pageno=1,
test_pdf=out,
test_pageno=3)
assert correlation > 0.80
def test_autorotate_threshold_high(spoof_tesseract_cache):
out = check_ocrmypdf('cardinal.pdf', 'test_autorotate_threshold_high.pdf',
def test_autorotate_threshold_high(spoof_tesseract_cache, resources, outdir):
out = check_ocrmypdf(resources / 'cardinal.pdf', outdir / 'out.pdf',
'--rotate-pages-threshold', '99',
'-r', '-v', '1', env=spoof_tesseract_cache)
# High threshold -> never rotate -> expect low correlation since
# test page will not be rotated
correlation = check_monochrome_correlation(
reference_pdf=_infile('cardinal.pdf'),
outdir,
reference_pdf=resources / 'cardinal.pdf',
reference_pageno=1,
test_pdf=out,
test_pageno=3)
@@ -417,25 +359,26 @@ def test_autorotate_threshold_high(spoof_tesseract_cache):
'hocr',
'tesseract',
])
def test_ocr_timeout(renderer):
out = check_ocrmypdf('skew.pdf', 'test_timeout_%s.pdf' % renderer,
def test_ocr_timeout(renderer, resources, outpdf):
out = check_ocrmypdf(resources / 'skew.pdf', outpdf,
'--tesseract-timeout', '1.0')
pdfinfo = pdf_get_all_pageinfo(out)
pdfinfo = pdf_get_all_pageinfo(str(out))
assert not pdfinfo[0]['has_text']
def test_skip_big(spoof_tesseract_cache):
out = check_ocrmypdf('enormous.pdf', 'test_enormous.pdf',
def test_skip_big(spoof_tesseract_cache, resources, outpdf):
out = check_ocrmypdf(resources / 'enormous.pdf', outpdf,
'--skip-big', '10', env=spoof_tesseract_cache)
pdfinfo = pdf_get_all_pageinfo(out)
pdfinfo = pdf_get_all_pageinfo(str(out))
assert not pdfinfo[0]['has_text']
@pytest.mark.parametrize('renderer', ['hocr', 'tesseract'])
@pytest.mark.parametrize('output_type', ['pdf', 'pdfa'])
def test_maximum_options(spoof_tesseract_cache, renderer, output_type):
def test_maximum_options(spoof_tesseract_cache, renderer, output_type,
resources, outpdf):
check_ocrmypdf(
'multipage.pdf', 'test_multipage%s.pdf' % renderer,
resources / 'multipage.pdf', outpdf,
'-d', '-c', '-i', '-g', '-f', '-k', '--oversample', '300',
'--remove-background',
'--skip-big', '10', '--title', 'Too Many Weird Files',
@@ -444,101 +387,102 @@ def test_maximum_options(spoof_tesseract_cache, renderer, output_type):
env=spoof_tesseract_cache)
def test_tesseract_missing_tessdata():
def test_tesseract_missing_tessdata(resources, no_outpdf):
env = os.environ.copy()
env['TESSDATA_PREFIX'] = '/tmp'
p, _, err = run_ocrmypdf(
'graph_ocred.pdf', 'not_a_pdfa.pdf', '-v', '1', '--skip-text', env=env)
resources / 'graph_ocred.pdf', no_outpdf,
'-v', '1', '--skip-text', env=env)
assert p.returncode == ExitCode.missing_dependency, err
def test_invalid_input_pdf():
def test_invalid_input_pdf(resources, no_outpdf):
p, out, err = run_ocrmypdf(
'invalid.pdf', 'wont_be_created.pdf')
resources / 'invalid.pdf', no_outpdf)
assert p.returncode == ExitCode.input_file, err
def test_blank_input_pdf():
def test_blank_input_pdf(resources, outpdf):
p, out, err = run_ocrmypdf(
'blank.pdf', 'still_blank.pdf')
resources / 'blank.pdf', outpdf)
assert p.returncode == ExitCode.ok
def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash):
def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources,
no_outpdf):
# As a correctness test, make sure that --force-ocr on a PDF with no
# content still triggers tesseract. If tesseract crashes, then it was
# called.
p, _, err = run_ocrmypdf(
'blank.pdf', 'wont_be_created.pdf', '--force-ocr',
resources / 'blank.pdf', no_outpdf, '--force-ocr',
env=spoof_tesseract_crash)
assert p.returncode == ExitCode.child_process_error, err
assert not os.path.exists(_outfile('wontwork.pdf'))
assert not os.path.exists(no_outpdf)
def test_french(spoof_tesseract_cache):
def test_french(spoof_tesseract_cache, resources, outpdf):
p, out, err = run_ocrmypdf(
'francais.pdf', 'francais.pdf', '-l', 'fra', env=spoof_tesseract_cache)
resources / 'francais.pdf', outpdf, '-l', 'fra',
env=spoof_tesseract_cache)
assert p.returncode == ExitCode.ok, \
"This test may fail if Tesseract language packs are missing"
def test_klingon():
def test_klingon(resources, outpdf):
p, out, err = run_ocrmypdf(
'francais.pdf', 'francais.pdf', '-l', 'klz')
resources / 'francais.pdf', outpdf, '-l', 'klz')
assert p.returncode == ExitCode.bad_args
def test_missing_docinfo(spoof_tesseract_noop):
def test_missing_docinfo(spoof_tesseract_noop, resources, outpdf):
p, out, err = run_ocrmypdf(
'missing_docinfo.pdf', 'missing_docinfo.pdf', '-l', 'eng', '-c',
resources / 'missing_docinfo.pdf', outpdf, '-l', 'eng', '-c',
env=spoof_tesseract_noop)
assert p.returncode == ExitCode.ok, err
@pytest.mark.skipif(pytest.helpers.running_in_docker(),
reason="writes to tests/resources")
def test_uppercase_extension(spoof_tesseract_noop):
shutil.copy(_infile("skew.pdf"), _infile("UPPERCASE.PDF"))
try:
check_ocrmypdf("UPPERCASE.PDF", "UPPERCASE_OUT.PDF",
env=spoof_tesseract_noop)
finally:
os.unlink(_infile("UPPERCASE.PDF"))
reason="<no longer true> writes to tests/resources")
def test_uppercase_extension(spoof_tesseract_noop, resources, outdir):
shutil.copy(
str(resources / "skew.pdf"),
str(outdir / "UPPERCASE.PDF"))
check_ocrmypdf(outdir / "UPPERCASE.PDF", outdir / "UPPERCASE_OUT.PDF",
env=spoof_tesseract_noop)
def test_input_file_not_found():
def test_input_file_not_found(no_outpdf):
input_file = "does not exist.pdf"
p, out, err = run_ocrmypdf(
_infile(input_file),
_outfile("will not happen.pdf"))
input_file,
no_outpdf)
assert p.returncode == ExitCode.input_file
assert (input_file in out or input_file in err)
def test_input_file_not_a_pdf():
def test_input_file_not_a_pdf(no_outpdf):
input_file = __file__ # Try to OCR this file
p, out, err = run_ocrmypdf(
_infile(input_file),
_outfile("will not happen.pdf"))
input_file,
no_outpdf)
assert p.returncode == ExitCode.input_file
assert (input_file in out or input_file in err)
def test_qpdf_repair_fails():
env = os.environ.copy()
env['OCRMYPDF_QPDF'] = os.path.abspath('./spoof/qpdf_dummy_return2.py')
def test_qpdf_repair_fails(spoof_qpdf_always_error, resources, no_outpdf):
p, out, err = run_ocrmypdf(
'-v', '1',
'c02-22.pdf', 'wont_be_created_repair_fail.pdf', env=env)
resources / 'c02-22.pdf', no_outpdf, env=spoof_qpdf_always_error)
print(out)
print(err)
assert p.returncode == ExitCode.input_file
def test_encrypted():
def test_encrypted(resources, no_outpdf):
p, out, err = run_ocrmypdf(
'skew-encrypted.pdf', 'wont_be_created_test_enc.pdf')
resources / 'skew-encrypted.pdf', no_outpdf)
assert p.returncode == ExitCode.encrypted_pdf
assert out.find('password')
@@ -547,9 +491,9 @@ def test_encrypted():
'hocr',
'tesseract',
])
def test_pagesegmode(renderer, spoof_tesseract_cache):
def test_pagesegmode(renderer, spoof_tesseract_cache, resources, outpdf):
check_ocrmypdf(
'skew.pdf', 'test_psm_%s.pdf' % renderer,
resources / 'skew.pdf', outpdf,
'--tesseract-pagesegmode', '7',
'-v', '1',
'--pdf-renderer', renderer, env=spoof_tesseract_cache)
@@ -559,21 +503,23 @@ def test_pagesegmode(renderer, spoof_tesseract_cache):
'hocr',
'tesseract',
])
def test_tesseract_crash(renderer, spoof_tesseract_crash):
def test_tesseract_crash(renderer, spoof_tesseract_crash,
resources, no_outpdf):
p, out, err = run_ocrmypdf(
'ccitt.pdf', 'wontwork.pdf', '-v', '1',
resources / 'ccitt.pdf', no_outpdf, '-v', '1',
'--pdf-renderer', renderer, env=spoof_tesseract_crash)
assert p.returncode == ExitCode.child_process_error
assert not os.path.exists(_outfile('wontwork.pdf'))
assert not os.path.exists(no_outpdf)
assert "ERROR" in err
def test_tesseract_crash_autorotate(spoof_tesseract_crash):
def test_tesseract_crash_autorotate(spoof_tesseract_crash,
resources, no_outpdf):
p, out, err = run_ocrmypdf(
'ccitt.pdf', 'wontwork.pdf',
resources / 'ccitt.pdf', no_outpdf,
'-r', env=spoof_tesseract_crash)
assert p.returncode == ExitCode.child_process_error
assert not os.path.exists(_outfile('wontwork.pdf'))
assert not os.path.exists(no_outpdf)
assert "ERROR" in err
print(out)
print(err)
@@ -583,76 +529,77 @@ def test_tesseract_crash_autorotate(spoof_tesseract_crash):
'hocr',
'tesseract',
])
def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error):
def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error,
resources, outpdf):
check_ocrmypdf(
'hugemono.pdf', 'hugemono_%s.pdf' % renderer, '-r',
resources / 'hugemono.pdf', outpdf, '-r',
'--pdf-renderer', renderer, env=spoof_tesseract_big_image_error)
def test_no_unpaper():
def test_no_unpaper(resources, no_outpdf):
env = os.environ.copy()
env['OCRMYPDF_UNPAPER'] = os.path.abspath('./spoof/no_unpaper_here.py')
p, out, err = run_ocrmypdf(
'c02-22.pdf', 'wont_be_created.pdf', '--clean', env=env)
resources / 'c02-22.pdf', no_outpdf, '--clean', env=env)
assert p.returncode == ExitCode.missing_dependency
def test_old_unpaper():
def test_old_unpaper(resources, no_outpdf):
env = os.environ.copy()
env['OCRMYPDF_UNPAPER'] = os.path.abspath('./spoof/unpaper_oldversion.py')
p, out, err = run_ocrmypdf(
'c02-22.pdf', 'wont_be_created.pdf', '--clean', env=env)
resources / 'c02-22.pdf', no_outpdf, '--clean', env=env)
assert p.returncode == ExitCode.missing_dependency
def test_algo4():
p, _, _ = run_ocrmypdf('encrypted_algo4.pdf', 'wontwork.pdf')
def test_algo4(resources, no_outpdf):
p, _, _ = run_ocrmypdf(resources / 'encrypted_algo4.pdf', no_outpdf)
assert p.returncode == ExitCode.encrypted_pdf
@pytest.mark.parametrize('renderer', [
'hocr']) # tesseract cannot pass this test - resamples to square image
def test_non_square_resolution(renderer, spoof_tesseract_cache):
def test_non_square_resolution(renderer, spoof_tesseract_cache,
resources, outpdf):
# Confirm input image is non-square resolution
in_pageinfo = pdf_get_all_pageinfo(_infile('aspect.pdf'))
in_pageinfo = pdf_get_all_pageinfo(str(resources / 'aspect.pdf'))
assert in_pageinfo[0]['xres'] != in_pageinfo[0]['yres']
out = 'aspect_%s.pdf' % renderer
check_ocrmypdf(
'aspect.pdf', out,
resources / 'aspect.pdf', outpdf,
'--pdf-renderer', renderer, env=spoof_tesseract_cache)
out_pageinfo = pdf_get_all_pageinfo(_outfile(out))
out_pageinfo = pdf_get_all_pageinfo(str(outpdf))
# Confirm resolution was kept the same
assert in_pageinfo[0]['xres'] == out_pageinfo[0]['xres']
assert in_pageinfo[0]['yres'] == out_pageinfo[0]['yres']
def test_image_to_pdf(spoof_tesseract_noop):
def test_image_to_pdf(spoof_tesseract_noop, resources, outpdf):
check_ocrmypdf(
'LinnSequencer.jpg', 'image_to_pdf.pdf', '--image-dpi', '200',
resources / 'LinnSequencer.jpg', outpdf, '--image-dpi', '200',
env=spoof_tesseract_noop)
def test_jbig2_passthrough(spoof_tesseract_cache):
def test_jbig2_passthrough(spoof_tesseract_cache, resources, outpdf):
out = check_ocrmypdf(
'jbig2.pdf', 'jbig2_out.pdf',
resources / 'jbig2.pdf', outpdf,
'--output-type', 'pdf',
'--pdf-renderer', 'hocr',
env=spoof_tesseract_cache)
out_pageinfo = pdf_get_all_pageinfo(out)
out_pageinfo = pdf_get_all_pageinfo(str(out))
assert out_pageinfo[0]['images'][0]['enc'] == 'jbig2'
def test_stdin(spoof_tesseract_noop):
input_file = _infile('francais.pdf')
output_file = _outfile('test_stdin.pdf')
def test_stdin(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
input_file = str(resources / 'francais.pdf')
output_file = str(outpdf)
# Runs: ocrmypdf - output.pdf < testfile.pdf
with open(input_file, 'rb') as input_stream:
p_args = OCRMYPDF + ['-', output_file]
p_args = ocrmypdf_exec + ['-', output_file]
p = Popen(
p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
stdin=input_stream, env=spoof_tesseract_noop)
@@ -661,13 +608,13 @@ def test_stdin(spoof_tesseract_noop):
assert p.returncode == ExitCode.ok
def test_stdout(spoof_tesseract_noop):
input_file = _infile('francais.pdf')
output_file = _outfile('test_stdout.pdf')
def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
input_file = str(resources / 'francais.pdf')
output_file = str(outpdf)
# Runs: ocrmypdf francais.pdf - > test_stdout.pdf
with open(output_file, 'wb') as output_stream:
p_args = OCRMYPDF + [input_file, '-']
p_args = ocrmypdf_exec + [input_file, '-']
p = Popen(
p_args, close_fds=True, stdout=output_stream, stderr=PIPE,
stdin=DEVNULL, env=spoof_tesseract_noop)
@@ -679,17 +626,18 @@ def test_stdout(spoof_tesseract_noop):
assert qpdf.check(output_file, log=None)
def test_masks(spoof_tesseract_noop):
check_ocrmypdf('masks.pdf', 'test_masks.pdf', env=spoof_tesseract_noop)
def test_masks(spoof_tesseract_noop, resources, outpdf):
check_ocrmypdf(resources / 'masks.pdf', outpdf, env=spoof_tesseract_noop)
def test_linearized_pdf_and_indirect_object(spoof_tesseract_noop):
def test_linearized_pdf_and_indirect_object(spoof_tesseract_noop,
resources, outpdf):
check_ocrmypdf(
'epson.pdf', 'test_epson.pdf',
resources / 'epson.pdf', outpdf,
env=spoof_tesseract_noop)
def test_rotated_skew_timeout():
def test_rotated_skew_timeout(resources, outpdf):
"""This document contains an image that is rotated 90 into place with a
/Rotate tag and intentionally skewed by altering the transformation matrix.
@@ -697,7 +645,7 @@ def test_rotated_skew_timeout():
timeout produced a page whose dimensions did not match the original's.
"""
input_file = _infile('rotated_skew.pdf')
input_file = str(resources / 'rotated_skew.pdf')
in_pageinfo = pdf_get_all_pageinfo(input_file)[0]
assert in_pageinfo['height_pixels'] < in_pageinfo['width_pixels'], \
@@ -705,11 +653,11 @@ def test_rotated_skew_timeout():
assert in_pageinfo['rotate'] == 90, "Expected a rotated page"
out = check_ocrmypdf(
'rotated_skew.pdf', 'test_rotated_skew.pdf',
input_file, outpdf,
'--pdf-renderer', 'hocr',
'--deskew', '--tesseract-timeout', '0')
out_pageinfo = pdf_get_all_pageinfo(out)[0]
out_pageinfo = pdf_get_all_pageinfo(str(out))[0]
assert out_pageinfo['height_pixels'] > out_pageinfo['width_pixels'], \
"Expected the output page to be portrait"
@@ -722,19 +670,20 @@ def test_rotated_skew_timeout():
"Expected page rotation to be baked in"
def test_ghostscript_pdfa_failure(spoof_no_tess_no_pdfa):
def test_ghostscript_pdfa_failure(spoof_no_tess_no_pdfa, resources, outpdf):
p, out, err = run_ocrmypdf(
'ccitt.pdf', 'test_pdfa_failure.pdf',
resources / 'ccitt.pdf', outpdf,
env=spoof_no_tess_no_pdfa)
assert p.returncode == 4, "Expected return code 4 when PDF/A fails"
def test_ghostscript_feature_elision(spoof_no_tess_pdfa_warning):
check_ocrmypdf('ccitt.pdf', 'test_feature_elision.pdf',
def test_ghostscript_feature_elision(spoof_no_tess_pdfa_warning,
resources, outpdf):
check_ocrmypdf(resources / 'ccitt.pdf', outpdf,
env=spoof_no_tess_pdfa_warning)
def test_very_high_dpi(spoof_tesseract_cache):
def test_very_high_dpi(spoof_tesseract_cache, resources, outpdf):
"Checks for a Decimal quantize error with high DPI, etc"
check_ocrmypdf('2400dpi.pdf', 'test_2400dpi.pdf',
check_ocrmypdf(resources / '2400dpi.pdf', outpdf,
env=spoof_tesseract_cache)
+27 -54
View File
@@ -14,34 +14,9 @@ import pytest
import sys
TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
TEST_OUTPUT = os.environ.get(
'OCRMYPDF_TEST_OUTPUT',
default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'pageinfo'))
def setup_module():
with suppress(FileNotFoundError):
shutil.rmtree(TEST_OUTPUT)
with suppress(FileExistsError):
os.makedirs(TEST_OUTPUT)
def _make_input(input_basename):
return os.path.join(TEST_RESOURCES, input_basename)
def _make_output(output_basename):
return os.path.join(TEST_OUTPUT, output_basename)
def test_single_page_text():
filename = os.path.join(TEST_OUTPUT, 'text.pdf')
pdf = Canvas(filename, pagesize=(8*72, 6*72))
def test_single_page_text(outdir):
filename = outdir / 'text.pdf'
pdf = Canvas(str(filename), pagesize=(8*72, 6*72))
text = pdf.beginText()
text.setFont('Helvetica', 12)
text.setTextOrigin(1*72, 3*72)
@@ -51,7 +26,7 @@ def test_single_page_text():
pdf.showPage()
pdf.save()
pdfinfo = pageinfo.pdf_get_all_pageinfo(filename)
pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename))
assert len(pdfinfo) == 1
page = pdfinfo[0]
@@ -60,28 +35,26 @@ def test_single_page_text():
assert len(page['images']) == 0
def test_single_page_image():
filename = os.path.join(TEST_OUTPUT, 'image-mono.pdf')
def test_single_page_image(outdir):
filename = outdir / 'image-mono.pdf'
with NamedTemporaryFile(mode='wb+', suffix='.png') as im_tmp:
im = Image.new('1', (8, 8), 0)
for n in range(8):
im.putpixel((n, n), 1)
im.save(im_tmp.name, format='PNG')
im_tmp = outdir / 'tmp.png'
im = Image.new('1', (8, 8), 0)
for n in range(8):
im.putpixel((n, n), 1)
im.save(str(im_tmp), format='PNG')
imgsize = ((img2pdf.ImgSize.dpi, 8), (img2pdf.ImgSize.dpi, 8))
layout_fun = img2pdf.get_layout_fun(None, imgsize, None, None, None)
imgsize = ((img2pdf.ImgSize.dpi, 8), (img2pdf.ImgSize.dpi, 8))
layout_fun = img2pdf.get_layout_fun(None, imgsize, None, None, None)
im_tmp.seek(0)
im_bytes = im_tmp.read()
pdf_bytes = img2pdf.convert(
im_bytes, producer="img2pdf", with_pdfrw=False,
layout_fun=layout_fun)
im_bytes = im_tmp.read_bytes()
pdf_bytes = img2pdf.convert(
im_bytes, producer="img2pdf", with_pdfrw=False,
layout_fun=layout_fun)
with open(filename, 'wb') as pdf:
pdf.write(pdf_bytes)
with open(filename, 'wb') as pdf:
pdf.write(pdf_bytes)
pdfinfo = pageinfo.pdf_get_all_pageinfo(filename)
pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename))
assert len(pdfinfo) == 1
page = pdfinfo[0]
@@ -98,9 +71,9 @@ def test_single_page_image():
assert abs(pdfimage['dpi_h'] - 8) < 1e-5
def test_single_page_inline_image():
filename = os.path.join(TEST_OUTPUT, 'image-mono-inline.pdf')
pdf = Canvas(filename, pagesize=(8*72, 6*72))
def test_single_page_inline_image(outdir):
filename = outdir / 'image-mono-inline.pdf'
pdf = Canvas(str(filename), pagesize=(8*72, 6*72))
with NamedTemporaryFile() as im_tmp:
im = Image.new('1', (8, 8), 0)
for n in range(8):
@@ -111,7 +84,7 @@ def test_single_page_inline_image():
pdf.showPage()
pdf.save()
pdfinfo = pageinfo.pdf_get_all_pageinfo(filename)
pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename))
print(pdfinfo)
pdfimage = pdfinfo[0]['images'][0]
assert (pdfimage['dpi_w'] - 8) < 1e-5
@@ -119,10 +92,10 @@ def test_single_page_inline_image():
assert pdfimage['width'] == 8
def test_jpeg():
filename = _make_input('c02-22.pdf')
def test_jpeg(resources, outdir):
filename = resources / 'c02-22.pdf'
pdfinfo = pageinfo.pdf_get_all_pageinfo(filename)
pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename))
pdfimage = pdfinfo[0]['images'][0]
assert pdfimage['enc'] == 'jpeg'
+4 -40
View File
@@ -13,7 +13,6 @@ from ocrmypdf.exceptions import ExitCode
from ocrmypdf import leptonica
from ocrmypdf.pdfa import file_claims_pdfa
from ocrmypdf.exec import tesseract
from common import is_linux, running_in_docker
TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
@@ -28,47 +27,12 @@ pytestmark = pytest.mark.skipif(not tesseract.v4(),
reason="tesseract 4.0 required")
def _infile(input_basename):
return os.path.join(TEST_RESOURCES, input_basename)
def check_ocrmypdf(input_basename, output, *args, env=None):
"Run ocrmypdf and confirmed that a valid file was created"
input_file = _infile(input_basename)
p, out, err = run_ocrmypdf(input_basename, output, *args, env=env)
print(err) # ensure py.test collects the output, use -s to view
assert p.returncode == 0
assert os.path.exists(output), "Output file not created"
assert os.stat(output).st_size > 100, "PDF too small or empty"
assert out == "", \
"The following was written to stdout and should not have been: \n" + \
"<stdout>\n" + out + "\n</stdout>"
return output
def run_ocrmypdf(input_basename, output, *args, env=None):
"Run ocrmypdf and let caller deal with results"
input_file = _infile(input_basename)
if env is None:
env = os.environ
p_args = OCRMYPDF + list(args) + [input_file, output]
p = Popen(
p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
universal_newlines=True, env=env)
out, err = p.communicate()
print(err)
return p, out, err
@pytest.mark.skipif(not tesseract.has_textonly_pdf(),
reason="requires textonly_pdf parameter")
def test_textonly_pdf(self, tmpdir):
output = str(tmpdir.join("linn_textonly.pdf"))
check_ocrmypdf('linn.pdf', output, '--pdf-renderer', 'tess4')
def test_textonly_pdf(resources, outdir):
pytest.helpers.check_ocrmypdf(
resources / 'linn.pdf',
outdir / 'linn_textonly.pdf', '--pdf-renderer', 'tess4')