From 58c29ffb5cc037aeb8f00e09237a5933ea1e31f0 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 18 Apr 2019 15:12:48 -0700 Subject: [PATCH] weave: use explicit pdf.close(), drastically reduce open file handles With the new pikepdf 1.2.0 we no longer need to hold file handles open because of the "copy to memory" functionality. We retain the behavior of closing/reopening the output PDF every 100 pages as a way to limit memory usage. --- setup.py | 2 +- src/ocrmypdf/_weave.py | 23 ++++++++++++----------- tests/test_weave.py | 2 +- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/setup.py b/setup.py index 560434f8..81003662 100644 --- a/setup.py +++ b/setup.py @@ -99,7 +99,7 @@ setup( 'cffi >= 1.9.1', # must be a setup and install requirement 'img2pdf >= 0.3.0, < 0.4', # pure Python, so track HEAD closely 'pdfminer.six == 20181108 ; sys_platform != "darwin"', - 'pikepdf >= 1.1.0, < 2', + 'pikepdf >= 1.2.0, < 2', 'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"', # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3 # block 5.1.0, broken wheels diff --git a/src/ocrmypdf/_weave.py b/src/ocrmypdf/_weave.py index 17e37144..a66244a2 100644 --- a/src/ocrmypdf/_weave.py +++ b/src/ocrmypdf/_weave.py @@ -25,7 +25,7 @@ from .exec import tesseract from .helpers import flatten_groups, page_number -MAX_OPEN_PAGE_PDFS = int(os.environ.get('_OCRMYPDF_MAX_OPEN_PAGE_PDFS', 100)) +MAX_REPLACE_PAGES = int(os.environ.get('_OCRMYPDF_MAX_REPLACE_PAGES', 100)) def _update_page_resources(*, page, font, font_key, procset): @@ -321,7 +321,6 @@ def weave_layers(infiles, output_file, log, context): base = list(basegroup)[0] path_base = Path(base).resolve() pdf_base = pikepdf.open(path_base) - keep_open = [] font, font_key, procset = None, None, None pdfinfo = context.get_pdfinfo() pagerefs = {} @@ -330,6 +329,8 @@ def weave_layers(infiles, output_file, log, context): pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]') ) + replacements = 0 + # Iterate rest for page_num, layers in groups: layers = list(layers) @@ -353,7 +354,7 @@ def weave_layers(infiles, output_file, log, context): old_objgen = pdf_base.pages[page_num - 1].objgen with pikepdf.open(image) as pdf_image: - keep_open.append(pdf_image) + replacements += 1 image_page = pdf_image.pages[0] pdf_base.pages[page_num - 1] = image_page @@ -394,20 +395,19 @@ def weave_layers(infiles, output_file, log, context): content_rotation - autorotate_correction ) % 360 - if len(keep_open) > MAX_OPEN_PAGE_PDFS: - # qpdf limitations require us to keep files open when we intend - # to copy content from them before saving. However, we want to keep - # a lid on file handles and memory usage, so for big files we're - # going to stop and save periodically. Attach the font to page 1 - # even if page 1 doesn't use it, so we have a way to get it back. + if replacements % MAX_REPLACE_PAGES == 0: + # Periodically save and reload the Pdf object. This will keep a + # lid on our memory usage for very large files. Attach the font to + # page 1 even if page 1 doesn't use it, so we have a way to get it + # back. + # TODO refactor this to outside the loop page0 = pdf_base.pages[0] _update_page_resources( page=page0, font=font, font_key=font_key, procset=procset ) interim = output_file + f'_working{page_num}.pdf' pdf_base.save(interim) - del pdf_base - keep_open = [] + pdf_base.close() pdf_base = pikepdf.open(interim) procset = pdf_base.pages[0].Resources.ProcSet @@ -415,3 +415,4 @@ def weave_layers(infiles, output_file, log, context): _fix_toc(pdf_base, pagerefs, log) pdf_base.save(output_file) + pdf_base.close() diff --git a/tests/test_weave.py b/tests/test_weave.py index 34452215..372eddca 100644 --- a/tests/test_weave.py +++ b/tests/test_weave.py @@ -53,7 +53,7 @@ def test_no_glyphless_weave(resources, outdir): pdf.save(outdir / 'test.pdf') env = os.environ.copy() - env['_OCRMYPDF_MAX_OPEN_PAGE_PDFS'] = '2' + env['_OCRMYPDF_MAX_REPLACE_PAGES'] = '2' check_ocrmypdf( outdir / 'test.pdf', outdir / 'out.pdf',