From 632dab2cc0cb6ea89b8734611a97c44a654edd4f Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 13 Dec 2018 18:13:30 -0800 Subject: [PATCH] Replace Ghostscript DOCINFO and fix 9.25 metadata date regression We no longer use Ghostscript to manage PDF metadata, instead omitting the DOCINFO segment from the pdfmark file we generate. Instead all of the relevant metadata code has been migrated to pikepdf, and we use that API. This should be more consistent and fixes the Ghostscript version-depedent quirks. Also removes our python-xmp-toolkit dependency, except for testing. --- setup.py | 1 - src/ocrmypdf/_pipeline.py | 43 ++++++++++--------------- src/ocrmypdf/pdfa.py | 67 ++++----------------------------------- tests/test_metadata.py | 9 +----- 4 files changed, 24 insertions(+), 96 deletions(-) diff --git a/setup.py b/setup.py index 611d4c16..fb96d1e0 100644 --- a/setup.py +++ b/setup.py @@ -255,7 +255,6 @@ setup( 'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"', # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3 # block 5.1.0, broken wheels - 'python-xmp-toolkit >= 2, < 3', 'reportlab >= 3.3.0', # oldest released version with sane image handling 'ruffus >= 2.7.0', ], diff --git a/src/ocrmypdf/_pipeline.py b/src/ocrmypdf/_pipeline.py index 25c0af01..efb72492 100644 --- a/src/ocrmypdf/_pipeline.py +++ b/src/ocrmypdf/_pipeline.py @@ -795,7 +795,7 @@ def ocr_tesseract_textonly_pdf( log=log) -def get_pdfmark(base_pdf, options): +def get_docinfo(base_pdf, options): def from_document_info(key): try: s = base_pdf.metadata[key] @@ -840,21 +840,7 @@ def generate_postscript_stub( context): options = context.get_options() pdf = pikepdf.open(input_file) - pdfmark = get_pdfmark(pdf, options) - - ascii_docinfo = False - if ghostscript.version() >= '9.24': - ascii_docinfo = True - try: - for v in pdfmark.values(): - v.encode('ascii', errors='strict') - except UnicodeEncodeError: - log.warning( - "Ghostscript 9.24+ does not support Unicode strings in " - " metadata. These will be converted to ASCII if possible." - ) - - generate_pdfa_ps(output_file, pdfmark, ascii_docinfo=ascii_docinfo) + generate_pdfa_ps(output_file) def metadata_fixup( @@ -874,6 +860,8 @@ def metadata_fixup( ps = next( (ii for ii in input_files if ii.endswith('.ps')), None ) + metadata = pikepdf.open(metadata_file) + docinfo = get_docinfo(metadata, options) if options.output_type.startswith('pdfa'): input_pdfinfo = context.get_pdfinfo() @@ -886,18 +874,21 @@ def metadata_fixup( threads=options.jobs or 1, pdfa_part=options.output_type[-1] # is pdfa-1, pdfa-2, or pdfa-3 ) + pdf = pikepdf.open(output_file) + with pdf.open_metadata() as meta: + # Note Ghostscript will populate xmp:CreateDate or /CreationDate + meta.load_from_docinfo(docinfo, delete_missing=False) + pdf.save(output_file) else: - metadata = pikepdf.open(metadata_file) - pdfmark = get_pdfmark(metadata, options) pdf = pikepdf.open(layers_file) - pdf.metadata = pdf.make_indirect(pikepdf.Dictionary(pdfmark)) - try: - pdf.save(output_file, compress_streams=True, - object_stream_mode=pikepdf.ObjectStreamMode.generate) - except AttributeError: - # pikepdf <= 0.3.4 - pdf.save(output_file, - stream_data_mode=pikepdf.StreamDataMode.compress) + with pdf.open_metadata() as meta: + meta.load_from_docinfo(docinfo, delete_missing=False) + # If xmp:CreateDate is missing, set it to the modify date to + # match Ghostscript, for consistency + if 'xmp:CreateDate' not in meta: + meta['xmp:CreateDate'] = meta.get('xmp:ModifyDate', '') + pdf.save(output_file, compress_streams=True, + object_stream_mode=pikepdf.ObjectStreamMode.generate) def optimize_pdf( diff --git a/src/ocrmypdf/pdfa.py b/src/ocrmypdf/pdfa.py index 539bcb41..413003a7 100644 --- a/src/ocrmypdf/pdfa.py +++ b/src/ocrmypdf/pdfa.py @@ -61,9 +61,6 @@ pdfa_def_template = u"""%! /ICCProfile $icc_profile def -[$docinfo - /DOCINFO pdfmark - % Define an ICC profile : [/_objdef {icc_PDFA} /type /stream /OBJ pdfmark @@ -94,6 +91,7 @@ def """ +@deprecated def encode_text_string(s: str) -> str: """ Encode text string to hex string for use in a PDF @@ -149,35 +147,7 @@ def decode_pdf_date(*args, **kwargs): return _decode_date(*args, **kwargs) -def _get_pdfmark_dates(pdfmark): - """ - Encode dates in the expected format for pdfmark Postscript - - The best way to deal with amissing date entry is set it to null, because if - the key is omitted Ghostscript will set it to now - we do not want to erase - the fact that the value was unknown. Setting to an empty string breaks - Ghostscript 9.22 as reported here: - https://bugs.ghostscript.com/show_bug.cgi?id=699182 - """ - - for key in ('/CreationDate', '/ModDate'): - if key not in pdfmark: - continue - if pdfmark[key].strip() == '': - yield ' {} null'.format(key) - continue - date_str = pdfmark[key] - if date_str.startswith('D:'): - date_str = date_str[2:] - try: - yield ' {} (D:{})'.format( - key, - _encode_date(_decode_date(date_str))) - except ValueError: - yield ' {} null'.format(key) - - -def _get_pdfa_def(icc_profile, icc_identifier, pdfmark, ascii_docinfo=False): +def _get_pdfa_def(icc_profile, icc_identifier, pdfmark=None, ascii_docinfo=None): """ Create a Postscript pdfmark file for Ghostscript. @@ -187,43 +157,18 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark, ascii_docinfo=False): :param icc_profile: filename of the ICC profile to include in pdfmark :param icc_identifier: ICC identifier such as 'sRGB' :param pdfmark: a dictionary containing keys to include the pdfmark - :param ascii_docinfo: if True, the docinfo block must be encoded in pure - ASCII and may not contain UTF-16BE-BOM-hex encoded strings, as - required for Ghostscript 9.24+ + :param ascii_docinfo: parameter is no longer meaningful :returns: a string containing the entire pdfmark - """ - # Ghostscript <= 9.21 has a bug where null entries in DOCINFO might produce - # ERROR: VMerror (-25) on closing pdfwrite device. - # https://bugs.ghostscript.com/show_bug.cgi?id=697684 - # Work around this by only adding keys that have a nontrivial value - docinfo_keys = ('/Title', '/Author', '/Subject', '/Creator', '/Keywords') - - def docinfo_gen(): - if not ascii_docinfo: - docinfo_line_template = ' {key} <{value}>' - encode = encode_text_string - else: - docinfo_line_template = ' {key} ({value})' - encode = _encode_ascii - yield from _get_pdfmark_dates(pdfmark) - for key in docinfo_keys: - if key in pdfmark and pdfmark[key].strip() != '': - line = docinfo_line_template.format( - key=key, value=encode(pdfmark[key])) - yield line - docinfo = '\n'.join(docinfo_gen()) - t = Template(pdfa_def_template) result = t.substitute(icc_profile=icc_profile, - icc_identifier=icc_identifier, - docinfo=docinfo) + icc_identifier=icc_identifier) return result -def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB', ascii_docinfo=False): +def generate_pdfa_ps(target_filename, pdfmark=None, icc='sRGB', ascii_docinfo=None): if icc == 'sRGB': icc_profile = SRGB_ICC_PROFILE else: @@ -240,7 +185,7 @@ def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB', ascii_docinfo=False): hex_icc_profile = hexlify(bytes_icc_profile) icc_profile = '<' + hex_icc_profile.decode('ascii') + '>' - ps = _get_pdfa_def(icc_profile, icc, pdfmark, ascii_docinfo=ascii_docinfo) + ps = _get_pdfa_def(icc_profile, icc, pdfmark) # We should have encoded everything to pure ASCII by this point, and # to be safe, only allow ASCII in PostScript diff --git a/tests/test_metadata.py b/tests/test_metadata.py index a7e4b8fb..dbfed2bc 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -94,9 +94,6 @@ def test_override_metadata(spoof_tesseract_noop, output_type, resources, before = pikepdf.open(input_file) after = pikepdf.open(outpdf) - if ghostscript.version() >= '9.24': - pytest.xfail('Ghostscript 9.24+ does not support Unicode DOCINFO') - assert after.metadata.Title == german, after.metadata assert after.metadata.Author == chinese, after.metadata assert after.metadata.get('/Keywords', '') == '' @@ -166,11 +163,7 @@ def test_creation_date_preserved(spoof_tesseract_noop, output_type, resources, after = pdf_after.trailer.get('/Info', {}) if not before: - # If there was input creation date, none should be output - # because of Ghostscript quirks we set it to null - # This test would be better if we had a test file with /DocumentInfo but - # no /CreationDate, which we don't - assert after.get('/CreationDate', '') == '' + assert after.get('/CreationDate', '') != '' else: # We expect that the creation date stayed the same date_before = decode_pdf_date(str(before['/CreationDate']))