Refactor -migrate metadata repair to new module
This commit is contained in:
James R. Barlow
@@ -0,0 +1,157 @@
|
||||
# SPDX-FileCopyrightText: 2023 James R. Barlow
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
"""OCRmyPDF page processing pipeline functions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pikepdf
|
||||
from pikepdf.models.metadata import encode_pdf_date
|
||||
|
||||
from ocrmypdf._jobcontext import PdfContext
|
||||
from ocrmypdf._version import PROGRAM_NAME
|
||||
from ocrmypdf._version import __version__ as VERSION
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_docinfo(base_pdf: pikepdf.Pdf, context: PdfContext) -> dict[str, str]:
|
||||
"""Read the document info and store it in a dictionary."""
|
||||
options = context.options
|
||||
|
||||
def from_document_info(key):
|
||||
try:
|
||||
s = base_pdf.docinfo[key]
|
||||
return str(s)
|
||||
except (KeyError, TypeError):
|
||||
return ''
|
||||
|
||||
pdfmark = {
|
||||
k: from_document_info(k)
|
||||
for k in ('/Title', '/Author', '/Keywords', '/Subject', '/CreationDate')
|
||||
}
|
||||
if options.title:
|
||||
pdfmark['/Title'] = options.title
|
||||
if options.author:
|
||||
pdfmark['/Author'] = options.author
|
||||
if options.keywords:
|
||||
pdfmark['/Keywords'] = options.keywords
|
||||
if options.subject:
|
||||
pdfmark['/Subject'] = options.subject
|
||||
|
||||
creator_tag = context.plugin_manager.hook.get_ocr_engine().creator_tag(options)
|
||||
|
||||
pdfmark['/Creator'] = f'{PROGRAM_NAME} {VERSION} / {creator_tag}'
|
||||
pdfmark['/Producer'] = f'pikepdf {pikepdf.__version__}'
|
||||
pdfmark['/ModDate'] = encode_pdf_date(datetime.now(timezone.utc))
|
||||
return pdfmark
|
||||
|
||||
|
||||
def report_on_metadata(options, missing):
|
||||
if not missing:
|
||||
return
|
||||
if options.output_type.startswith('pdfa'):
|
||||
log.warning(
|
||||
"Some input metadata could not be copied because it is not "
|
||||
"permitted in PDF/A. You may wish to examine the output "
|
||||
"PDF's XMP metadata."
|
||||
)
|
||||
log.debug("The following metadata fields were not copied: %r", missing)
|
||||
else:
|
||||
log.error(
|
||||
"Some input metadata could not be copied."
|
||||
"You may wish to examine the output PDF's XMP metadata."
|
||||
)
|
||||
log.info("The following metadata fields were not copied: %r", missing)
|
||||
|
||||
|
||||
def repair_docinfo_nuls(pdf):
|
||||
"""If the DocumentInfo block contains NUL characters, remove them.
|
||||
|
||||
If the DocumentInfo block is malformed, log an error and continue.
|
||||
"""
|
||||
modified = False
|
||||
try:
|
||||
if not isinstance(pdf.docinfo, pikepdf.Dictionary):
|
||||
raise TypeError("DocumentInfo is not a dictionary")
|
||||
for k, v in pdf.docinfo.items():
|
||||
if isinstance(v, str) and b'\x00' in bytes(v):
|
||||
pdf.docinfo[k] = bytes(v).replace(b'\x00', b'')
|
||||
modified = True
|
||||
except TypeError:
|
||||
# TypeError can also be raised if dictionary items are unexpected types
|
||||
log.error("File contains a malformed DocumentInfo block - continuing anyway.")
|
||||
return modified
|
||||
|
||||
|
||||
def should_linearize(working_file: Path, context: PdfContext) -> bool:
|
||||
"""Determine whether the PDF should be linearized.
|
||||
|
||||
For smaller files, linearization is not worth the effort.
|
||||
"""
|
||||
filesize = os.stat(working_file).st_size
|
||||
if filesize > (context.options.fast_web_view * 1_000_000):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def metadata_fixup(
|
||||
working_file: Path, context: PdfContext, pdf_save_settings: dict[str, Any]
|
||||
) -> Path:
|
||||
"""Fix certain metadata fields after Ghostscript PDF/A conversion.
|
||||
|
||||
Also report on metadata in the input file that was not retained during
|
||||
PDF/A conversion.
|
||||
"""
|
||||
output_file = context.get_path('metafix.pdf')
|
||||
options = context.options
|
||||
|
||||
with pikepdf.open(context.origin) as original, pikepdf.open(working_file) as pdf:
|
||||
docinfo = get_docinfo(original, context)
|
||||
with original.open_metadata(
|
||||
set_pikepdf_as_editor=False, update_docinfo=False, strict=False
|
||||
) as meta_original, pdf.open_metadata() as meta_pdf:
|
||||
meta_pdf.load_from_docinfo(
|
||||
docinfo, delete_missing=False, raise_failure=False
|
||||
)
|
||||
# If xmp:CreateDate is missing, set it to the modify date to
|
||||
# ensure consistency with Ghostscript.
|
||||
if 'xmp:CreateDate' not in meta_pdf:
|
||||
meta_pdf['xmp:CreateDate'] = meta_pdf.get('xmp:ModifyDate', '')
|
||||
if meta_pdf.get('dc:title') == 'Untitled':
|
||||
# Ghostscript likes to set title to Untitled if omitted from input.
|
||||
# Reverse this, because PDF/A TechNote 0003:Metadata in PDF/A-1
|
||||
# and the XMP Spec do not make this recommendation.
|
||||
if 'dc:title' not in meta_original:
|
||||
del meta_pdf['dc:title']
|
||||
# If the user explicitly specified an empty string for any of the
|
||||
# following, they should be unset and not reported as missing in
|
||||
# the output pdf. Note that some metadata fields use differing names
|
||||
# between PDF-A and PDF.
|
||||
for meta in [meta_pdf, meta_original]:
|
||||
if options.title == '' and 'dc:title' in meta:
|
||||
del meta['dc:title'] # PDF-A and PDF
|
||||
if options.author == '':
|
||||
if 'dc:creator' in meta:
|
||||
del meta['dc:creator'] # PDF-A (Not xmp:CreatorTool)
|
||||
if 'pdf:Author' in meta:
|
||||
del meta['pdf:Author'] # PDF
|
||||
if options.subject == '':
|
||||
if 'dc:description' in meta:
|
||||
del meta['dc:description'] # PDF-A
|
||||
if 'dc:subject' in meta:
|
||||
del meta['dc:subject'] # PDF
|
||||
if options.keywords == '' and 'pdf:Keywords' in meta:
|
||||
del meta['pdf:Keywords'] # PDF-A and PDF
|
||||
meta_missing = set(meta_original.keys()) - set(meta_pdf.keys())
|
||||
report_on_metadata(options, meta_missing)
|
||||
|
||||
pdf.save(output_file, **pdf_save_settings)
|
||||
|
||||
return output_file
|
||||
+2
-136
@@ -12,21 +12,18 @@ import re
|
||||
import sys
|
||||
from collections.abc import Iterable, Iterator, Sequence
|
||||
from contextlib import suppress
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from shutil import copyfileobj
|
||||
from typing import Any, BinaryIO, TypeVar, cast
|
||||
|
||||
import img2pdf
|
||||
import pikepdf
|
||||
from pikepdf.models.metadata import encode_pdf_date
|
||||
from PIL import Image, ImageColor, ImageDraw
|
||||
|
||||
from ocrmypdf._concurrent import Executor
|
||||
from ocrmypdf._exec import unpaper
|
||||
from ocrmypdf._jobcontext import PageContext, PdfContext
|
||||
from ocrmypdf._version import PROGRAM_NAME
|
||||
from ocrmypdf._version import __version__ as VERSION
|
||||
from ocrmypdf._metadata import repair_docinfo_nuls
|
||||
from ocrmypdf.exceptions import (
|
||||
DigitalSignatureError,
|
||||
DpiError,
|
||||
@@ -767,38 +764,6 @@ def ocr_engine_textonly_pdf(
|
||||
return (output_pdf, output_text)
|
||||
|
||||
|
||||
def get_docinfo(base_pdf: pikepdf.Pdf, context: PdfContext) -> dict[str, str]:
|
||||
"""Read the document info and store it in a dictionary."""
|
||||
options = context.options
|
||||
|
||||
def from_document_info(key):
|
||||
try:
|
||||
s = base_pdf.docinfo[key]
|
||||
return str(s)
|
||||
except (KeyError, TypeError):
|
||||
return ''
|
||||
|
||||
pdfmark = {
|
||||
k: from_document_info(k)
|
||||
for k in ('/Title', '/Author', '/Keywords', '/Subject', '/CreationDate')
|
||||
}
|
||||
if options.title:
|
||||
pdfmark['/Title'] = options.title
|
||||
if options.author:
|
||||
pdfmark['/Author'] = options.author
|
||||
if options.keywords:
|
||||
pdfmark['/Keywords'] = options.keywords
|
||||
if options.subject:
|
||||
pdfmark['/Subject'] = options.subject
|
||||
|
||||
creator_tag = context.plugin_manager.hook.get_ocr_engine().creator_tag(options)
|
||||
|
||||
pdfmark['/Creator'] = f'{PROGRAM_NAME} {VERSION} / {creator_tag}'
|
||||
pdfmark['/Producer'] = f'pikepdf {pikepdf.__version__}'
|
||||
pdfmark['/ModDate'] = encode_pdf_date(datetime.now(timezone.utc))
|
||||
return pdfmark
|
||||
|
||||
|
||||
def generate_postscript_stub(context: PdfContext) -> Path:
|
||||
"""Generates a PostScript file stub for the given PDF context.
|
||||
|
||||
@@ -833,7 +798,7 @@ def convert_to_pdfa(input_pdf: Path, input_ps_stub: Path, context: PdfContext) -
|
||||
# pikepdf can deal with this, but we make the world a better place by
|
||||
# stamping them out as soon as possible.
|
||||
with pikepdf.open(input_pdf) as pdf_file:
|
||||
if _repair_docinfo_nuls(pdf_file):
|
||||
if repair_docinfo_nuls(pdf_file):
|
||||
pdf_file.save(fix_docinfo_file)
|
||||
else:
|
||||
safe_symlink(input_pdf, fix_docinfo_file)
|
||||
@@ -856,25 +821,6 @@ def convert_to_pdfa(input_pdf: Path, input_ps_stub: Path, context: PdfContext) -
|
||||
return output_file
|
||||
|
||||
|
||||
def _repair_docinfo_nuls(pdf):
|
||||
"""If the DocumentInfo block contains NUL characters, remove them.
|
||||
|
||||
If the DocumentInfo block is malformed, log an error and continue.
|
||||
"""
|
||||
modified = False
|
||||
try:
|
||||
if not isinstance(pdf.docinfo, pikepdf.Dictionary):
|
||||
raise TypeError("DocumentInfo is not a dictionary")
|
||||
for k, v in pdf.docinfo.items():
|
||||
if isinstance(v, str) and b'\x00' in bytes(v):
|
||||
pdf.docinfo[k] = bytes(v).replace(b'\x00', b'')
|
||||
modified = True
|
||||
except TypeError:
|
||||
# TypeError can also be raised if dictionary items are unexpected types
|
||||
log.error("File contains a malformed DocumentInfo block - continuing anyway.")
|
||||
return modified
|
||||
|
||||
|
||||
def should_linearize(working_file: Path, context: PdfContext) -> bool:
|
||||
"""Determine whether the PDF should be linearized.
|
||||
|
||||
@@ -909,86 +855,6 @@ def get_pdf_save_settings(output_type: str) -> dict[str, Any]:
|
||||
)
|
||||
|
||||
|
||||
def metadata_fixup(working_file: Path, context: PdfContext) -> Path:
|
||||
"""Fix certain metadata fields after Ghostscript PDF/A conversion.
|
||||
|
||||
Also report on metadata in the input file that was not retained during
|
||||
PDF/A conversion.
|
||||
"""
|
||||
output_file = context.get_path('metafix.pdf')
|
||||
options = context.options
|
||||
|
||||
def report_on_metadata(missing):
|
||||
if not missing:
|
||||
return
|
||||
if options.output_type.startswith('pdfa'):
|
||||
log.warning(
|
||||
"Some input metadata could not be copied because it is not "
|
||||
"permitted in PDF/A. You may wish to examine the output "
|
||||
"PDF's XMP metadata."
|
||||
)
|
||||
log.debug("The following metadata fields were not copied: %r", missing)
|
||||
else:
|
||||
log.error(
|
||||
"Some input metadata could not be copied."
|
||||
"You may wish to examine the output PDF's XMP metadata."
|
||||
)
|
||||
log.info("The following metadata fields were not copied: %r", missing)
|
||||
|
||||
with pikepdf.open(context.origin) as original, pikepdf.open(working_file) as pdf:
|
||||
docinfo = get_docinfo(original, context)
|
||||
with original.open_metadata(
|
||||
set_pikepdf_as_editor=False, update_docinfo=False, strict=False
|
||||
) as meta_original, pdf.open_metadata() as meta_pdf:
|
||||
meta_pdf.load_from_docinfo(
|
||||
docinfo, delete_missing=False, raise_failure=False
|
||||
)
|
||||
# If xmp:CreateDate is missing, set it to the modify date to
|
||||
# ensure consistency with Ghostscript.
|
||||
if 'xmp:CreateDate' not in meta_pdf:
|
||||
meta_pdf['xmp:CreateDate'] = meta_pdf.get('xmp:ModifyDate', '')
|
||||
if meta_pdf.get('dc:title') == 'Untitled':
|
||||
# Ghostscript likes to set title to Untitled if omitted from input.
|
||||
# Reverse this, because PDF/A TechNote 0003:Metadata in PDF/A-1
|
||||
# and the XMP Spec do not make this recommendation.
|
||||
if 'dc:title' not in meta_original:
|
||||
del meta_pdf['dc:title']
|
||||
# If the user explicitly specified an empty string for any of the
|
||||
# following, they should be unset and not reported as missing in
|
||||
# the output pdf. Note that some metadata fields use differing names
|
||||
# between PDF-A and PDF.
|
||||
for meta in [meta_pdf, meta_original]:
|
||||
if options.title == '' and 'dc:title' in meta:
|
||||
del meta['dc:title'] # PDF-A and PDF
|
||||
if options.author == '':
|
||||
if 'dc:creator' in meta:
|
||||
del meta['dc:creator'] # PDF-A (Not xmp:CreatorTool)
|
||||
if 'pdf:Author' in meta:
|
||||
del meta['pdf:Author'] # PDF
|
||||
if options.subject == '':
|
||||
if 'dc:description' in meta:
|
||||
del meta['dc:description'] # PDF-A
|
||||
if 'dc:subject' in meta:
|
||||
del meta['dc:subject'] # PDF
|
||||
if options.keywords == '' and 'pdf:Keywords' in meta:
|
||||
del meta['pdf:Keywords'] # PDF-A and PDF
|
||||
meta_missing = set(meta_original.keys()) - set(meta_pdf.keys())
|
||||
report_on_metadata(meta_missing)
|
||||
|
||||
optimizing = context.plugin_manager.hook.is_optimization_enabled(
|
||||
context=context
|
||||
)
|
||||
pdf.save(
|
||||
output_file,
|
||||
**get_pdf_save_settings(options.output_type),
|
||||
linearize=( # Don't linearize if optimize() will be linearizing too
|
||||
not optimizing and should_linearize(working_file, context)
|
||||
),
|
||||
)
|
||||
|
||||
return output_file
|
||||
|
||||
|
||||
def _file_size_ratio(
|
||||
input_file: Path, output_file: Path
|
||||
) -> tuple[float | None, float | None]:
|
||||
|
||||
@@ -15,7 +15,8 @@ from pikepdf.models.metadata import decode_pdf_date
|
||||
|
||||
from ocrmypdf._exec import ghostscript
|
||||
from ocrmypdf._jobcontext import PdfContext
|
||||
from ocrmypdf._pipeline import convert_to_pdfa, metadata_fixup
|
||||
from ocrmypdf._metadata import metadata_fixup
|
||||
from ocrmypdf._pipeline import convert_to_pdfa
|
||||
from ocrmypdf._plugin_manager import get_parser_options_plugins, get_plugin_manager
|
||||
from ocrmypdf.exceptions import ExitCode
|
||||
from ocrmypdf.pdfa import file_claims_pdfa, generate_pdfa_ps
|
||||
@@ -332,7 +333,9 @@ def test_metadata_fixup_warning(resources, outdir, caplog):
|
||||
context = PdfContext(
|
||||
options, outdir, outdir / 'graph.pdf', None, get_plugin_manager([])
|
||||
)
|
||||
metadata_fixup(working_file=outdir / 'graph.pdf', context=context)
|
||||
metadata_fixup(
|
||||
working_file=outdir / 'graph.pdf', context=context, pdf_save_settings={}
|
||||
)
|
||||
for record in caplog.records:
|
||||
assert record.levelname != 'WARNING', "Unexpected warning"
|
||||
|
||||
@@ -345,7 +348,9 @@ def test_metadata_fixup_warning(resources, outdir, caplog):
|
||||
context = PdfContext(
|
||||
options, outdir, outdir / 'graph_mod.pdf', None, get_plugin_manager([])
|
||||
)
|
||||
metadata_fixup(working_file=outdir / 'graph.pdf', context=context)
|
||||
metadata_fixup(
|
||||
working_file=outdir / 'graph.pdf', context=context, pdf_save_settings={}
|
||||
)
|
||||
assert any(record.levelname == 'WARNING' for record in caplog.records)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user