Goodbye, so long, farewell, shell...
This commit is contained in:
Jim Barlow
+1
-308
@@ -3,311 +3,4 @@
|
||||
# Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
|
||||
##############################################################################
|
||||
|
||||
# Darwin/OS X has not evolved a proper readlink yet
|
||||
if [ $(uname) == "Darwin" ]; then
|
||||
function readlink() {
|
||||
python3 -c 'import os,sys; print(os.path.realpath(sys.argv[1]))' "$2"
|
||||
}
|
||||
fi
|
||||
|
||||
# Import required scripts
|
||||
BASEPATH="$(dirname $(readlink -f $0))"
|
||||
. "$BASEPATH/src/config.sh"
|
||||
|
||||
# Set variables corresponding to the input parameters
|
||||
ARGUMENTS="$@"
|
||||
|
||||
START=`date +%s`
|
||||
|
||||
usage() {
|
||||
cat << EOF
|
||||
--------------------------------------------------------------------------------------
|
||||
Script aimed at generating a searchable PDF file from a PDF file containing only images.
|
||||
(The script performs optical character recognition of each respective page using the
|
||||
tesseract engine)
|
||||
|
||||
Copyright: fritz-hh from Github (https://github.com/fritz-hh)
|
||||
Version: $VERSION
|
||||
|
||||
Usage: OCRmyPDF.sh [-h] [-v] [-g] [-k] [-d] [-c] [-i] [-o dpi] [-f] [-l language] [-C filename] inputfile outputfile
|
||||
|
||||
-h : Display this help message
|
||||
-v : Increase the verbosity (this option can be used more than once) (e.g. -vvv)
|
||||
-k : Do not delete the temporary files
|
||||
-g : Activate debug mode:
|
||||
- Generates a PDF file containing each page twice (once with the image, once without the image
|
||||
but with the OCRed text as well as the detected bounding boxes)
|
||||
- Set the verbosity to the highest possible
|
||||
- Do not delete the temporary files
|
||||
-d : Deskew each page before performing OCR
|
||||
-c : Clean each page before performing OCR
|
||||
-i : Incorporate the cleaned image in the final PDF file (by default the original image
|
||||
image, or the deskewed image if the -d option is set)
|
||||
-o : If the resolution of an image is lower than dpi value provided as argument, provide the OCR engine with
|
||||
an oversampled image having the latter dpi value. This can improve the OCR results but can lead to a larger output PDF file.
|
||||
(default: no oversampling performed)
|
||||
-f : Force to OCR the whole document, even if some page already contain font data
|
||||
(which should not be the case for PDF files built from scanned images)
|
||||
-s : If pages contain font data, do not perform processing on that page, but include the page in the final output.
|
||||
-b : Skip big pages
|
||||
-e : Use exact PDF pages with no changes other than inserting hidden OCR text layer (mutually exclusive with -d/-c/-i/-f)
|
||||
-l : Set the language of the PDF file in order to improve OCR results (default "eng")
|
||||
Any language supported by tesseract is supported (Tesseract uses 3-character ISO 639-2 language codes)
|
||||
Multiple languages may be specified, separated by '+' characters.
|
||||
-C : Pass an additional configuration file to the tesseract OCR engine.
|
||||
(this option can be used more than once)
|
||||
Note 1: The configuration file must be available in the "tessdata/configs" folder of your tesseract installation
|
||||
inputfile : PDF file to be OCRed
|
||||
outputfile : The PDF/A file that will be generated
|
||||
--------------------------------------------------------------------------------------
|
||||
EOF
|
||||
}
|
||||
|
||||
|
||||
#################################################
|
||||
# Get an absolute path from a relative path to a file
|
||||
#
|
||||
# Param1 : Relative path
|
||||
# Returns: 1 if the folder in which the file is located does not exist
|
||||
# 0 otherwise
|
||||
#################################################
|
||||
absolutePath() {
|
||||
local wdsave absolutepath
|
||||
wdsave="$(pwd)"
|
||||
! cd "$(dirname "$1")" 1> /dev/null 2> /dev/null && return 1
|
||||
absolutepath="$(pwd)/$(basename "$1")"
|
||||
cd "$wdsave"
|
||||
echo "$absolutepath"
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
# Initialization the configuration parameters with default values
|
||||
VERBOSITY="$LOG_ERR" # default verbosity level
|
||||
LAN="eng" # default language of the PDF file (required to get good OCR results)
|
||||
KEEP_TMP="0" # 0=no, 1=yes (keep the temporary files)
|
||||
PREPROCESS_DESKEW="0" # 0=no, 1=yes (deskew image)
|
||||
PREPROCESS_CLEAN="0" # 0=no, 1=yes (clean image to improve OCR)
|
||||
PREPROCESS_CLEANTOPDF="0" # 0=no, 1=yes (put cleaned image in final PDF)
|
||||
OVERSAMPLING_DPI="0" # 0=do not perform oversampling (dpi value under which oversampling should be performed)
|
||||
PDF_NOIMG="0" # 0=no, 1=yes (generates each PDF page twice, with and without image)
|
||||
FORCE_OCR="0" # 0=do not force, 1=force (force to OCR the whole document, even if some page already contain font data)
|
||||
SKIP_TEXT="0" # 0=do not skip text pages, 1=skip text pages
|
||||
SKIP_BIG="0"
|
||||
EXACT_IMAGE="0"
|
||||
TESS_CFG_FILES="" # list of additional configuration files to be used by tesseract
|
||||
|
||||
# Parse optional command line arguments
|
||||
while getopts ":hvgkdcio:fsbel:C:" opt; do
|
||||
case $opt in
|
||||
h) usage ; exit 0 ;;
|
||||
v) VERBOSITY=$(($VERBOSITY+1)) ;;
|
||||
k) KEEP_TMP="1" ;;
|
||||
g) PDF_NOIMG="1"; VERBOSITY="$LOG_DEBUG"; KEEP_TMP="1" ;;
|
||||
d) PREPROCESS_DESKEW="1" ;;
|
||||
c) PREPROCESS_CLEAN="1" ;;
|
||||
i) PREPROCESS_CLEANTOPDF="1" ;;
|
||||
o) OVERSAMPLING_DPI="$OPTARG" ;;
|
||||
f) FORCE_OCR="1" ;;
|
||||
s) SKIP_TEXT="1" ;;
|
||||
b) SKIP_BIG="1" ;;
|
||||
e) EXACT_IMAGE="1" ;;
|
||||
l) LAN="$OPTARG" ;;
|
||||
C) TESS_CFG_FILES="$OPTARG $TESS_CFG_FILES" ;;
|
||||
\?)
|
||||
echo "Invalid option: -$OPTARG"
|
||||
usage
|
||||
exit $EXIT_BAD_ARGS ;;
|
||||
:)
|
||||
echo "Option -$OPTARG requires an argument"
|
||||
usage
|
||||
exit $EXIT_BAD_ARGS ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Remove the optional arguments parsed above.
|
||||
shift $((OPTIND-1))
|
||||
|
||||
# Check if the number of mandatory parameters provided is as expected
|
||||
if [ "$#" -ne "2" ]; then
|
||||
echo "Exactly two mandatory argument shall be provided ($# arguments provided)"
|
||||
usage
|
||||
exit $EXIT_BAD_ARGS
|
||||
fi
|
||||
|
||||
if [ "$SKIP_TEXT" -eq "1" -a "$FORCE_OCR" -eq "1" ]; then
|
||||
echo "Options -f and -s are mutually exclusive; choose one or the other"
|
||||
usage
|
||||
exit $EXIT_BAD_ARGSor
|
||||
fi
|
||||
|
||||
! absolutePath "$1" > /dev/null \
|
||||
&& echo "The folder in which the input file should be located does not exist. Exiting..." && exit $EXIT_BAD_ARGS
|
||||
FILE_INPUT_PDF="`absolutePath "$1"`"
|
||||
! absolutePath "$2" > /dev/null \
|
||||
&& echo "The folder in which the output file should be generated does not exist. Exiting..." && exit $EXIT_BAD_ARGS
|
||||
FILE_OUTPUT_PDFA="`absolutePath "$2"`"
|
||||
|
||||
|
||||
# set script path as working directory
|
||||
cd "$BASEPATH"
|
||||
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "$TOOLNAME version: $VERSION"
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Arguments: $ARGUMENTS"
|
||||
|
||||
# check if the required utilities are installed
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Checking if all dependencies are installed"
|
||||
! command -v identify > /dev/null && echo "Please install ImageMagick. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
|
||||
! command -v parallel > /dev/null && echo "Please install GNU Parallel. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
|
||||
! command -v pdfimages > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
|
||||
! command -v pdffonts > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
|
||||
! command -v pdftoppm > /dev/null && echo "Please install poppler-utils with the option --enable-splash-output enabled. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
|
||||
! command -v pdfseparate > /dev/null && echo "Please install or update poppler-utils to at least 0.24.5. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
|
||||
[ $PREPROCESS_CLEAN -eq 1 ] && ! command -v unpaper > /dev/null && echo "Please install unpaper. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
|
||||
! command -v tesseract > /dev/null && echo "Please install tesseract and tesseract-data. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
|
||||
! command -v python2 > /dev/null && echo "Please install python v2.x. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
|
||||
! python3 -c 'import lxml' 2>/dev/null && echo "Please install the python library lxml. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
|
||||
! python3 -c 'import reportlab' 2>/dev/null && echo "Please install the python library reportlab. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
|
||||
! command -v gs > /dev/null && echo "Please install ghostscript. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
|
||||
! command -v java > /dev/null && echo "Please install java. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
|
||||
|
||||
|
||||
# ensure the right tesseract version is installed
|
||||
# older versions are known to produce malformed hocr output and should not be used
|
||||
# Even 3.02.01 fails in few cases (see issue #28). I decided to allow this version anyway because
|
||||
# 3.02.02 is not yet available for some widespread linux distributions
|
||||
reqtessversion="3.02.01"
|
||||
tessversion=`tesseract -v 2>&1 | grep "tesseract" | sed s/[^0-9.]//g`
|
||||
tesstooold=$(echo "`echo $tessversion | sed s/[.]//2`-`echo $reqtessversion | sed s/[.]//2` < 0" | bc)
|
||||
[ "$tesstooold" -eq "1" ] \
|
||||
&& echo "Please install tesseract ${reqtessversion} or newer (currently installed version is ${tessversion})" && exit $EXIT_MISSING_DEPENDENCY
|
||||
|
||||
# ensure the right GNU parallel version is installed
|
||||
# older version do not support -q flag (required to escape special characters)
|
||||
reqparallelversion="20121122"
|
||||
parallelversion=`parallel --minversion 0`
|
||||
! parallel --minversion "$reqparallelversion" > /dev/null \
|
||||
&& echo "Please install GNU parallel ${reqparallelversion} or newer (currently installed version is ${parallelversion})" && exit $EXIT_MISSING_DEPENDENCY
|
||||
|
||||
# ensure pdftoppm is provided by poppler-utils, not the older xpdf version
|
||||
! pdftoppm -v 2>&1 | grep -q 'Poppler' && echo "Please remove xpdf and install poppler-utils. Exiting..." && $EXIT_MISSING_DEPENDENCY
|
||||
|
||||
|
||||
# Display the version of the tools if log level is LOG_DEBUG
|
||||
if [ $VERBOSITY -ge $LOG_DEBUG ]; then
|
||||
echo "--------------------------------"
|
||||
echo "ImageMagick version:"
|
||||
identify --version
|
||||
echo "--------------------------------"
|
||||
echo "GNU Parallel version:"
|
||||
parallel --version
|
||||
echo "--------------------------------"
|
||||
echo "Poppler-utils version:"
|
||||
pdfimages -v
|
||||
pdftoppm -v
|
||||
pdffonts -v
|
||||
pdfseparate -v
|
||||
echo "--------------------------------"
|
||||
echo "unpaper version:"
|
||||
unpaper --version
|
||||
echo "--------------------------------"
|
||||
echo "tesseract version:"
|
||||
tesseract --version
|
||||
echo "--------------------------------"
|
||||
echo "python2 version:"
|
||||
python2 --version
|
||||
echo "--------------------------------"
|
||||
echo "Ghostscript version:"
|
||||
gs --version
|
||||
echo "--------------------------------"
|
||||
echo "Java version:"
|
||||
java -version
|
||||
echo "--------------------------------"
|
||||
fi
|
||||
|
||||
|
||||
# check if the languages passed to tesseract are all supported
|
||||
for currentlan in `echo "$LAN" | sed 's/+/ /g'`; do
|
||||
if ! tesseract --list-langs 2>&1 | grep "^$currentlan\$" > /dev/null; then
|
||||
echo "The language \"$currentlan\" is not supported by tesseract."
|
||||
tesseract --list-langs 2>&1 | tr '\n' ' '; echo
|
||||
echo "Exiting..."
|
||||
exit $EXIT_BAD_ARGS
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
# Initialize path to temporary files using mktemp
|
||||
# Goal: save tmp file in a sub-folder of the $TMPDIR environment variable (or in "/tmp" if unset)
|
||||
# Unfortunately, Linux mktemp is not compatible with FreeBSD/OSX mktemp
|
||||
# Linux version requires no arg
|
||||
# FreeBSD requires '-t prefix' to be used so that $TMPDIR is taken into account
|
||||
# But in Linux '-t template' is handled differently than in FreeBSD
|
||||
# Therefore different calls must be used for Linux and for FreeBSD
|
||||
prefix="com.github.ocrmypdf.$(date +"%Y%m%d_%H%M").$(basename "$FILE_INPUT_PDF" | sed 's/[.][^.]*$//')" # prefix made of date, time and pdf file name without extension
|
||||
TMP_FLD=`mktemp -d 2>/dev/null || mktemp -d -t "${prefix}" 2>/dev/null` # try Linux syntax first, if it fails try FreeBSD/OSX
|
||||
if [ $? -ne 0 ]; then
|
||||
if [ -z "$TMPDIR" ]; then
|
||||
echo "Could not create folder for temporary files. Please ensure you have sufficient right and \"/tmp\" exists"
|
||||
else
|
||||
echo "Could not create folder for temporary files. Please ensure you have sufficient right and \"$TMPDIR\" exists"
|
||||
fi
|
||||
exit $EXIT_FILE_ACCESS_ERROR
|
||||
fi
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Created temporary folder: \"$TMP_FLD\""
|
||||
|
||||
FILE_TMP="${TMP_FLD}/tmp.txt" # temporary file with a very short lifetime (may be used for several things)
|
||||
FILE_PAGES_INFO="${TMP_FLD}/pages-info.txt" # for each page: page #; width in pt; height in pt
|
||||
FILE_VALIDATION_LOG="${TMP_FLD}/pdf_validation.log" # log file containing the results of the validation of the PDF/A file
|
||||
|
||||
|
||||
# get the size of each pdf page (width / height) in pt (i.e. inch/72)
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Input file: Extracting size of each page (in pt)"
|
||||
! identify -format "%w %h\n" "$FILE_INPUT_PDF" > "$FILE_TMP" \
|
||||
&& echo "Could not get size of PDF pages. Exiting..." && exit $EXIT_BAD_INPUT_FILE
|
||||
# removing empty lines (last one should be) and add page # before each line
|
||||
sed '/^$/d' "$FILE_TMP" | awk '{printf "%04d %s\n", NR, $0}' > "$FILE_PAGES_INFO"
|
||||
numpages=`tail -n 1 "$FILE_PAGES_INFO" | cut -f1 -d" "`
|
||||
|
||||
# process each page of the input pdf file
|
||||
parallel --gnu -q -k --halt-on-error 1 python3 -m src.ocrpage \
|
||||
"$FILE_INPUT_PDF" "{}" "$numpages" "$TMP_FLD" \
|
||||
"$VERBOSITY" "$LAN" "$KEEP_TMP" "$PREPROCESS_DESKEW" "$PREPROCESS_CLEAN" "$PREPROCESS_CLEANTOPDF" "$OVERSAMPLING_DPI" \
|
||||
"$PDF_NOIMG" "$FORCE_OCR" "$SKIP_TEXT" "$SKIP_BIG" "$EXACT_IMAGE" "$TESS_CFG_FILES" < "$FILE_PAGES_INFO"
|
||||
ret_code="$?"
|
||||
[ $ret_code -ne 0 ] && exit $ret_code
|
||||
|
||||
# concatenate all pages and convert the pdf file to match PDF/A format
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Concatenating all pages to the final PDF/A file"
|
||||
! gs -dQUIET -dBATCH -dNOPAUSE -sDEVICE=pdfwrite -sColorConversionStrategy=/RGB \
|
||||
-sProcessColorModel=DeviceRGB -dPDFA -sPDFACompatibilityPolicy=2 \
|
||||
-sOutputICCProfile=srgb.icc \
|
||||
-sOutputFile="$FILE_OUTPUT_PDFA" "$(pwd)/PDFA_def.ps" "${TMP_FLD}/"*ocred*.pdf \
|
||||
&& echo "Could not concatenate all pages to the final PDF/A file. Exiting..." && exit $EXIT_OTHER_ERROR
|
||||
|
||||
# validate generated pdf file (compliance to PDF/A)
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Checking compliance to PDF/A standard"
|
||||
! java -jar "$JHOVE" -c "$JHOVE_CFG" -m PDF-hul "$FILE_OUTPUT_PDFA" 2> /dev/null 1> "$FILE_VALIDATION_LOG" \
|
||||
&& echo "Unexpected error while checking compliance to PDF/A file. Exiting..." && exit $EXIT_OTHER_ERROR
|
||||
grep -i "Status|Message" "$FILE_VALIDATION_LOG" # summary of the validation
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "The full validation log is available here: \"$FILE_VALIDATION_LOG\""
|
||||
|
||||
# check the validation results
|
||||
pdf_valid=1
|
||||
grep -i 'ErrorMessage' "$FILE_VALIDATION_LOG" && pdf_valid=0
|
||||
grep -i 'Status.*not valid' "$FILE_VALIDATION_LOG" && pdf_valid=0
|
||||
grep -i 'Status.*Not well-formed' "$FILE_VALIDATION_LOG" && pdf_valid=0
|
||||
! grep -i 'Profile:.*PDF/A-1' "$FILE_VALIDATION_LOG" > /dev/null && echo "PDF file profile is not PDF/A-1" && pdf_valid=0
|
||||
[ $pdf_valid -ne 1 ] && echo "Output file: The generated PDF/A file is INVALID"
|
||||
[ $pdf_valid -eq 1 ] && [ $VERBOSITY -ge $LOG_INFO ] && echo "Output file: The generated PDF/A file is VALID"
|
||||
|
||||
# delete temporary files
|
||||
if [ $KEEP_TMP -eq 0 ]; then
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Deleting temporary files"
|
||||
rm -r -f "${TMP_FLD}"
|
||||
fi
|
||||
|
||||
END=`date +%s`
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Script took $(($END-$START)) seconds"
|
||||
|
||||
[ $pdf_valid -ne 1 ] && exit $EXIT_INVALID_OUTPUT_PDFA || exit 0
|
||||
python3 -m src.ocrmypdf "$@"
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
#####################################################################################
|
||||
# The following parameters might be changed by the user
|
||||
#####################################################################################
|
||||
|
||||
DEFAULT_DPI=300 # dpi value used as fall back if the page dpi cannot be determined
|
||||
|
||||
#####################################################################################
|
||||
# Do NOT change the following parameters
|
||||
#####################################################################################
|
||||
|
||||
TOOLNAME="OCRmyPDF"
|
||||
VERSION="v2.1-stable"
|
||||
|
||||
# possible exit codes
|
||||
EXIT_BAD_ARGS="1"
|
||||
EXIT_BAD_INPUT_FILE="2"
|
||||
EXIT_MISSING_DEPENDENCY="3"
|
||||
EXIT_INVALID_OUTPUT_PDFA="4"
|
||||
EXIT_FILE_ACCESS_ERROR="5"
|
||||
EXIT_OTHER_ERROR="15"
|
||||
|
||||
# possible log levels
|
||||
LOG_ERR="0" # only error messages
|
||||
LOG_WARN="1" # error messages and warnings
|
||||
LOG_INFO="2" # error messages, warnings and some infos
|
||||
LOG_DEBUG="3" # debug level logging
|
||||
|
||||
# various paths
|
||||
SRC="./src" # location of the source folder (except source of external tools like jhove)
|
||||
OCR_PAGE="$SRC/ocrpage.py" # path to the script aimed at OCRing one page
|
||||
JHOVE="./jhove/bin/JhoveApp.jar" # java SW for validating the final PDF/A
|
||||
JHOVE_CFG="./jhove/conf/jhove.conf" # location of the jhove config file
|
||||
-230
@@ -1,230 +0,0 @@
|
||||
#!/bin/sh
|
||||
##############################################################################
|
||||
# Script aimed at OCRing a single page of a PDF file
|
||||
#
|
||||
# Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
|
||||
##############################################################################
|
||||
|
||||
. "./src/config.sh"
|
||||
|
||||
# Initialization of variables passed by arguments
|
||||
FILE_INPUT_PDF="$1" # PDF file containing the page to be OCRed
|
||||
PAGE_INFO="$2" # Various characteristics of the page to be OCRed
|
||||
NUM_PAGES="$3" # Total number of page of the PDF file (required for logging)
|
||||
TMP_FLD="$4" # Folder where the temporary files should be placed
|
||||
VERBOSITY="$5" # Requested verbosity
|
||||
LAN="$6" # Language of the file to be OCRed
|
||||
KEEP_TMP="$7" # Keep the temporary files after processing (helpful for debugging)
|
||||
PREPROCESS_DESKEW="$8" # Deskew the page to be OCRed
|
||||
PREPROCESS_CLEAN="$9" # Clean the page to be OCRed
|
||||
PREPROCESS_CLEANTOPDF="${10}" # Put the cleaned paged in the OCRed PDF
|
||||
OVERSAMPLING_DPI="${11}" # Oversampling resolution in dpi
|
||||
PDF_NOIMG="${12}" # Request to generate also a PDF page containing only the OCRed text but no image (helpful for debugging)
|
||||
FORCE_OCR="${13}" # Force to OCR, even if the page already contains fonts
|
||||
SKIP_TEXT="${14}" # Skip OCR on pages that contain fonts and include the page anyway
|
||||
TESS_CFG_FILES="${15}" # Specific configuration files to be used by Tesseract during OCRing
|
||||
|
||||
##################################
|
||||
# Detect the characteristics of the embedded image for
|
||||
# the page number provided as parameter
|
||||
#
|
||||
# Param 1: page number
|
||||
# Param 2: PDF page width in pt
|
||||
# Param 3: PDF page height in pt
|
||||
# Param 4: temporary file path (Path of the file in which the output should be written)
|
||||
# Output: A file containing the characteristics of the embedded image. File structure:
|
||||
# DPI=<dpi>
|
||||
# COLOR_SPACE=<colorspace>
|
||||
# DEPTH=<colordepth>
|
||||
# Returns:
|
||||
# - 0: if no error occurs
|
||||
# - 1: in case the page already contains fonts (which should be the case for PDF generated from scanned pages)
|
||||
# - 2: in case the page contains more than one image
|
||||
##################################
|
||||
getImgInfo() {
|
||||
local page widthPDF heightPDF curImgInfo nbImg curImg propCurImg widthCurImg heightCurImg colorspaceCurImg depthCurImg dpi
|
||||
|
||||
# page number
|
||||
page="$1"
|
||||
# width / height of PDF page (in pt)
|
||||
widthPDF="$2"
|
||||
heightPDF="$3"
|
||||
# path of the file in which the output should be written
|
||||
curImgInfo="$4"
|
||||
|
||||
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Size ${heightPDF}x${widthPDF} (h*w in pt)"
|
||||
|
||||
|
||||
# check if the page already contains fonts (which should not be the case for PDF based on scanned files
|
||||
if [ `pdffonts -f $page -l $page "${FILE_INPUT_PDF}" | wc -l` -gt 2 ]; then
|
||||
[ "$SKIP_TEXT" -eq "0" ] && echo "Page $page: Page already contains font data !!!"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# extract raw image from pdf file to compute resolution
|
||||
# unfortunately this image can have another orientation than in the pdf...
|
||||
# so we will have to extract it again later using pdftoppm
|
||||
pdfimages -f $page -l $page -j "$FILE_INPUT_PDF" "$curOrigImg" 1>&2
|
||||
# count number of extracted images
|
||||
nbImg=$((`ls -1 "$curOrigImg"* 2>/dev/null | wc -l`))
|
||||
if [ $nbImg -ne "1" ]; then
|
||||
[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Expecting exactly 1 image covering the whole page (found $nbImg). Cannot compute dpi value."
|
||||
return 2
|
||||
fi
|
||||
# Get characteristics of the extracted image
|
||||
curImg=`ls -1 "$curOrigImg"* 2>/dev/null`
|
||||
propCurImg=`identify -format "%w %h %[colorspace] %[depth]" "$curImg"`
|
||||
widthCurImg=`echo "$propCurImg" | cut -f1 -d" "`
|
||||
heightCurImg=`echo "$propCurImg" | cut -f2 -d" "`
|
||||
colorspaceCurImg=`echo "$propCurImg" | cut -f3 -d" "`
|
||||
depthCurImg=`echo "$propCurImg" | cut -f4 -d" "`
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Size ${heightCurImg}x${widthCurImg} (in pixel)"
|
||||
|
||||
# compute the resolution of the image (making the assumption that x & y resolution are equal)
|
||||
# and round it to the nearest integer
|
||||
dpi=`echo "scale=5;sqrt($widthCurImg*72*$heightCurImg*72/$widthPDF/$heightPDF)+0.5" | bc`
|
||||
dpi=`echo "scale=0;$dpi/1" | bc`
|
||||
|
||||
# save the image characteristics
|
||||
echo "DPI=$dpi" > "$curImgInfo"
|
||||
echo "COLOR_SPACE=$colorspaceCurImg" >> "$curImgInfo"
|
||||
echo "DEPTH=$depthCurImg" >> "$curImgInfo"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
page=`echo $PAGE_INFO | cut -f1 -d" "`
|
||||
[ $VERBOSITY -ge $LOG_INFO ] && echo "Processing page $page / $NUM_PAGES"
|
||||
|
||||
# get width / height of PDF page (in pt)
|
||||
widthPDF=`echo $PAGE_INFO | cut -f2 -d" "`
|
||||
heightPDF=`echo $PAGE_INFO | cut -f3 -d" "`
|
||||
|
||||
# create the name of the required temporary files
|
||||
curOrigImg="$TMP_FLD/${page}.orig-img" # original image available in the current PDF page
|
||||
# (the image file may have a different orientation than in the pdf file)
|
||||
curHocr="$TMP_FLD/${page}.hocr" # hocr file to be generated by the OCR SW for the current page
|
||||
curOCRedPDF="$TMP_FLD/${page}.ocred.pdf" # PDF file containing the image + the OCRed text for the current page
|
||||
curOCRedPDFDebug="$TMP_FLD/${page}.ocred.todebug.pdf" # PDF file containing data required to find out if OCR worked correctly
|
||||
curImgInfo="$TMP_FLD/${page}.orig-img-info.txt" # Detected characteristics of the embedded image
|
||||
|
||||
|
||||
# auto-detect the characteristics of the embedded image
|
||||
depthCurImg="8" # default color depth
|
||||
colorspaceCurImg="sRGB" # default color space
|
||||
dpi=$DEFAULT_DPI # default resolution
|
||||
|
||||
getImgInfo "$page" "$widthPDF" "$heightPDF" "$curImgInfo"
|
||||
ret_code="$?"
|
||||
|
||||
# Handle pages that already contain a text layer
|
||||
if ([ "$ret_code" -eq "1" ] && [ "$SKIP_TEXT" -eq "1" ]); then
|
||||
echo "Page $page: Skipping OCR on this page since it already contains text"
|
||||
pdfseparate -f $page -l $page ${FILE_INPUT_PDF} $curOCRedPDF
|
||||
exit 0
|
||||
elif ([ "$ret_code" -eq "1" ] && [ "$FORCE_OCR" -eq "0" ]); then
|
||||
echo "Page $page: Exiting... (Use the -f option to force OCRing, even though fonts are available in the input file)" && exit $EXIT_BAD_INPUT_FILE
|
||||
elif ([ "$ret_code" -eq "1" ] && [ "$FORCE_OCR" -eq "1" ]); then
|
||||
[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: OCRing anyway, assuming a default resolution of $dpi dpi"
|
||||
# in case the page contains more than one image, warn the user but go on with default parameters
|
||||
elif [ "$ret_code" -eq "2" ]; then
|
||||
[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Continuing anyway, assuming a default resolution of $dpi dpi"
|
||||
else
|
||||
# read the image characteristics from the file
|
||||
dpi=`cat "$curImgInfo" | grep "^DPI=" | cut -f2 -d"="`
|
||||
colorspaceCurImg=`cat "$curImgInfo" | grep "^COLOR_SPACE=" | cut -f2 -d"="`
|
||||
depthCurImg=`cat "$curImgInfo" | grep "^DEPTH=" | cut -f2 -d"="`
|
||||
fi
|
||||
|
||||
# perform oversampling if the resolution is not sufficient to get good OCR results
|
||||
if [ "$dpi" -lt "$OVERSAMPLING_DPI" ]; then
|
||||
[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Low image resolution detected ($dpi dpi). Performing oversampling ($OVERSAMPLING_DPI dpi) to try to get better OCR results."
|
||||
dpi="$OVERSAMPLING_DPI"
|
||||
elif [ "$dpi" -lt "200" ]; then
|
||||
[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Low image resolution detected ($dpi dpi). If needed, please use the \"-o\" to try to get better OCR results."
|
||||
fi
|
||||
|
||||
# Identify if page image should be saved as ppm (color), pgm (gray) or pbm (b&w)
|
||||
ext="ppm" # by default (color image) the extension of the extracted image is ppm
|
||||
opt="" # by default (color image) no option as to be passed to pdftoppm
|
||||
if [ "$colorspaceCurImg" = "Gray" ] && [ "$depthCurImg" = "1" ]; then # if monochrome (b&w)
|
||||
ext="pbm"
|
||||
opt="-mono"
|
||||
elif [ "$colorspaceCurImg" = "Gray" ]; then # if gray
|
||||
ext="pgm"
|
||||
opt="-gray"
|
||||
fi
|
||||
curImgPixmap="$TMP_FLD/$page.$ext"
|
||||
curImgPixmapDeskewed="$TMP_FLD/$page.deskewed.$ext"
|
||||
curImgPixmapClean="$TMP_FLD/$page.cleaned.$ext"
|
||||
|
||||
# extract current page as image with correct orientation and resolution
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracting image as $ext file (${dpi} dpi)"
|
||||
! pdftoppm -f $page -l $page -r $dpi $opt "$FILE_INPUT_PDF" > "$curImgPixmap" \
|
||||
&& echo "Could not extract page $page as $ext from \"$FILE_INPUT_PDF\". Exiting..." && exit $EXIT_OTHER_ERROR
|
||||
|
||||
# if requested deskew image (without changing its size in pixel)
|
||||
widthCurImg=$(($dpi*$widthPDF/72))
|
||||
heightCurImg=$(($dpi*$heightPDF/72))
|
||||
if [ "$PREPROCESS_DESKEW" -eq "1" ]; then
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Deskewing image"
|
||||
! python2 $SRC/leptonica.py deskew -r $dpi "$curImgPixmap" "$curImgPixmapDeskewed" && exit $?
|
||||
else
|
||||
ln -s `basename "$curImgPixmap"` "$curImgPixmapDeskewed"
|
||||
fi
|
||||
|
||||
# if requested clean image with unpaper to get better OCR results
|
||||
if [ "$PREPROCESS_CLEAN" -eq "1" ]; then
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Cleaning image with unpaper"
|
||||
! unpaper --dpi $dpi --mask-scan-size 100 \
|
||||
--no-deskew --no-grayfilter --no-blackfilter --no-mask-center --no-border-align \
|
||||
"$curImgPixmapDeskewed" "$curImgPixmapClean" 1> /dev/null \
|
||||
&& echo "Could not clean \"$curImgPixmapDeskewed\". Exiting..." && exit $EXIT_OTHER_ERROR
|
||||
else
|
||||
ln -s `basename "$curImgPixmapDeskewed"` "$curImgPixmapClean"
|
||||
fi
|
||||
|
||||
# perform OCR
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Performing OCR"
|
||||
! tesseract -l "$LAN" "$curImgPixmapClean" "$curHocr" hocr $TESS_CFG_FILES 1> /dev/null 2> /dev/null \
|
||||
&& echo "Could not OCR file \"$curImgPixmapClean\". Exiting..." && exit $EXIT_OTHER_ERROR
|
||||
# Tesseract names the output files differently in some distributions.
|
||||
if [ -e "$curHocr.html" ]; then
|
||||
mv "$curHocr.html" "$curHocr"
|
||||
elif [ -e "$curHocr.hocr" ]; then
|
||||
mv "$curHocr.hocr" "$curHocr"
|
||||
elif [ ! -e "$curHocr" ]; then
|
||||
echo "\"$curHocr[.html|.hocr]\" not found. Exiting..." && exit $EXIT_OTHER_ERROR
|
||||
fi
|
||||
|
||||
# embed text and image to new pdf file
|
||||
if [ "$PREPROCESS_CLEANTOPDF" -eq "1" ]; then
|
||||
image4finalPDF="$curImgPixmapClean"
|
||||
else
|
||||
image4finalPDF="$curImgPixmapDeskewed"
|
||||
fi
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF"
|
||||
! python2 $SRC/hocrtransform.py -r $dpi -i "$image4finalPDF" "$curHocr" "$curOCRedPDF" \
|
||||
&& echo "Could not create PDF file from \"$curHocr\". Exiting..." && exit $EXIT_OTHER_ERROR
|
||||
|
||||
# if requested generate special debug PDF page with visible OCR text
|
||||
if [ $PDF_NOIMG -eq "1" ] ; then
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF (debug page)"
|
||||
! python2 $SRC/hocrtransform.py -b -r $dpi "$curHocr" "$curOCRedPDFDebug" \
|
||||
&& echo "Could not create PDF file from \"$curHocr\". Exiting..." && exit $EXIT_OTHER_ERROR
|
||||
fi
|
||||
|
||||
# delete temporary files created for the current page
|
||||
# to avoid using to much disk space in case of PDF files having many pages
|
||||
if [ $KEEP_TMP -eq 0 ]; then
|
||||
rm -f "$curOrigImg"*
|
||||
rm -f "$curHocr"
|
||||
rm -f "$curImgPixmap"
|
||||
rm -f "$curImgPixmapDeskewed"
|
||||
rm -f "$curImgPixmapClean"
|
||||
rm -f "$curImgInfo"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
+1
-1
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import print_function
|
||||
from subprocess import Popen, PIPE
|
||||
import os
|
||||
import shutil
|
||||
@@ -7,7 +8,6 @@ from contextlib import suppress
|
||||
import sys
|
||||
|
||||
if sys.version_info.major < 3:
|
||||
from __future__ import print_function
|
||||
print("Requires Python 3.4+")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user