basic implementation of parallel page processing
- basic implementation of parallel page processing using GNU parallel - processing around 40% faster on dual core processor
This commit is contained in:
fritz-hh
+17
-129
@@ -65,15 +65,16 @@ absolutePath() {
|
||||
|
||||
|
||||
# Initialization of constants
|
||||
EXIT_BAD_ARGS="1" # possible exit codes
|
||||
EXIT_BAD_ARGS="1" # possible exit codes
|
||||
EXIT_BAD_INPUT_FILE="2"
|
||||
EXIT_MISSING_DEPENDENCY="3"
|
||||
EXIT_INVALID_OUPUT_PDFA="4"
|
||||
EXIT_OTHER_ERROR="5"
|
||||
LOG_ERR="0" # 0=only error messages
|
||||
LOG_INFO="1" # 1=error messages and some infos
|
||||
LOG_DEBUG="2" # 2=debug level logging
|
||||
LOG_ERR="0" # 0=only error messages
|
||||
LOG_INFO="1" # 1=error messages and some infos
|
||||
LOG_DEBUG="2" # 2=debug level logging
|
||||
SRC="./src" # location of the source folder (except source of external tools like jhove)
|
||||
OCR_PAGE="$SRC/ocrPage.sh" # path to the script aimed at OCRing one page
|
||||
JHOVE="./jhove/bin/JhoveApp.jar" # java SW for validating the final PDF/A
|
||||
JHOVE_CFG="./jhove/conf/jhove.conf" # location of the jhove config file
|
||||
|
||||
@@ -121,9 +122,11 @@ if [ "$#" -ne "2" ]; then
|
||||
exit $EXIT_BAD_ARGS
|
||||
fi
|
||||
|
||||
! absolutePath "$1" > /dev/null && echo "The folder in which the input file should be located does not exist. Exiting..." >&2 && exit $EXIT_BAD_ARGS
|
||||
! absolutePath "$1" > /dev/null \
|
||||
&& echo "The folder in which the input file should be located does not exist. Exiting..." >&2 && exit $EXIT_BAD_ARGS
|
||||
FILE_INPUT_PDF="`absolutePath "$1"`"
|
||||
! absolutePath "$2" > /dev/null && echo "The folder in which the output file should be generated does not exist. Exiting..." >&2 && exit $EXIT_BAD_ARGS
|
||||
! absolutePath "$2" > /dev/null \
|
||||
&& echo "The folder in which the output file should be generated does not exist. Exiting..." >&2 && exit $EXIT_BAD_ARGS
|
||||
FILE_OUTPUT_PDFA="`absolutePath "$2"`"
|
||||
|
||||
|
||||
@@ -175,129 +178,14 @@ sed '/^$/d' "$FILE_TMP" | awk '{printf "%04d %s\n", NR, $0}' > "$FILE_PAGES_INFO
|
||||
numpages=`tail -n 1 "$FILE_PAGES_INFO" | cut -f1 -d" "`
|
||||
|
||||
# Itterate the pages of the input pdf file
|
||||
while read pageInfo ; do
|
||||
|
||||
page=`echo $pageInfo | cut -f1 -d" "`
|
||||
[ $VERBOSITY -ge $LOG_INFO ] && echo "Processing page $page / $numpages"
|
||||
|
||||
# create the name of the required file
|
||||
curOrigImg="$TMP_FLD/${page}_Image" # original image available in the current PDF page
|
||||
# (the image file may have a different orientation than in the pdf file)
|
||||
curHocr="$TMP_FLD/$page.hocr" # hocr file to be generated by the OCR SW for the current page
|
||||
curOCRedPDF="$TMP_FLD/${page}-ocred.pdf" # PDF file containing the image + the OCRed text for the current page
|
||||
curOCRedPDFDebug="$TMP_FLD/${page}-debug-ocred.pdf" # PDF file containing data required to find out if OCR worked correctly
|
||||
|
||||
# get width / height of PDF page (in pt)
|
||||
widthPDF=`echo $pageInfo | cut -f2 -d" "`
|
||||
heightPDF=`echo $pageInfo | cut -f3 -d" "`
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: size ${heightPDF}x${widthPDF} (h*w in pt)"
|
||||
# extract raw image from pdf file to compute resolution
|
||||
# unfortunatelly this image can have another orientation than in the pdf...
|
||||
# so we will have to extract it again later using pdftoppm
|
||||
pdfimages -f $page -l $page -j "$FILE_INPUT_PDF" "$curOrigImg" 1>&2
|
||||
# count number of extracted images
|
||||
nbImg=`ls -1 "$curOrigImg"* | wc -l`
|
||||
[ $nbImg -ne "1" ] && echo "Expecting exactly 1 image on page $page (found $nbImg). Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE
|
||||
# Get characteristics of the extracted image
|
||||
curImg=`ls -1 "$curOrigImg"*`
|
||||
propCurImg=`identify -format "%w %h %[colorspace]" "$curImg"`
|
||||
widthCurImg=`echo "$propCurImg" | cut -f1 -d" "`
|
||||
heightCurImg=`echo "$propCurImg" | cut -f2 -d" "`
|
||||
colorspaceCurImg=`echo "$propCurImg" | cut -f3 -d" "`
|
||||
# switch height/width values if the image has not the right orientation
|
||||
# we make here the assumption that vertical/horizontal dpi are equal
|
||||
# we will check that later
|
||||
if [ $((($heightPDF-$widthPDF)*($heightCurImg-$widthCurImg))) -lt 0 ]; then
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracted image has wrong orientation. Inverting image height/width values"
|
||||
tmpval=$heightCurImg
|
||||
heightCurImg=$widthCurImg
|
||||
widthCurImg=$tmpval
|
||||
fi
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: size ${heightCurImg}x${widthCurImg} (h*w pixel)"
|
||||
# compute the resolution of the image
|
||||
dpi_x=`echo "scale=5;$widthCurImg*72/$widthPDF" | bc`
|
||||
dpi_y=`echo "scale=5;$heightCurImg*72/$heightPDF" | bc`
|
||||
# compute the maximum allowed resolution difference that can be cause by:
|
||||
# - the truncated PDF with/height in pt
|
||||
# - the precision of dpi value
|
||||
epsilon=`echo "scale=5;($widthCurImg*72/$widthPDF^2)+($heightCurImg*72/$heightPDF^2)+0.00002" | bc` # max inaccuracy due to truncation of PDF size in pt
|
||||
[ `echo "($dpi_x - $dpi_y) < $epsilon " | bc` -eq 0 -o `echo "($dpi_y - $dpi_x) < $epsilon " | bc` -eq 0 ] \
|
||||
&& echo "Resolutions difference ($dpi_x/$dpi_y) higher than expected ($epsilon). Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE
|
||||
dpi=`echo "scale=5;($dpi_x+$dpi_y)/2+0.5" | bc` # adding 0.5 is required for rounding
|
||||
dpi=`echo "scale=0;$dpi/1" | bc` # round to the nearest integer
|
||||
|
||||
# Identify if page image should be saved as ppm (color) or pgm (gray)
|
||||
ext="ppm"
|
||||
opt=""
|
||||
if [ $colorspaceCurImg == "Gray" ]; then
|
||||
ext="pgm"
|
||||
opt="-gray"
|
||||
fi
|
||||
curImgPixmap="$TMP_FLD/$page.$ext"
|
||||
curImgPixmapDeskewed="$TMP_FLD/$page.deskewed.$ext"
|
||||
curImgPixmapClean="$TMP_FLD/$page.cleaned.$ext"
|
||||
|
||||
# extract current page as image with right orientation and resoltution
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracting image as $ext file (${dpi} dpi)"
|
||||
! pdftoppm -f $page -l $page -r $dpi $opt "$FILE_INPUT_PDF" > "$curImgPixmap" \
|
||||
&& echo "Could not extract page $page as $ext from \"$FILE_INPUT_PDF\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
|
||||
|
||||
# if requested deskew image (without changing its size in pixel)
|
||||
if [ "$PREPROCESS_DESKEW" -eq "1" ]; then
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Deskewing image"
|
||||
! convert "$curImgPixmap" -deskew 40% -gravity center -extent ${widthCurImg}x${heightCurImg} "$curImgPixmapDeskewed" \
|
||||
&& echo "Could not deskew \"$curImgPixmap\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
|
||||
else
|
||||
cp "$curImgPixmap" "$curImgPixmapDeskewed"
|
||||
fi
|
||||
|
||||
# if requested clean image with unpaper to get better OCR results
|
||||
if [ "$PREPROCESS_CLEAN" -eq "1" ]; then
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Cleaning image with unpaper"
|
||||
! unpaper --dpi $dpi --mask-scan-size 100 \
|
||||
--no-deskew --no-grayfilter --no-blackfilter --no-mask-center --no-border-align \
|
||||
"$curImgPixmapDeskewed" "$curImgPixmapClean" 1> /dev/null \
|
||||
&& echo "Could not clean \"$curImgPixmapDeskewed\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
|
||||
else
|
||||
cp "$curImgPixmapDeskewed" "$curImgPixmapClean"
|
||||
fi
|
||||
|
||||
# perform OCR
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Performing OCR"
|
||||
! tesseract -l "$LAN" "$curImgPixmapClean" "$curHocr" hocr $TESS_CFG_FILES 1> /dev/null 2> /dev/null \
|
||||
&& echo "Could not OCR file \"$curImgPixmapClean\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
|
||||
mv "$curHocr.html" "$curHocr"
|
||||
|
||||
# embed text and image to new pdf file
|
||||
if [ "$PREPROCESS_CLEANTOPDF" -eq "1" ]; then
|
||||
image4finalPDF="$curImgPixmapClean"
|
||||
else
|
||||
image4finalPDF="$curImgPixmapDeskewed"
|
||||
fi
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF"
|
||||
! python $SRC/hocrTransform.py -r $dpi -i "$image4finalPDF" "$curHocr" "$curOCRedPDF" \
|
||||
&& echo "Could not create PDF file from \"$curHocr\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
|
||||
|
||||
# if requested generate special debug PDF page with visible OCR text
|
||||
if [ $PDF_NOIMG -eq "1" ] ; then
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF (debug page)"
|
||||
! python $SRC/hocrTransform.py -b -r $dpi "$curHocr" "$curOCRedPDFDebug" \
|
||||
&& echo "Could not create PDF file from \"$curHocr\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
|
||||
fi
|
||||
|
||||
# delete temporary files created for the current page
|
||||
# to avoid using to much disk space in case of PDF files having many pages
|
||||
if [ $KEEP_TMP -eq 0 ]; then
|
||||
rm "$curOrigImg"*.*
|
||||
rm "$curHocr"
|
||||
rm "$curImgPixmap"
|
||||
rm "$curImgPixmapDeskewed"
|
||||
rm "$curImgPixmapClean"
|
||||
fi
|
||||
|
||||
done < "$FILE_PAGES_INFO"
|
||||
|
||||
|
||||
! parallel -k --halt-on-error 1 "$OCR_PAGE" "$FILE_INPUT_PDF" "{}" "$numpages" "$TMP_FLD" \
|
||||
"$VERBOSITY" "$LAN" "$KEEP_TMP" "$PREPROCESS_DESKEW" "$PREPROCESS_CLEAN" "$PREPROCESS_CLEANTOPDF" "$PDF_NOIMG" "$TESS_CFG_FILES" < "$FILE_PAGES_INFO" \
|
||||
&& exit $?
|
||||
#while read pageInfo ; do
|
||||
# ! "$OCR_PAGE" "$FILE_INPUT_PDF" "$pageInfo" "$numpages" "$TMP_FLD" \
|
||||
# "$VERBOSITY" "$LAN" "$KEEP_TMP" "$PREPROCESS_DESKEW" "$PREPROCESS_CLEAN" "$PREPROCESS_CLEANTOPDF" "$PDF_NOIMG" "$TESS_CFG_FILES" \
|
||||
# && exit $?
|
||||
#done < "$FILE_PAGES_INFO"
|
||||
|
||||
|
||||
# concatenate all pages
|
||||
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
#!/bin/sh
|
||||
##############################################################################
|
||||
# Copyright (c) 2013: fritz-hh from Github (https://github.com/fritz-hh)
|
||||
##############################################################################
|
||||
|
||||
# Initialization of constants
|
||||
EXIT_BAD_ARGS="1" # possible exit codes
|
||||
EXIT_BAD_INPUT_FILE="2"
|
||||
EXIT_MISSING_DEPENDENCY="3"
|
||||
EXIT_INVALID_OUPUT_PDFA="4"
|
||||
EXIT_OTHER_ERROR="5"
|
||||
LOG_ERR="0" # 0=only error messages
|
||||
LOG_INFO="1" # 1=error messages and some infos
|
||||
LOG_DEBUG="2" # 2=debug level logging
|
||||
|
||||
SRC="./src"
|
||||
|
||||
|
||||
# Initialization of variables passed by args
|
||||
FILE_INPUT_PDF="$1"
|
||||
pageInfo="$2"
|
||||
numpages="$3"
|
||||
TMP_FLD="$4"
|
||||
VERBOSITY="$5"
|
||||
LAN="$6"
|
||||
KEEP_TMP="$7"
|
||||
PREPROCESS_DESKEW="$8"
|
||||
PREPROCESS_CLEAN="$9"
|
||||
PREPROCESS_CLEANTOPDF="${10}"
|
||||
PDF_NOIMG="${11}"
|
||||
TESS_CFG_FILES="${12}"
|
||||
|
||||
|
||||
|
||||
page=`echo $pageInfo | cut -f1 -d" "`
|
||||
[ $VERBOSITY -ge $LOG_INFO ] && echo "Processing page $page / $numpages"
|
||||
|
||||
# create the name of the required file
|
||||
curOrigImg="$TMP_FLD/${page}_Image" # original image available in the current PDF page
|
||||
# (the image file may have a different orientation than in the pdf file)
|
||||
curHocr="$TMP_FLD/$page.hocr" # hocr file to be generated by the OCR SW for the current page
|
||||
curOCRedPDF="$TMP_FLD/${page}-ocred.pdf" # PDF file containing the image + the OCRed text for the current page
|
||||
curOCRedPDFDebug="$TMP_FLD/${page}-debug-ocred.pdf" # PDF file containing data required to find out if OCR worked correctly
|
||||
|
||||
# get width / height of PDF page (in pt)
|
||||
widthPDF=`echo $pageInfo | cut -f2 -d" "`
|
||||
heightPDF=`echo $pageInfo | cut -f3 -d" "`
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: size ${heightPDF}x${widthPDF} (h*w in pt)"
|
||||
# extract raw image from pdf file to compute resolution
|
||||
# unfortunatelly this image can have another orientation than in the pdf...
|
||||
# so we will have to extract it again later using pdftoppm
|
||||
pdfimages -f $page -l $page -j "$FILE_INPUT_PDF" "$curOrigImg" 1>&2
|
||||
# count number of extracted images
|
||||
nbImg=`ls -1 "$curOrigImg"* | wc -l`
|
||||
[ $nbImg -ne "1" ] && echo "Expecting exactly 1 image on page $page (found $nbImg). Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE
|
||||
# Get characteristics of the extracted image
|
||||
curImg=`ls -1 "$curOrigImg"*`
|
||||
propCurImg=`identify -format "%w %h %[colorspace]" "$curImg"`
|
||||
widthCurImg=`echo "$propCurImg" | cut -f1 -d" "`
|
||||
heightCurImg=`echo "$propCurImg" | cut -f2 -d" "`
|
||||
colorspaceCurImg=`echo "$propCurImg" | cut -f3 -d" "`
|
||||
# switch height/width values if the image has not the right orientation
|
||||
# we make here the assumption that vertical/horizontal dpi are equal
|
||||
# we will check that later
|
||||
if [ $((($heightPDF-$widthPDF)*($heightCurImg-$widthCurImg))) -lt 0 ]; then
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracted image has wrong orientation. Inverting image height/width values"
|
||||
tmpval=$heightCurImg
|
||||
heightCurImg=$widthCurImg
|
||||
widthCurImg=$tmpval
|
||||
fi
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: size ${heightCurImg}x${widthCurImg} (h*w pixel)"
|
||||
# compute the resolution of the image
|
||||
dpi_x=`echo "scale=5;$widthCurImg*72/$widthPDF" | bc`
|
||||
dpi_y=`echo "scale=5;$heightCurImg*72/$heightPDF" | bc`
|
||||
# compute the maximum allowed resolution difference that can be cause by:
|
||||
# - the truncated PDF with/height in pt
|
||||
# - the precision of dpi value
|
||||
epsilon=`echo "scale=5;($widthCurImg*72/$widthPDF^2)+($heightCurImg*72/$heightPDF^2)+0.00002" | bc` # max inaccuracy due to truncation of PDF size in pt
|
||||
[ `echo "($dpi_x - $dpi_y) < $epsilon " | bc` -eq 0 -o `echo "($dpi_y - $dpi_x) < $epsilon " | bc` -eq 0 ] \
|
||||
&& echo "Resolutions difference ($dpi_x/$dpi_y) higher than expected ($epsilon). Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE
|
||||
dpi=`echo "scale=5;($dpi_x+$dpi_y)/2+0.5" | bc` # adding 0.5 is required for rounding
|
||||
dpi=`echo "scale=0;$dpi/1" | bc` # round to the nearest integer
|
||||
|
||||
# Identify if page image should be saved as ppm (color) or pgm (gray)
|
||||
ext="ppm"
|
||||
opt=""
|
||||
if [ $colorspaceCurImg == "Gray" ]; then
|
||||
ext="pgm"
|
||||
opt="-gray"
|
||||
fi
|
||||
curImgPixmap="$TMP_FLD/$page.$ext"
|
||||
curImgPixmapDeskewed="$TMP_FLD/$page.deskewed.$ext"
|
||||
curImgPixmapClean="$TMP_FLD/$page.cleaned.$ext"
|
||||
|
||||
# extract current page as image with right orientation and resoltution
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracting image as $ext file (${dpi} dpi)"
|
||||
! pdftoppm -f $page -l $page -r $dpi $opt "$FILE_INPUT_PDF" > "$curImgPixmap" \
|
||||
&& echo "Could not extract page $page as $ext from \"$FILE_INPUT_PDF\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
|
||||
|
||||
# if requested deskew image (without changing its size in pixel)
|
||||
if [ "$PREPROCESS_DESKEW" -eq "1" ]; then
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Deskewing image"
|
||||
! convert "$curImgPixmap" -deskew 40% -gravity center -extent ${widthCurImg}x${heightCurImg} "$curImgPixmapDeskewed" \
|
||||
&& echo "Could not deskew \"$curImgPixmap\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
|
||||
else
|
||||
cp "$curImgPixmap" "$curImgPixmapDeskewed"
|
||||
fi
|
||||
|
||||
# if requested clean image with unpaper to get better OCR results
|
||||
if [ "$PREPROCESS_CLEAN" -eq "1" ]; then
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Cleaning image with unpaper"
|
||||
! unpaper --dpi $dpi --mask-scan-size 100 \
|
||||
--no-deskew --no-grayfilter --no-blackfilter --no-mask-center --no-border-align \
|
||||
"$curImgPixmapDeskewed" "$curImgPixmapClean" 1> /dev/null \
|
||||
&& echo "Could not clean \"$curImgPixmapDeskewed\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
|
||||
else
|
||||
cp "$curImgPixmapDeskewed" "$curImgPixmapClean"
|
||||
fi
|
||||
|
||||
# perform OCR
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Performing OCR"
|
||||
! tesseract -l "$LAN" "$curImgPixmapClean" "$curHocr" hocr $TESS_CFG_FILES 1> /dev/null 2> /dev/null \
|
||||
&& echo "Could not OCR file \"$curImgPixmapClean\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
|
||||
mv "$curHocr.html" "$curHocr"
|
||||
|
||||
# embed text and image to new pdf file
|
||||
if [ "$PREPROCESS_CLEANTOPDF" -eq "1" ]; then
|
||||
image4finalPDF="$curImgPixmapClean"
|
||||
else
|
||||
image4finalPDF="$curImgPixmapDeskewed"
|
||||
fi
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF"
|
||||
! python $SRC/hocrTransform.py -r $dpi -i "$image4finalPDF" "$curHocr" "$curOCRedPDF" \
|
||||
&& echo "Could not create PDF file from \"$curHocr\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
|
||||
|
||||
# if requested generate special debug PDF page with visible OCR text
|
||||
if [ $PDF_NOIMG -eq "1" ] ; then
|
||||
[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF (debug page)"
|
||||
! python $SRC/hocrTransform.py -b -r $dpi "$curHocr" "$curOCRedPDFDebug" \
|
||||
&& echo "Could not create PDF file from \"$curHocr\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
|
||||
fi
|
||||
|
||||
# delete temporary files created for the current page
|
||||
# to avoid using to much disk space in case of PDF files having many pages
|
||||
if [ $KEEP_TMP -eq 0 ]; then
|
||||
rm "$curOrigImg"*.*
|
||||
rm "$curHocr"
|
||||
rm "$curImgPixmap"
|
||||
rm "$curImgPixmapDeskewed"
|
||||
rm "$curImgPixmapClean"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
Reference in New Issue
Block a user