From 2b0103a4e6dcb7a5c3f37a14c05e0b1e89d96e57 Mon Sep 17 00:00:00 2001 From: fritz-hh Date: Sun, 5 May 2013 22:33:54 +0200 Subject: [PATCH] basic implementation of parallel page processing - basic implementation of parallel page processing using GNU parallel - processing around 40% faster on dual core processor --- OCRmyPDF.sh | 146 ++++++---------------------------------------- src/ocrPage.sh | 153 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 129 deletions(-) diff --git a/OCRmyPDF.sh b/OCRmyPDF.sh index 2c88f26a..7b3ef942 100644 --- a/OCRmyPDF.sh +++ b/OCRmyPDF.sh @@ -65,15 +65,16 @@ absolutePath() { # Initialization of constants -EXIT_BAD_ARGS="1" # possible exit codes +EXIT_BAD_ARGS="1" # possible exit codes EXIT_BAD_INPUT_FILE="2" EXIT_MISSING_DEPENDENCY="3" EXIT_INVALID_OUPUT_PDFA="4" EXIT_OTHER_ERROR="5" -LOG_ERR="0" # 0=only error messages -LOG_INFO="1" # 1=error messages and some infos -LOG_DEBUG="2" # 2=debug level logging +LOG_ERR="0" # 0=only error messages +LOG_INFO="1" # 1=error messages and some infos +LOG_DEBUG="2" # 2=debug level logging SRC="./src" # location of the source folder (except source of external tools like jhove) +OCR_PAGE="$SRC/ocrPage.sh" # path to the script aimed at OCRing one page JHOVE="./jhove/bin/JhoveApp.jar" # java SW for validating the final PDF/A JHOVE_CFG="./jhove/conf/jhove.conf" # location of the jhove config file @@ -121,9 +122,11 @@ if [ "$#" -ne "2" ]; then exit $EXIT_BAD_ARGS fi -! absolutePath "$1" > /dev/null && echo "The folder in which the input file should be located does not exist. Exiting..." >&2 && exit $EXIT_BAD_ARGS +! absolutePath "$1" > /dev/null \ + && echo "The folder in which the input file should be located does not exist. Exiting..." >&2 && exit $EXIT_BAD_ARGS FILE_INPUT_PDF="`absolutePath "$1"`" -! absolutePath "$2" > /dev/null && echo "The folder in which the output file should be generated does not exist. Exiting..." >&2 && exit $EXIT_BAD_ARGS +! absolutePath "$2" > /dev/null \ + && echo "The folder in which the output file should be generated does not exist. Exiting..." >&2 && exit $EXIT_BAD_ARGS FILE_OUTPUT_PDFA="`absolutePath "$2"`" @@ -175,129 +178,14 @@ sed '/^$/d' "$FILE_TMP" | awk '{printf "%04d %s\n", NR, $0}' > "$FILE_PAGES_INFO numpages=`tail -n 1 "$FILE_PAGES_INFO" | cut -f1 -d" "` # Itterate the pages of the input pdf file -while read pageInfo ; do - - page=`echo $pageInfo | cut -f1 -d" "` - [ $VERBOSITY -ge $LOG_INFO ] && echo "Processing page $page / $numpages" - - # create the name of the required file - curOrigImg="$TMP_FLD/${page}_Image" # original image available in the current PDF page - # (the image file may have a different orientation than in the pdf file) - curHocr="$TMP_FLD/$page.hocr" # hocr file to be generated by the OCR SW for the current page - curOCRedPDF="$TMP_FLD/${page}-ocred.pdf" # PDF file containing the image + the OCRed text for the current page - curOCRedPDFDebug="$TMP_FLD/${page}-debug-ocred.pdf" # PDF file containing data required to find out if OCR worked correctly - - # get width / height of PDF page (in pt) - widthPDF=`echo $pageInfo | cut -f2 -d" "` - heightPDF=`echo $pageInfo | cut -f3 -d" "` - [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: size ${heightPDF}x${widthPDF} (h*w in pt)" - # extract raw image from pdf file to compute resolution - # unfortunatelly this image can have another orientation than in the pdf... - # so we will have to extract it again later using pdftoppm - pdfimages -f $page -l $page -j "$FILE_INPUT_PDF" "$curOrigImg" 1>&2 - # count number of extracted images - nbImg=`ls -1 "$curOrigImg"* | wc -l` - [ $nbImg -ne "1" ] && echo "Expecting exactly 1 image on page $page (found $nbImg). Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE - # Get characteristics of the extracted image - curImg=`ls -1 "$curOrigImg"*` - propCurImg=`identify -format "%w %h %[colorspace]" "$curImg"` - widthCurImg=`echo "$propCurImg" | cut -f1 -d" "` - heightCurImg=`echo "$propCurImg" | cut -f2 -d" "` - colorspaceCurImg=`echo "$propCurImg" | cut -f3 -d" "` - # switch height/width values if the image has not the right orientation - # we make here the assumption that vertical/horizontal dpi are equal - # we will check that later - if [ $((($heightPDF-$widthPDF)*($heightCurImg-$widthCurImg))) -lt 0 ]; then - [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracted image has wrong orientation. Inverting image height/width values" - tmpval=$heightCurImg - heightCurImg=$widthCurImg - widthCurImg=$tmpval - fi - [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: size ${heightCurImg}x${widthCurImg} (h*w pixel)" - # compute the resolution of the image - dpi_x=`echo "scale=5;$widthCurImg*72/$widthPDF" | bc` - dpi_y=`echo "scale=5;$heightCurImg*72/$heightPDF" | bc` - # compute the maximum allowed resolution difference that can be cause by: - # - the truncated PDF with/height in pt - # - the precision of dpi value - epsilon=`echo "scale=5;($widthCurImg*72/$widthPDF^2)+($heightCurImg*72/$heightPDF^2)+0.00002" | bc` # max inaccuracy due to truncation of PDF size in pt - [ `echo "($dpi_x - $dpi_y) < $epsilon " | bc` -eq 0 -o `echo "($dpi_y - $dpi_x) < $epsilon " | bc` -eq 0 ] \ - && echo "Resolutions difference ($dpi_x/$dpi_y) higher than expected ($epsilon). Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE - dpi=`echo "scale=5;($dpi_x+$dpi_y)/2+0.5" | bc` # adding 0.5 is required for rounding - dpi=`echo "scale=0;$dpi/1" | bc` # round to the nearest integer - - # Identify if page image should be saved as ppm (color) or pgm (gray) - ext="ppm" - opt="" - if [ $colorspaceCurImg == "Gray" ]; then - ext="pgm" - opt="-gray" - fi - curImgPixmap="$TMP_FLD/$page.$ext" - curImgPixmapDeskewed="$TMP_FLD/$page.deskewed.$ext" - curImgPixmapClean="$TMP_FLD/$page.cleaned.$ext" - - # extract current page as image with right orientation and resoltution - [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracting image as $ext file (${dpi} dpi)" - ! pdftoppm -f $page -l $page -r $dpi $opt "$FILE_INPUT_PDF" > "$curImgPixmap" \ - && echo "Could not extract page $page as $ext from \"$FILE_INPUT_PDF\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR - - # if requested deskew image (without changing its size in pixel) - if [ "$PREPROCESS_DESKEW" -eq "1" ]; then - [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Deskewing image" - ! convert "$curImgPixmap" -deskew 40% -gravity center -extent ${widthCurImg}x${heightCurImg} "$curImgPixmapDeskewed" \ - && echo "Could not deskew \"$curImgPixmap\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR - else - cp "$curImgPixmap" "$curImgPixmapDeskewed" - fi - - # if requested clean image with unpaper to get better OCR results - if [ "$PREPROCESS_CLEAN" -eq "1" ]; then - [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Cleaning image with unpaper" - ! unpaper --dpi $dpi --mask-scan-size 100 \ - --no-deskew --no-grayfilter --no-blackfilter --no-mask-center --no-border-align \ - "$curImgPixmapDeskewed" "$curImgPixmapClean" 1> /dev/null \ - && echo "Could not clean \"$curImgPixmapDeskewed\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR - else - cp "$curImgPixmapDeskewed" "$curImgPixmapClean" - fi - - # perform OCR - [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Performing OCR" - ! tesseract -l "$LAN" "$curImgPixmapClean" "$curHocr" hocr $TESS_CFG_FILES 1> /dev/null 2> /dev/null \ - && echo "Could not OCR file \"$curImgPixmapClean\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR - mv "$curHocr.html" "$curHocr" - - # embed text and image to new pdf file - if [ "$PREPROCESS_CLEANTOPDF" -eq "1" ]; then - image4finalPDF="$curImgPixmapClean" - else - image4finalPDF="$curImgPixmapDeskewed" - fi - [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF" - ! python $SRC/hocrTransform.py -r $dpi -i "$image4finalPDF" "$curHocr" "$curOCRedPDF" \ - && echo "Could not create PDF file from \"$curHocr\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR - - # if requested generate special debug PDF page with visible OCR text - if [ $PDF_NOIMG -eq "1" ] ; then - [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF (debug page)" - ! python $SRC/hocrTransform.py -b -r $dpi "$curHocr" "$curOCRedPDFDebug" \ - && echo "Could not create PDF file from \"$curHocr\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR - fi - - # delete temporary files created for the current page - # to avoid using to much disk space in case of PDF files having many pages - if [ $KEEP_TMP -eq 0 ]; then - rm "$curOrigImg"*.* - rm "$curHocr" - rm "$curImgPixmap" - rm "$curImgPixmapDeskewed" - rm "$curImgPixmapClean" - fi - -done < "$FILE_PAGES_INFO" - - +! parallel -k --halt-on-error 1 "$OCR_PAGE" "$FILE_INPUT_PDF" "{}" "$numpages" "$TMP_FLD" \ + "$VERBOSITY" "$LAN" "$KEEP_TMP" "$PREPROCESS_DESKEW" "$PREPROCESS_CLEAN" "$PREPROCESS_CLEANTOPDF" "$PDF_NOIMG" "$TESS_CFG_FILES" < "$FILE_PAGES_INFO" \ + && exit $? +#while read pageInfo ; do +# ! "$OCR_PAGE" "$FILE_INPUT_PDF" "$pageInfo" "$numpages" "$TMP_FLD" \ +# "$VERBOSITY" "$LAN" "$KEEP_TMP" "$PREPROCESS_DESKEW" "$PREPROCESS_CLEAN" "$PREPROCESS_CLEANTOPDF" "$PDF_NOIMG" "$TESS_CFG_FILES" \ +# && exit $? +#done < "$FILE_PAGES_INFO" # concatenate all pages diff --git a/src/ocrPage.sh b/src/ocrPage.sh index e69de29b..ef5df7ca 100644 --- a/src/ocrPage.sh +++ b/src/ocrPage.sh @@ -0,0 +1,153 @@ +#!/bin/sh +############################################################################## +# Copyright (c) 2013: fritz-hh from Github (https://github.com/fritz-hh) +############################################################################## + +# Initialization of constants +EXIT_BAD_ARGS="1" # possible exit codes +EXIT_BAD_INPUT_FILE="2" +EXIT_MISSING_DEPENDENCY="3" +EXIT_INVALID_OUPUT_PDFA="4" +EXIT_OTHER_ERROR="5" +LOG_ERR="0" # 0=only error messages +LOG_INFO="1" # 1=error messages and some infos +LOG_DEBUG="2" # 2=debug level logging + +SRC="./src" + + +# Initialization of variables passed by args +FILE_INPUT_PDF="$1" +pageInfo="$2" +numpages="$3" +TMP_FLD="$4" +VERBOSITY="$5" +LAN="$6" +KEEP_TMP="$7" +PREPROCESS_DESKEW="$8" +PREPROCESS_CLEAN="$9" +PREPROCESS_CLEANTOPDF="${10}" +PDF_NOIMG="${11}" +TESS_CFG_FILES="${12}" + + + +page=`echo $pageInfo | cut -f1 -d" "` +[ $VERBOSITY -ge $LOG_INFO ] && echo "Processing page $page / $numpages" + +# create the name of the required file +curOrigImg="$TMP_FLD/${page}_Image" # original image available in the current PDF page + # (the image file may have a different orientation than in the pdf file) +curHocr="$TMP_FLD/$page.hocr" # hocr file to be generated by the OCR SW for the current page +curOCRedPDF="$TMP_FLD/${page}-ocred.pdf" # PDF file containing the image + the OCRed text for the current page +curOCRedPDFDebug="$TMP_FLD/${page}-debug-ocred.pdf" # PDF file containing data required to find out if OCR worked correctly + +# get width / height of PDF page (in pt) +widthPDF=`echo $pageInfo | cut -f2 -d" "` +heightPDF=`echo $pageInfo | cut -f3 -d" "` +[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: size ${heightPDF}x${widthPDF} (h*w in pt)" +# extract raw image from pdf file to compute resolution +# unfortunatelly this image can have another orientation than in the pdf... +# so we will have to extract it again later using pdftoppm +pdfimages -f $page -l $page -j "$FILE_INPUT_PDF" "$curOrigImg" 1>&2 +# count number of extracted images +nbImg=`ls -1 "$curOrigImg"* | wc -l` +[ $nbImg -ne "1" ] && echo "Expecting exactly 1 image on page $page (found $nbImg). Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE +# Get characteristics of the extracted image +curImg=`ls -1 "$curOrigImg"*` +propCurImg=`identify -format "%w %h %[colorspace]" "$curImg"` +widthCurImg=`echo "$propCurImg" | cut -f1 -d" "` +heightCurImg=`echo "$propCurImg" | cut -f2 -d" "` +colorspaceCurImg=`echo "$propCurImg" | cut -f3 -d" "` +# switch height/width values if the image has not the right orientation +# we make here the assumption that vertical/horizontal dpi are equal +# we will check that later +if [ $((($heightPDF-$widthPDF)*($heightCurImg-$widthCurImg))) -lt 0 ]; then + [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracted image has wrong orientation. Inverting image height/width values" + tmpval=$heightCurImg + heightCurImg=$widthCurImg + widthCurImg=$tmpval +fi +[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: size ${heightCurImg}x${widthCurImg} (h*w pixel)" +# compute the resolution of the image +dpi_x=`echo "scale=5;$widthCurImg*72/$widthPDF" | bc` +dpi_y=`echo "scale=5;$heightCurImg*72/$heightPDF" | bc` +# compute the maximum allowed resolution difference that can be cause by: +# - the truncated PDF with/height in pt +# - the precision of dpi value +epsilon=`echo "scale=5;($widthCurImg*72/$widthPDF^2)+($heightCurImg*72/$heightPDF^2)+0.00002" | bc` # max inaccuracy due to truncation of PDF size in pt +[ `echo "($dpi_x - $dpi_y) < $epsilon " | bc` -eq 0 -o `echo "($dpi_y - $dpi_x) < $epsilon " | bc` -eq 0 ] \ + && echo "Resolutions difference ($dpi_x/$dpi_y) higher than expected ($epsilon). Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE +dpi=`echo "scale=5;($dpi_x+$dpi_y)/2+0.5" | bc` # adding 0.5 is required for rounding +dpi=`echo "scale=0;$dpi/1" | bc` # round to the nearest integer + +# Identify if page image should be saved as ppm (color) or pgm (gray) +ext="ppm" +opt="" +if [ $colorspaceCurImg == "Gray" ]; then + ext="pgm" + opt="-gray" +fi +curImgPixmap="$TMP_FLD/$page.$ext" +curImgPixmapDeskewed="$TMP_FLD/$page.deskewed.$ext" +curImgPixmapClean="$TMP_FLD/$page.cleaned.$ext" + +# extract current page as image with right orientation and resoltution +[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracting image as $ext file (${dpi} dpi)" +! pdftoppm -f $page -l $page -r $dpi $opt "$FILE_INPUT_PDF" > "$curImgPixmap" \ + && echo "Could not extract page $page as $ext from \"$FILE_INPUT_PDF\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR + +# if requested deskew image (without changing its size in pixel) +if [ "$PREPROCESS_DESKEW" -eq "1" ]; then + [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Deskewing image" + ! convert "$curImgPixmap" -deskew 40% -gravity center -extent ${widthCurImg}x${heightCurImg} "$curImgPixmapDeskewed" \ + && echo "Could not deskew \"$curImgPixmap\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR +else + cp "$curImgPixmap" "$curImgPixmapDeskewed" +fi + +# if requested clean image with unpaper to get better OCR results +if [ "$PREPROCESS_CLEAN" -eq "1" ]; then + [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Cleaning image with unpaper" + ! unpaper --dpi $dpi --mask-scan-size 100 \ + --no-deskew --no-grayfilter --no-blackfilter --no-mask-center --no-border-align \ + "$curImgPixmapDeskewed" "$curImgPixmapClean" 1> /dev/null \ + && echo "Could not clean \"$curImgPixmapDeskewed\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR +else + cp "$curImgPixmapDeskewed" "$curImgPixmapClean" +fi + +# perform OCR +[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Performing OCR" +! tesseract -l "$LAN" "$curImgPixmapClean" "$curHocr" hocr $TESS_CFG_FILES 1> /dev/null 2> /dev/null \ + && echo "Could not OCR file \"$curImgPixmapClean\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR +mv "$curHocr.html" "$curHocr" + +# embed text and image to new pdf file +if [ "$PREPROCESS_CLEANTOPDF" -eq "1" ]; then + image4finalPDF="$curImgPixmapClean" +else + image4finalPDF="$curImgPixmapDeskewed" +fi +[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF" +! python $SRC/hocrTransform.py -r $dpi -i "$image4finalPDF" "$curHocr" "$curOCRedPDF" \ + && echo "Could not create PDF file from \"$curHocr\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR + +# if requested generate special debug PDF page with visible OCR text +if [ $PDF_NOIMG -eq "1" ] ; then + [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF (debug page)" + ! python $SRC/hocrTransform.py -b -r $dpi "$curHocr" "$curOCRedPDFDebug" \ + && echo "Could not create PDF file from \"$curHocr\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR +fi + +# delete temporary files created for the current page +# to avoid using to much disk space in case of PDF files having many pages +if [ $KEEP_TMP -eq 0 ]; then + rm "$curOrigImg"*.* + rm "$curHocr" + rm "$curImgPixmap" + rm "$curImgPixmapDeskewed" + rm "$curImgPixmapClean" +fi + +exit 0 \ No newline at end of file