RELEASE_NOTES update prior delivery of v2.0-stable

Allow tesseract 3.02.01 to be used.
Even 3.02.01 fails in few cases (see issue #28). I decided to allow this
version anyway because 3.02.02 is not yet available for some widespread
linux distributions

LICENSE.md

@@ -1,3 +1,5 @@
+Copyright (c) 2013 fritz-hh from Github
+
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights

OCRmyPDF.sh

@@ -1,10 +1,13 @@
 #!/bin/sh
 ##############################################################################
-# Copyright (c) 2013: fritz-hh from Github (https://github.com/fritz-hh)
+# Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
 ##############################################################################
 
-TOOLNAME="OCRmyPDF"
-VERSION="v1.0-stable"
+# Import required scripts
+. "`dirname $0`/src/config.sh"
+
+# Set variables corresponding to the input parameters
+ARGUMENTS="$@"
 
 START=`date +%s`
 
@@ -15,13 +18,13 @@ Script aimed at generating a searchable PDF file from a PDF file containing only
 (The script performs optical character recognition of each respective page using the
 tesseract engine)
 
-Copyright: fritz from NAS4Free forum
+Copyright: fritz-hh  from Github (https://github.com/fritz-hh)
 Version: $VERSION
 
-Usage: OCRmyPDF.sh  [-h] [-v] [-g] [-k] [-d] [-c] [-i] [-l language] [-C filename] inputfile outputfile
+Usage: OCRmyPDF.sh  [-h] [-v] [-g] [-k] [-d] [-c] [-i] [-o dpi] [-f] [-l language] [-C filename] inputfile outputfile
 
 -h : Display this help message
--v : Increase the verbosity (this option can be used more than once)
+-v : Increase the verbosity (this option can be used more than once) (e.g. -vvv)
 -k : Do not delete the temporary files
 -g : Activate debug mode:
      - Generates a PDF file containing each page twice (once with the image, once without the image
@@ -32,14 +35,19 @@ Usage: OCRmyPDF.sh  [-h] [-v] [-g] [-k] [-d] [-c] [-i] [-l language] [-C filenam
 -c : Clean each page before performing OCR
 -i : Incorporate the cleaned image in the final PDF file (by default the original image	
      image, or the deskewed image if the -d option is set, is incorporated)
+-o : If the resolution of an image is lower than dpi value provided as argument, provide the OCR engine with 
+     an oversampled image having the latter dpi value. This can improve the OCR results but can lead to a larger output PDF file.
+     (default: no oversampling performed)
+-f : Force to OCR the whole document, even if some page already contain font data 
+     (which should not be the case for PDF files built from scnanned images) 
 -l : Set the language of the PDF file in order to improve OCR results (default "eng")
-     Any language supported by tesseract is supported.
+     Any language supported by tesseract is supported (Tesseract uses 3-character ISO 639-2 language codes)
+     Multiple languages may be specified, separated by '+' characters.
 -C : Pass an additional configuration file to the tesseract OCR engine.
      (this option can be used more than once)
-     Note: The configuration file must be available in the "tessdata/configs" folder
-     of your tesseract installation
+     Note 1: The configuration file must be available in the "tessdata/configs" folder of your tesseract installation
 inputfile  : PDF file to be OCRed
-outputfile : The PDF/A file to be generated 
+outputfile : The PDF/A file that will be generated 
 --------------------------------------------------------------------------------------
 EOF
 }
@@ -63,48 +71,38 @@ absolutePath() {
 }
 
 
-
-# Initialization of constants
-EXIT_BAD_ARGS="1"		# possible exit codes
-EXIT_BAD_INPUT_FILE="2"
-EXIT_MISSING_DEPENDENCY="3"
-EXIT_INVALID_OUPUT_PDFA="4"
-EXIT_OTHER_ERROR="5"
-LOG_ERR="0"			# 0=only error messages
-LOG_INFO="1"			# 1=error messages and some infos
-LOG_DEBUG="2"			# 2=debug level logging
-SRC="./src"				# location of the source folder (except source of external tools like jhove)
-JHOVE="./jhove/bin/JhoveApp.jar"	# java SW for validating the final PDF/A
-JHOVE_CFG="./jhove/conf/jhove.conf"	# location of the jhove config file
-
 # Initialization the configuration parameters with default values
 VERBOSITY="$LOG_ERR"		# default verbosity level
 LAN="eng"			# default language of the PDF file (required to get good OCR results)
-KEEP_TMP="0"			# do not delete the temporary files (default)
+KEEP_TMP="0"			# 0=no, 1=yes (keep the temporary files)
 PREPROCESS_DESKEW="0"		# 0=no, 1=yes (deskew image)
 PREPROCESS_CLEAN="0"		# 0=no, 1=yes (clean image to improve OCR)
 PREPROCESS_CLEANTOPDF="0"	# 0=no, 1=yes (put cleaned image in final PDF)
+OVERSAMPLING_DPI="0"		# 0=do not perform oversampling (dpi value under which oversampling should be performed)
 PDF_NOIMG="0"			# 0=no, 1=yes (generates each PDF page twice, with and without image)
+FORCE_OCR="0"			# 0=do not force, 1=force (force to OCR the whole document, even if some page already contain font data)
 TESS_CFG_FILES=""		# list of additional configuration files to be used by tesseract
 
 # Parse optional command line arguments
-while getopts ":hvgkdcil:C:" opt; do
+while getopts ":hvgkdcio:fl:C:" opt; do
 	case $opt in
 		h) usage ; exit 0 ;;
 		v) VERBOSITY=$(($VERBOSITY+1)) ;;
 		k) KEEP_TMP="1" ;;
-		g) PDF_NOIMG="1"; VERBOSITY="10"; KEEP_TMP="1" ;;
+		g) PDF_NOIMG="1"; VERBOSITY="$LOG_DEBUG"; KEEP_TMP="1" ;;
 		d) PREPROCESS_DESKEW="1" ;;
 		c) PREPROCESS_CLEAN="1" ;;
 		i) PREPROCESS_CLEANTOPDF="1" ;;
+		o) OVERSAMPLING_DPI="$OPTARG" ;;
+		f) FORCE_OCR="1" ;;
 		l) LAN="$OPTARG" ;;
 		C) TESS_CFG_FILES="$OPTARG $TESS_CFG_FILES" ;;
 		\?)
-			echo "Invalid option: -$OPTARG" >&2
+			echo "Invalid option: -$OPTARG"
 			usage
 			exit $EXIT_BAD_ARGS ;;
 		:)
-			echo "Option -$OPTARG requires an argument" >&2
+			echo "Option -$OPTARG requires an argument"
 			usage
 			exit $EXIT_BAD_ARGS ;;
 	esac
@@ -113,17 +111,18 @@ done
 # Remove the optional arguments parsed above.
 shift $((OPTIND-1))
 
-# Check if the number of mandatory parameters
-# provided is as expected
+# Check if the number of mandatory parameters provided is as expected
 if [ "$#" -ne "2" ]; then
-	echo "Exactly two mandatory argument shall be provided ($# arguments provided)" >&2
+	echo "Exactly two mandatory argument shall be provided ($# arguments provided)"
 	usage
 	exit $EXIT_BAD_ARGS
 fi
 
-! absolutePath "$1" > /dev/null && echo "The folder in which the input file should be located does not exist. Exiting..." >&2 && exit $EXIT_BAD_ARGS
+! absolutePath "$1" > /dev/null \
+	&& echo "The folder in which the input file should be located does not exist. Exiting..." && exit $EXIT_BAD_ARGS
 FILE_INPUT_PDF="`absolutePath "$1"`"
-! absolutePath "$2" > /dev/null  && echo "The folder in which the output file should be generated does not exist. Exiting..." >&2 && exit $EXIT_BAD_ARGS
+! absolutePath "$2" > /dev/null \
+	&& echo "The folder in which the output file should be generated does not exist. Exiting..." && exit $EXIT_BAD_ARGS
 FILE_OUTPUT_PDFA="`absolutePath "$2"`"
 
 
@@ -131,206 +130,139 @@ FILE_OUTPUT_PDFA="`absolutePath "$2"`"
 # set script path as working directory
 cd "`dirname $0`"
 
-[ $VERBOSITY -ge $LOG_INFO ] && echo "$TOOLNAME version: $VERSION"
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "$TOOLNAME version: $VERSION"
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Arguments: $ARGUMENTS"
 
 # check if the required utilities are installed
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Checking if all dependencies are installed"
-! command -v identify > /dev/null && echo "Please install ImageMagick. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v pdfimages > /dev/null && echo "Please install poppler-utils. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v pdftoppm > /dev/null && echo "Please install poppler-utils. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v pdftk > /dev/null && echo "Please install pdftk. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-[ $PREPROCESS_CLEAN -eq 1 ] && ! command -v unpaper > /dev/null && echo "Please install unpaper. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v tesseract > /dev/null && echo "Please install tesseract and tesseract-data. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v python > /dev/null && echo "Please install python, and the python libraries: reportlab, lxml. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v gs > /dev/null && echo "Please install ghostcript. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v java > /dev/null && echo "Please install java. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
+! command -v identify > /dev/null && echo "Please install ImageMagick. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v parallel > /dev/null && echo "Please install GNU Parallel. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v pdfimages > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v pdftoppm > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v pdffonts > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+[ $PREPROCESS_CLEAN -eq 1 ] && ! command -v unpaper > /dev/null && echo "Please install unpaper. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v tesseract > /dev/null && echo "Please install tesseract and tesseract-data. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v python2 > /dev/null && echo "Please install python v2.x. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! python2 -c 'import lxml' 2>/dev/null && echo "Please install the python library lxml. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! python2 -c 'import reportlab' 2>/dev/null && echo "Please install the python library reportlab. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v gs > /dev/null && echo "Please install ghostcript. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v java > /dev/null && echo "Please install java. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
 
 
 
+# ensure the right tesseract version is installed
+# older versions are known to produce malformed hocr output and should not be used
+# Even 3.02.01 fails in few cases (see issue #28). I decided to allow this version anyway because
+# 3.02.02 is not yet available for some widespread linux distributions
+reqtessversion="3.02.01"
+tessversion=`tesseract -v 2>&1 | grep "tesseract" | sed s/[^0-9.]//g`
+tesstooold=$(echo "`echo $tessversion | sed s/[.]//2`-`echo $reqtessversion | sed s/[.]//2` < 0" | bc)
+[ "$tesstooold" -eq "1" ] \
+	&& echo "Please install tesseract ${reqtessversion} or newer (currently installed version is ${tessversion})" && exit $EXIT_MISSING_DEPENDENCY
 
-# Initialize path to temporary files
-today=$(date +"%Y%m%d_%H%M")
-fld=$(basename "$FILE_INPUT_PDF" | sed 's/[.][^.]*//')
-TMP_FLD="./tmp/$today.filename.$fld"
-FILE_TMP="$TMP_FLD/tmp.txt"						# temporary file with a very short lifetime (may be used for several things)
-FILE_SIZE_PAGES="$TMP_FLD/page-sizes.txt"				# size in pt of the respective page of the input PDF file
-FILE_OUTPUT_PDF_CAT="${TMP_FLD}/ocred.pdf"				# concatenated OCRed PDF files
-FILE_OUTPUT_PDFA_WO_META="${TMP_FLD}/ocred-pdfa-wo-metadata.pdf"	# PDFA file before appending metadata
+# ensure the right GNU parallel version is installed
+# older version do not support -q flag (required to escape special characters)
+reqparallelversion="20130222"
+parallelversion=`parallel --minversion 0`
+! parallel --minversion "$reqparallelversion" > /dev/null \
+	&& echo "Please install GNU parallel ${reqparallelversion} or newer (currently installed version is ${parallelversion})" && exit $EXIT_MISSING_DEPENDENCY
+
+# ensure pdftoppm is provided by poppler-utils, not the older xpdf version
+! pdftoppm -v 2>&1 | grep -q 'Poppler' && echo "Please remove xpdf and install poppler-utils. Exiting..." && $EXIT_MISSING_DEPENDENCY
+
+
+
+# Display the version of the tools if log level is LOG_DEBUG
+if [ $VERBOSITY -ge $LOG_DEBUG ]; then
+	echo "--------------------------------"
+	echo "ImageMagick version:"
+	identify --version
+	echo "--------------------------------"
+	echo "GNU Parallel version:"
+	parallel --version
+	echo "--------------------------------"
+	echo "Poppler-utils version:"
+	pdfimages -v
+	pdftoppm -v
+	pdffonts -v
+	echo "--------------------------------"
+	echo "unpaper version:"
+	unpaper --version
+	echo "--------------------------------"
+	echo "tesseract version:"
+	tesseract --version
+	echo "--------------------------------"
+	echo "python2 version:"
+	python2 --version
+	echo "--------------------------------"
+	echo "Ghostscript version:"
+	gs --version
+	echo "--------------------------------"
+	echo "Java version:"
+	java -version
+	echo "--------------------------------"
+fi
+
+
+
+# check if the languages passed to tesseract are all supported
+for currentlan in `echo "$LAN" | sed 's/+/ /g'`; do
+	if ! tesseract --list-langs 2>&1 | grep "^$currentlan\$" > /dev/null; then
+		echo "The language \"$currentlan\" is not supported by tesseract."
+		tesseract --list-langs 2>&1 | tr '\n' ' '; echo
+		echo "Exiting..."
+		exit $EXIT_BAD_ARGS
+	fi
+done
+
+
+
+# Initialize path to temporary files using mktemp
+# Goal: save tmp file in a sub-folder of the $TMPDIR environment variable (or in "/tmp" if unset)
+# Unfortunately, Linux mktemp is not compatible with FreeBSD/OSX mktemp
+# Linux version requires no arg
+# FreeBSD requires '-t prefix' to be used so that $TMPDIR is taken into account
+# But in Linux '-t template' is handled differently than in FreeBSD
+# Therefore different calls must be used for Linux and for FreeBSD
+prefix="$(date +"%Y%m%d_%H%M").filename.$(basename "$FILE_INPUT_PDF" | sed 's/[.][^.]*$//')"	# prefix made of date, time and pdf file name without extension
+TMP_FLD=`mktemp -d 2>/dev/null || mktemp -d -t "${prefix}" 2>/dev/null`				# try Linux syntax first, if it fails try FreeBSD/OSX			
+if [ $? -ne 0 ]; then
+	if [ -z "$TMPDIR" ]; then
+		echo "Could not create folder for temporary files. Please ensure you have sufficient right and \"/tmp\" exists"
+	else
+		echo "Could not create folder for temporary files. Please ensure you have sufficient right and \"$TMPDIR\" exists"
+	fi
+	exit $EXIT_FILE_ACCESS_ERROR
+fi
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Created temporary folder: \"$TMP_FLD\""
+
+FILE_TMP="${TMP_FLD}/tmp.txt"						# temporary file with a very short lifetime (may be used for several things)
+FILE_PAGES_INFO="${TMP_FLD}/pages-info.txt"				# for each page: page #; width in pt; height in pt
 FILE_VALIDATION_LOG="${TMP_FLD}/pdf_validation.log"			# log file containing the results of the validation of the PDF/A file
 
-# Create tmp folder
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Creating temporary folder: \"$TMP_FLD\""
-rm -r -f "${TMP_FLD}"
-mkdir -p "${TMP_FLD}"
 
 
-
-
-# get the size of each pdf page (width / height) in pt (inch*72)
+# get the size of each pdf page (width / height) in pt (i.e. inch/72)
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Input file: Extracting size of each page (in pt)"
 ! identify -format "%w %h\n" "$FILE_INPUT_PDF" > "$FILE_TMP" \
-	&& echo "Could not get size of PDF pages. Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE
-# removing empty lines (last one should be) and prepend page # before each line
-sed '/^$/d' "$FILE_TMP" | awk '{printf "%04d %s\n", NR, $0}' > "$FILE_SIZE_PAGES"
-numpages=`tail -n 1 "$FILE_SIZE_PAGES" | cut -f1 -d" "`
+	&& echo "Could not get size of PDF pages. Exiting..." && exit $EXIT_BAD_INPUT_FILE
+# removing empty lines (last one should be) and add page # before each line
+sed '/^$/d' "$FILE_TMP" | awk '{printf "%04d %s\n", NR, $0}' > "$FILE_PAGES_INFO"
+numpages=`tail -n 1 "$FILE_PAGES_INFO" | cut -f1 -d" "`
 
-# Itterate the pages of the input pdf file
-while read pageSize ; do
+# process each page of the input pdf file
+parallel --gnu -q -k --halt-on-error 1 "$OCR_PAGE" "$FILE_INPUT_PDF" "{}" "$numpages" "$TMP_FLD" \
+	"$VERBOSITY" "$LAN" "$KEEP_TMP" "$PREPROCESS_DESKEW" "$PREPROCESS_CLEAN" "$PREPROCESS_CLEANTOPDF" "$OVERSAMPLING_DPI" \
+	"$PDF_NOIMG" "$TESS_CFG_FILES" "$FORCE_OCR" < "$FILE_PAGES_INFO"
+ret_code="$?"
+[ $ret_code -ne 0 ] && exit $ret_code 
 
-	page=`echo $pageSize | cut -f1 -d" "`
-	[ $VERBOSITY -ge $LOG_INFO ] && echo "Processing page $page / $numpages"
-	
-	# create the name of the required file
-	curOrigImg="$TMP_FLD/${page}_Image"			# original image available in the current PDF page 
-								# (the image file may have a different orientation than in the pdf file)
-	curHocr="$TMP_FLD/$page.hocr"				# hocr file to be generated by the OCR SW for the current page
-	curOCRedPDF="$TMP_FLD/${page}-ocred.pdf"		# PDF file containing the image + the OCRed text for the current page
-	curOCRedPDFDebug="$TMP_FLD/${page}-debug-ocred.pdf"	# PDF file containing data required to find out if OCR worked correctly
-	
-	# get width / height of PDF page (in pt)
-	widthPDF=`echo $pageSize | cut -f2 -d" "`
-	heightPDF=`echo $pageSize | cut -f3 -d" "`
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: size ${heightPDF}x${widthPDF} (h*w in pt)"
-	# extract raw image from pdf file to compute resolution
-	# unfortunatelly this image can have another orientation than in the pdf...
-	# so we will have to extract it again later using pdftoppm
-	pdfimages -f $page -l $page -j "$FILE_INPUT_PDF" "$curOrigImg" 1>&2	
-	# count number of extracted images
-	nbImg=`ls -1 "$curOrigImg"* | wc -l`
-	[ $nbImg -ne "1" ] && echo "Expecting exactly 1 image on page $page (found $nbImg). Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE
-	# Get characteristics of the extracted image
-	curImg=`ls -1 "$curOrigImg"*`
-	propCurImg=`identify -format "%w %h %[colorspace]" "$curImg"`
-	widthCurImg=`echo "$propCurImg" | cut -f1 -d" "`
-	heightCurImg=`echo "$propCurImg" | cut -f2 -d" "`
-	colorspaceCurImg=`echo "$propCurImg" | cut -f3 -d" "`
-	# switch height/width values if the image has not the right orientation
-	# we make here the assumption that vertical/horizontal dpi are equal
-	# we will check that later
-	if [ $((($heightPDF-$widthPDF)*($heightCurImg-$widthCurImg))) -lt 0 ]; then
-		[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracted image has wrong orientation. Inverting image height/width values"
-		tmpval=$heightCurImg
-		heightCurImg=$widthCurImg
-		widthCurImg=$tmpval
-	fi
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: size ${heightCurImg}x${widthCurImg} (h*w pixel)"	
-	# compute the resolution of the image
-	dpi_x=`echo "scale=5;$widthCurImg*72/$widthPDF" | bc`
-	dpi_y=`echo "scale=5;$heightCurImg*72/$heightPDF" | bc`
-	# compute the maximum allowed resolution difference that can be cause by:
-	# - the truncated PDF with/height in pt
-	# - the precision of dpi value
-	epsilon=`echo "scale=5;($widthCurImg*72/$widthPDF^2)+($heightCurImg*72/$heightPDF^2)+0.00002" | bc`	# max inaccuracy due to truncation of PDF size in pt
-	[ `echo "($dpi_x - $dpi_y) < $epsilon " | bc` -eq 0 -o `echo "($dpi_y - $dpi_x) < $epsilon " | bc` -eq 0 ] \
-		&& echo "Resolutions difference ($dpi_x/$dpi_y) higher than expected ($epsilon). Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE
-	dpi=`echo "scale=5;($dpi_x+$dpi_y)/2+0.5" | bc` # adding 0.5 is required for rounding
-	dpi=`echo "scale=0;$dpi/1" | bc`		# round to the nearest integer
-	
-	# Identify if page image should be saved as ppm (color) or pgm (gray)
-	ext="ppm"
-	opt=""		
-	if [ $colorspaceCurImg == "Gray" ]; then
-		ext="pgm"
-		opt="-gray"
-	fi
-	curImgPixmap="$TMP_FLD/$page.$ext"
-	curImgPixmapDeskewed="$TMP_FLD/$page.deskewed.$ext"
-	curImgPixmapClean="$TMP_FLD/$page.cleaned.$ext"
-	
-	# extract current page as image with right orientation and resoltution
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracting image as $ext file (${dpi} dpi)"
-	! pdftoppm -f $page -l $page -r $dpi $opt "$FILE_INPUT_PDF" > "$curImgPixmap" \
-		&& echo "Could not extract page $page as $ext from \"$FILE_INPUT_PDF\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
-
-	# if requested deskew image (without changing its size in pixel)
-	if [ "$PREPROCESS_DESKEW" -eq "1" ]; then
-		[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Deskewing image"
-		! convert "$curImgPixmap" -deskew 40% -gravity center -extent ${widthCurImg}x${heightCurImg} "$curImgPixmapDeskewed" \
-			&& echo "Could not deskew \"$curImgPixmap\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
-	else
-		cp "$curImgPixmap" "$curImgPixmapDeskewed"
-	fi
-
-	# if requested clean image with unpaper to get better OCR results
-	if [ "$PREPROCESS_CLEAN" -eq "1" ]; then
-		[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Cleaning image with unpaper"
-		! unpaper --dpi $dpi --mask-scan-size 100 \
-			--no-deskew --no-grayfilter --no-blackfilter --no-mask-center --no-border-align \
-			"$curImgPixmapDeskewed" "$curImgPixmapClean" 1> /dev/null \
-			&& echo "Could not clean \"$curImgPixmapDeskewed\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
-	else
-		cp "$curImgPixmapDeskewed" "$curImgPixmapClean"
-	fi
-
-	# perform OCR
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Performing OCR"
-	! tesseract -l "$LAN" "$curImgPixmapClean" "$curHocr" hocr $TESS_CFG_FILES 1> /dev/null 2> /dev/null \
-		&& echo "Could not OCR file \"$curImgPixmapClean\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
-	mv "$curHocr.html" "$curHocr"
-
-	# embed text and image to new pdf file
-	if [ "$PREPROCESS_CLEANTOPDF" -eq "1" ]; then
-		image4finalPDF="$curImgPixmapClean"
-	else
-		image4finalPDF="$curImgPixmapDeskewed"	
-	fi
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF"
-	! python $SRC/hocrTransform.py -r $dpi -i "$image4finalPDF" "$curHocr" "$curOCRedPDF" \
-		&& echo "Could not create PDF file from \"$curHocr\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
-	
-	# if requested generate special debug PDF page with visible OCR text
-	if [ $PDF_NOIMG -eq "1" ] ; then
-		[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF (debug page)"
-		! python $SRC/hocrTransform.py -b -r $dpi "$curHocr" "$curOCRedPDFDebug" \
-			&& echo "Could not create PDF file from \"$curHocr\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR	
-	fi
-	
-	# delete temporary files created for the current page
-	# to avoid using to much disk space in case of PDF files having many pages
-	if [ $KEEP_TMP -eq 0 ]; then
-		rm "$curOrigImg"*.*
-		rm "$curHocr"
-		rm "$curImgPixmap"
-		rm "$curImgPixmapDeskewed"
-		rm "$curImgPixmapClean"
-	fi
-	
-done < "$FILE_SIZE_PAGES"
-
-
-
-
-# concatenate all pages
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Concatenating all pages"
-! pdftk "${TMP_FLD}/"*-ocred.pdf cat output "$FILE_OUTPUT_PDF_CAT" \
-	&& echo "Could not concatenate individual PDF pages (\"${TMP_FLD}/*-ocred.pdf\") to one file. Exiting..." >&2 && exit $EXIT_OTHER_ERROR
-
-# convert the pdf file to match PDF/A format
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Converting to PDF/A" 
+# concatenate all pages and convert the pdf file to match PDF/A format
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Concatenating all pages to the final PDF/A file" 
 ! gs -dQUIET -dPDFA -dBATCH -dNOPAUSE -dUseCIEColor \
 	-sProcessColorModel=DeviceCMYK -sDEVICE=pdfwrite -sPDFACompatibilityPolicy=2 \
-	-sOutputFile="$FILE_OUTPUT_PDFA" "$FILE_OUTPUT_PDF_CAT" 1> /dev/null 2> /dev/null \
-	&& echo "Could not convert PDF file \"$FILE_OUTPUT_PDF_CAT\" to PDF/A. Exiting..." >&2 && exit $EXIT_OTHER_ERROR
-
-# # Write metadata
-# # Needs to be done after converting to PDF/A, as gs does not preserve metadata
-# [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Update metadata (creator, producer, and title)" 
-# title=`basename "$FILE_INPUT_PDF" | sed 's/[.][^.]*//' | \
-	# sed 's/_/ /g' | sed 's/-/ /g' | \
-	# sed 's/\([[:lower:]]\)\([[:upper:]]\)/\1 \2/g' | \
-	# sed 's/\([[:alpha:]]\)\([[:digit:]]\)/\1 \2/g' | \
-	# sed 's/\([[:digit:]]\)\([[:alpha:]]\)/\1 \2/g'`	# transform the file name (with extension) into distinct words
-# pdftk "$FILE_OUTPUT_PDFA_WO_META" update_info_utf8 - output "$FILE_OUTPUT_PDFA" << EOF
-# InfoBegin
-# InfoKey: Title
-# InfoValue: $title
-# InfoBegin
-# InfoKey: Creator
-# InfoValue: $TOOLNAME $VERSION
-# InfoBegin
-# InfoKey: Producer
-# InfoValue: ghostcript `gs --version`, pdftk
-# EOF
+	-sOutputFile="$FILE_OUTPUT_PDFA" "${TMP_FLD}/"*ocred*.pdf 1> /dev/null 2> /dev/null \
+	&& echo "Could not concatenate all pages to the final PDF/A file. Exiting..." && exit $EXIT_OTHER_ERROR
 
 # validate generated pdf file (compliance to PDF/A)
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Checking compliance to PDF/A standard" 
@@ -339,12 +271,12 @@ grep -i "Status|Message" "$FILE_VALIDATION_LOG" # summary of the validation
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "The full validation log is available here: \"$FILE_VALIDATION_LOG\""
 # check the validation results
 pdf_valid=1
-grep -i 'ErrorMessage' "$FILE_VALIDATION_LOG" >&2 && pdf_valid=0
-grep -i 'Status.*not valid' "$FILE_VALIDATION_LOG" >&2 && pdf_valid=0
-grep -i 'Status.*Not well-formed' "$FILE_VALIDATION_LOG" >&2 && pdf_valid=0
-! grep -i 'Profile:.*PDF/A-1' "$FILE_VALIDATION_LOG" > /dev/null && echo "PDF file profile is not PDF/A-1" >&2 && pdf_valid=0
-[ $pdf_valid -ne 1 ] && echo "Output file: The generated PDF/A file is INVALID" >&2
-[ $pdf_valid -ne 0 ] && [ $VERBOSITY -ge $LOG_INFO ] && echo "Output file: The generated PDF/A file is VALID"
+grep -i 'ErrorMessage' "$FILE_VALIDATION_LOG" && pdf_valid=0
+grep -i 'Status.*not valid' "$FILE_VALIDATION_LOG" && pdf_valid=0
+grep -i 'Status.*Not well-formed' "$FILE_VALIDATION_LOG" && pdf_valid=0
+! grep -i 'Profile:.*PDF/A-1' "$FILE_VALIDATION_LOG" > /dev/null && echo "PDF file profile is not PDF/A-1" && pdf_valid=0
+[ $pdf_valid -ne 1 ] && echo "Output file: The generated PDF/A file is INVALID"
+[ $pdf_valid -eq 1 ] && [ $VERBOSITY -ge $LOG_INFO ] && echo "Output file: The generated PDF/A file is VALID"
 
 
 

README.md

@@ -3,17 +3,21 @@ OCRmyPDF
 
 OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched
 
-To get the script usage, call: ./OCmyPDF.sh -h
+To get the script usage, call: sh ./OCRmyPDF.sh -h
 
-Features
+Main features
 --------
 
-- Generate a searchable PDF/A file from a PDF file containing only images
-- Place OCRed text accurately below the image to easy copy / paste
-- Keep the exact resolution of the original embedded images
-- If requested deskew and / or clean the image before performing OCR
-- Validate the generated file against the PDF/A specification using jhove
+- Generates a searchable PDF/A file from a PDF file containing only images
+- Places OCRed text accurately below the image to ease copy / paste
+- Keeps the exact resolution of the original embedded images
+    - or if requested oversamples the images before OCRing so as to get better results 
+- If requested deskews and / or clean the image before performing OCR
+- Validates the generated file against the PDF/A specification using jhove
 - Provides debug mode to enable easy verification of the OCR results
+- Processes several pages in parallel if more than one CPU core is available
+
+For details: please consult the release notes
 
 Motivation
 ----------
@@ -31,12 +35,34 @@ I found many, but none of them were really satisfying.
 ... so I decided to develop my own tool (using various existing scripts as an inspiration)
 
 Install
---------
+-------
 
-Download OCRmyPDF here: https://github.com/fritz-hh/OCRmyPDF/tags
+Download OCRmyPDF here: https://github.com/fritz-hh/OCRmyPDF/releases
 
 Copy the file in onto your linux/unix machine and extract it.
 
 Run: "sh ./OCRmyPDF.sh -h" to get the script usage
 
-If not yet installed, the script will notify you about dependencies that need to be installed
+If not yet installed, the script will notify you about dependencies that need to be installed.
+The script requires specific versions of the dependencies. Older version than the ones mentioned in the release notes are likely not to be compatible to OCRmyPDF.
+
+Support
+-------
+
+In case you detect an issue, please:
+
+- Check if your issue is already known
+- If no problem report exists on github, please create one here: https://github.com/fritz-hh/OCRmyPDF/issues
+- Describe your problem thoroughly
+- Append the console output of the script when running the debug mode (-g option)
+- If possible provide your input PDF file as well as the content of the temporary folder (using a file sharing service like www.file-upload.net)
+
+Press & Media
+-------------
+
+- c't 1-2014, page 59: Detailed presentation of OCRmyPDF v1.0 in the leading german IT magazine c't (http://www.heise.de/ct/inhalt/2014/1/58/)
+
+Disclaimer
+----------
+
+The software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

RELEASE_NOTES.md

@@ -5,6 +5,160 @@ Please always read this file before installing the package
 
 Download software here: https://github.com/fritz-hh/OCRmyPDF/tags
 
+v2.0-stable (2014-01-25):
+=======
+
+New features
+------------
+
+- Check if the language(s) passed using the -l option is supported by tesseract (fixes #60)
+
+Changes
+-------
+
+- Allow OCRmyPDF to be used with tesseract 3.02.01, even though OCR might fail for few PDF file (see issue #28). Rationale: For some linux distribution, no newer version than tesseract 3.02.01 is available
+
+Fixes
+-----
+
+- More robust algorithm for checking the version of the installed tesseract package
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.1
+- Dependencies:
+   - parallel 20130222
+   - poppler-utils 0.22.2
+   - ImageMagick 6.8.0-7 2013-03-30
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.3
+   - ghoscript (gs): 9.06
+   - java: openjdk version "1.7.0_17"
+
+v2.0-rc2 (2014-01-16):
+=======
+   
+New features
+------------
+
+- None
+
+Changes
+-------
+
+- Size reduction of final PDF file: (fixes #50)
+   - Support for monochrome (Black&White) images (massive size reduction in final PDF: >80%)
+   - Reduced size of grayscale images (by 13% on test PDF file)
+- Preventing fi, fl ligatures does not require anymore to pass an additional config file to tesseract using the -C option (fixes #58)
+- Location of temporary folder according to content of environment variable TMPDIR.
+- Dependency to pdftk removed
+- Check for compatible versions of dependencies: (fixes #51)
+   - parallel and tesseract
+   - python libraries reportlab and lxml
+
+Fixes
+-----
+
+- Improved portability with various shells (dash, bash, tcsh) and OS (FreeBSD, MAC OSX, Linux) (fixes #59)
+- Corrected bug in case the input PDF file contains a space character (fixes #48)
+- Prevent spurious error message in case there is no image in a PDF page
+- Prevent collision of temporary folder names (fixes #57)
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.1
+- Dependencies:
+   - parallel 20130222
+   - poppler-utils 0.22.2
+   - ImageMagick 6.8.0-7 2013-03-30
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.3
+   - ghoscript (gs): 9.06
+   - java: openjdk version "1.7.0_17"
+
+v2.0-rc1 (2014-01-07):
+====
+
+New features
+------------
+
+- Huge performance improvement on machines having multiple CPU/cores (processing of several pages concurrently) (fixes #18)
+- By default prevent from processing a PDF file already containing fonts (i.e. text)(it can be overridden with the -f flag) (fixes #16)
+- Warn if the resolution is too low to get reasonable OCR results (fixes #37)
+- New option (-o) to perform automatic oversampling if the image resolution is too low. This can improve OCR results.
+- Warn if using a tesseract version older than v3.02.02 (as older versions are known to produce invalid output) (fixes #41)
+- Echo version of the installed dependencies (e.g. tesseract) in debug mode in order to ease support (fixes #35)
+- Echo the arguments passed to the script in debug mode to ease support
+
+Changes
+-------
+
+- In debug mode: The debug page is now placed after the respective "normal" page
+- Reduced disk space usage in temporary folder if -d (deskew) or -c (cleanup) options are not selected
+- New file src/config.sh containing various configuration parameters
+- Documentation of the tesseract config file "tess-cfg/no_ligature" improved 
+- Improved consistency of the temporary file names
+
+Fixes
+-----
+
+- Improved robustness:
+   - in case vertical resolution differs from horizontal resolution (fixes #38)
+   - in case a PDF page contains more than one image (fixes #36)
+- Fix a problem occurring if python 3 is the standard interpreter (fixes #33)
+- Fix a problem occurring if the input PDF file contains special characters like "#" (fixes #34)
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.1
+- Dependencies:
+   - parallel 20130222
+   - poppler-utils 0.22.2
+   - ImageMagick 6.8.0-7 2013-03-30
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.3
+   - pdftk 1.45
+   - ghoscript (gs): 9.06
+   - java: openjdk version "1.7.0_17"
+
+v1.1-stable (2014-01-06):
+====
+
+New features
+------------
+
+- N/A
+
+Changes
+-------
+
+- N/A
+
+Fixes
+-----
+
+- Fixed syntax error (bashism) leading to an error message on certain systems (fixes #42)
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.1
+- Dependencies:
+   - poppler-utils 0.22.2
+   - ImageMagick 6.8.0-7 2013-03-30
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.3
+   - pdftk 1.45
+   - ghoscript (gs): 9.06
+   - java: openjdk version "1.7.0_17"
+
 v1.0-stable (2013-05-06):
 ====
 

src/config.sh

@@ -0,0 +1,32 @@
+#####################################################################################
+# The following parameters might be changed by the user
+#####################################################################################
+
+DEFAULT_DPI=300				# dpi value used as fall back if the page dpi cannot be determined
+
+#####################################################################################
+# Do NOT change the following parameters
+#####################################################################################
+
+TOOLNAME="OCRmyPDF"
+VERSION="v2.0-stable"
+
+# possible exit codes
+EXIT_BAD_ARGS="1"
+EXIT_BAD_INPUT_FILE="2"
+EXIT_MISSING_DEPENDENCY="3"
+EXIT_INVALID_OUPUT_PDFA="4"
+EXIT_FILE_ACCESS_ERROR="5"
+EXIT_OTHER_ERROR="15"
+
+# possible log levels
+LOG_ERR="0"				# only error messages
+LOG_WARN="1"				# error messages and warnings
+LOG_INFO="2"				# error messages, warnings and some infos
+LOG_DEBUG="3"				# debug level logging
+
+# various paths
+SRC="./src"				# location of the source folder (except source of external tools like jhove)
+OCR_PAGE="$SRC/ocrPage.sh"		# path to the script aimed at OCRing one page
+JHOVE="./jhove/bin/JhoveApp.jar"	# java SW for validating the final PDF/A
+JHOVE_CFG="./jhove/conf/jhove.conf"	# location of the jhove config file

src/hocrTransform.py

@@ -1,15 +1,101 @@
+#!/usr/local/bin/python2
+# coding: utf-8
 ##############################################################################
-# Copyright (c) 2013: fritz-hh from Github (https://github.com/fritz-hh)
+# Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
 #
 # Copyright (c) 2010: Jonathan Brinley from Github (https://github.com/jbrinley/HocrConverter)
 # Initial version by Jonathan Brinley, jonathanbrinley@gmail.com
 ##############################################################################
 from reportlab.pdfgen.canvas import Canvas
+from reportlab.pdfgen.pdfimages import PDFImage
 from reportlab.lib.units import inch
 from lxml import etree as ElementTree
-import Image, re, sys
+from PIL import Image
+import re, sys
 import argparse
 
+
+def monkeypatch_method(cls):
+	'''
+	Override a class method at runtime.
+
+	Rationale:
+	https://mail.python.org/pipermail/python-dev/2008-January/076194.html
+	'''
+	def decorator(func):
+		setattr(cls, func.__name__, func)
+		return func
+	return decorator
+
+
+@monkeypatch_method(PDFImage)
+def PIL_imagedata(self):
+	'''
+	Add ability to output greyscale and 1-bit PIL images without conversion to RGB.
+
+	The upstream Python 2.7 version of reportlab converts 1-bit PIL images to RGB
+	instead of saving them in a lower BPP format.  They have since added the following
+	fix to their Python 3.3 branch, but it has not been back-ported.
+
+	https://bitbucket.org/rptlab/reportlab/commits/177ddcbe4df6f9b461dac62612df9b8da3966a5d
+	'''
+	image = self.image
+	if image.format == 'JPEG':
+		fp = image.fp
+		fp.seek(0)
+		return self._jpg_imagedata(fp)
+
+	from reportlab.lib.utils import import_zlib
+	from reportlab import rl_config
+	from reportlab.pdfbase.pdfutils import _AsciiBase85Encode, _chunker
+
+	self.source = 'PIL'
+	zlib = import_zlib()
+	if not zlib:
+		return
+
+	bpc = 8
+	# Use the colorSpace in the image
+	if image.mode == 'CMYK':
+		myimage = image
+		colorSpace = 'DeviceCMYK'
+		bpp = 4
+	elif image.mode == '1':
+		myimage = image
+		colorSpace = 'DeviceGray'
+		bpp = 1
+		bpc = 1
+	elif image.mode == 'L':
+		myimage = image
+		colorSpace = 'DeviceGray'
+		bpp = 1
+	else:
+		myimage = image.convert('RGB')
+		colorSpace = 'RGB'
+		bpp = 3
+	imgwidth, imgheight = myimage.size
+
+	# this describes what is in the image itself
+	# *NB* according to the spec you can only use the short form in inline images
+
+	imagedata = ['BI /W %d /H %d /BPC %d /CS /%s /F [%s/Fl] ID' %
+				 (imgwidth, imgheight, bpc, colorSpace, rl_config.useA85 and '/A85 ' or '')]
+
+	# use a flate filter and, optionally, Ascii Base 85 to compress
+	raw = myimage.tostring()
+	rowstride = (imgwidth * bpc * bpp + 7) / 8
+	assert len(raw) == rowstride * imgheight, "Wrong amount of data for image"
+	data = zlib.compress(raw)  # this bit is very fast...
+
+	if rl_config.useA85:
+		# ...sadly this may not be
+		data = _AsciiBase85Encode(data)
+	# append in blocks of 60 characters
+	_chunker(data, imagedata)
+	imagedata.append('EI')
+	return (imagedata, imgwidth, imgheight)
+
+
 class hocrTransform():
 	"""
 	A class for converting documents from the hOCR format.
@@ -22,23 +108,24 @@ class hocrTransform():
 
 		self.hocr = ElementTree.ElementTree()
 		self.hocr.parse(hocrFileName)
- 
+
 		# if the hOCR file has a namespace, ElementTree requires its use to find elements
 		matches = re.match('({.*})html', self.hocr.getroot().tag)
 		self.xmlns = ''
 		if matches:
 			self.xmlns = matches.group(1)
-			
+
 		# get dimension in pt (not pixel!!!!) of the OCRed image
+		self.width, self.height = None, None
 		for div in self.hocr.findall(".//%sdiv[@class='ocr_page']"%(self.xmlns)):
 			coords = self.element_coordinates(div)
 			self.width = self.px2pt(coords[2]-coords[0])
 			self.height = self.px2pt(coords[3]-coords[1])
 			break # there shouldn't be more than one, and if there is, we don't want it
-			
+
 		# no width and heigh definition in the ocr_image element of the hocr file
 		if self.width is None:
-			print "No page dimension found in the hocr file"
+			print("No page dimension found in the hocr file")
 			sys.exit(1)
 
 	def __str__(self):
@@ -52,7 +139,7 @@ class hocrTransform():
 			return self._get_element_text(body).encode('utf-8') # XML gives unicode
 		else:
 			return ''
-  
+
 	def _get_element_text(self, element):
 		"""
 		Return the textual content of the element and its children
@@ -65,7 +152,7 @@ class hocrTransform():
 		if element.tail is not None:
 			text = text + element.tail
 		return text
-    
+
 	def element_coordinates(self, element):
 		"""
 		Returns a tuple containing the coordinates of the bounding box around
@@ -78,13 +165,24 @@ class hocrTransform():
 				coords = matches.group(1).split()
 				out = (int(coords[0]),int(coords[1]),int(coords[2]),int(coords[3]))
 		return out
-    
+
 	def px2pt(self, pxl):
 		"""
 		Returns the length in pt given length in pxl
 		"""
 		return float(pxl)/self.dpi*inch
-			
+
+	def replace_unsupported_chars(self, str):
+		"""
+		Given an input string, returns the corresponding string that:
+		- is available in the helvetica facetype
+		- does not contain any ligature (to allow easy search in the PDF file)
+		"""		
+		# The 'u' before the character to replace indicates that it is a unicode character
+		str=str.replace(u"fl","fl")
+		str=str.replace(u"fi","fi")
+		return str
+		
 	def to_pdf(self, outFileName, imageFileName, showBoundingboxes, fontname="Helvetica"):
 		"""
 		Creates a PDF file with an image superimposed on top of the text.
@@ -95,13 +193,13 @@ class hocrTransform():
 		"""
 		# create the PDF file
 		pdf = Canvas(outFileName, pagesize=(self.width, self.height), pageCompression=1) # page size in points (1/72 in.)
-		
+
 		# draw bounding box for each paragraph
 		pdf.setStrokeColorRGB(0,1,1)	# light blue for bounding box of paragraph
 		pdf.setFillColorRGB(0,1,1)	# light blue for bounding box of paragraph
 		pdf.setLineWidth(0)		# no line for bounding box
 		for elem in self.hocr.findall(".//%sp[@class='%s']" % (self.xmlns, "ocr_par")):
-		
+
 			elemtxt=self._get_element_text(elem).rstrip()
 			if len(elemtxt) == 0:
 				continue
@@ -111,12 +209,12 @@ class hocrTransform():
 			y1=self.px2pt(coords[1])
 			x2=self.px2pt(coords[2])
 			y2=self.px2pt(coords[3])
-			
+
 			# draw the bbox border
 			if showBoundingboxes == True:
-				pdf.rect(x1, self.height-y2, x2-x1, y2-y1, fill=1)		
-		
-		
+				pdf.rect(x1, self.height-y2, x2-x1, y2-y1, fill=1)
+
+
 		# check if element with class 'ocrx_word' are available
 		# otherwise use 'ocr_line' as fallback
 		elemclass="ocr_line"
@@ -131,6 +229,9 @@ class hocrTransform():
 		for elem in self.hocr.findall(".//%sspan[@class='%s']" % (self.xmlns, elemclass)):
 
 			elemtxt=self._get_element_text(elem).rstrip()
+			
+			elemtxt=self.replace_unsupported_chars(elemtxt)
+			
 			if len(elemtxt) == 0:
 				continue
 
@@ -139,7 +240,7 @@ class hocrTransform():
 			y1=self.px2pt(coords[1])
 			x2=self.px2pt(coords[2])
 			y2=self.px2pt(coords[3])
-			
+
 			# draw the bbox border
 			if showBoundingboxes == True:
 				pdf.rect(x1, self.height-y2, x2-x1, y2-y1, fill=0)
@@ -150,7 +251,7 @@ class hocrTransform():
 
 			# set cursor to bottom left corner of bbox (adjust for dpi)
 			text.setTextOrigin(x1, self.height-y2)
-       
+
 			# scale the width of the text to fill the width of the bbox
 			text.setHorizScale(100*(x2-x1)/pdf.stringWidth(elemtxt, fontname, fontsize))
 
@@ -160,13 +261,14 @@ class hocrTransform():
 
 		# put the image on the page, scaled to fill the page
 		if imageFileName != None:
-			im = Image.open(imageFileName)		
+			im = Image.open(imageFileName)
 			pdf.drawInlineImage(im, 0, 0, width=self.width, height=self.height)
-			
+
 		# finish up the page and save it
 		pdf.showPage()
 		pdf.save()
-  
+
+
 if __name__ == "__main__":
 	parser = argparse.ArgumentParser(description='Convert hocr file to PDF')
 	parser.add_argument('-b', '--boundingboxes', action="store_true", default=False, help='Show bounding boxes borders')
@@ -179,5 +281,5 @@ if __name__ == "__main__":
 	hocr = hocrTransform(args.hocrfile, args.resolution)
 	hocr.to_pdf(args.outputfile, args.image, args.boundingboxes)
 
-	
-	
+
+

src/ocrPage.sh

@@ -0,0 +1,220 @@
+#!/bin/sh
+##############################################################################
+# Script aimed at OCRing a single page of a PDF file
+#
+# Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
+##############################################################################
+
+. "./src/config.sh"
+
+
+# Initialization of variables passed by arguments
+FILE_INPUT_PDF="$1"			# PDF file containing the page to be OCRed
+PAGE_INFO="$2"				# Various characteristics of the page to be OCRed
+NUM_PAGES="$3"				# Total number of page of the PDF file (required for logging)
+TMP_FLD="$4"				# Folder where the temporary files should be placed
+VERBOSITY="$5"				# Requested verbosity
+LAN="$6"				# Language of the file to be OCRed
+KEEP_TMP="$7"				# Keep the temporary files after processing (helpful for debugging)
+PREPROCESS_DESKEW="$8"			# Deskew the page to be OCRed
+PREPROCESS_CLEAN="$9"			# Clean the page to be OCRed
+PREPROCESS_CLEANTOPDF="${10}"		# Put the cleaned paged in the OCRed PDF
+OVERSAMPLING_DPI="${11}"		# Oversampling resolution in dpi
+PDF_NOIMG="${12}"			# Request to generate also a PDF page containing only the OCRed text but no image (helpful for debugging) 
+TESS_CFG_FILES="${13}"			# Specific configuration files to be used by Tesseract during OCRing
+FORCE_OCR="${14}"			# Force to OCR, even if the page already contains fonts
+
+
+
+################################## 
+# Detect the characteristics of the embedded image for 
+# the page number provided as parameter
+#
+# Param 1: page number
+# Param 2: PDF page width in pt
+# Param 3: PDF page height in pt
+# Param 4: temporary file path (Path of the file in which the output should be written)
+# Output:  A file containing the characteristics of the embedded image. File structure:
+#          DPI=<dpi>
+#          COLOR_SPACE=<colorspace>
+#          DEPTH=<colordepth>
+# Returns:
+#       - 0: if no error occurs
+#       - 1: in case the page already contains fonts (which should be the case for PDF generated from scanned pages) 
+#       - 2: in case the page contains more than one image
+##################################
+getImgInfo() {
+	local page widthPDF heightPDF curImgInfo nbImg curImg propCurImg widthCurImg heightCurImg colorspaceCurImg depthCurImg dpi
+
+	# page number
+	page="$1"
+	# width / height of PDF page (in pt)
+	widthPDF="$2"
+	heightPDF="$3"
+	# path of the file in which the output should be written
+	curImgInfo="$4"
+
+	
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Size ${heightPDF}x${widthPDF} (h*w in pt)"
+	
+	
+	# check if the page already contains fonts (which should not be the case for PDF based on scanned files
+	[ `pdffonts -f $page -l $page "${FILE_INPUT_PDF}" | wc -l` -gt 2 ] && echo "Page $page: Page already contains font data !!!" && return 1
+
+	
+	# extract raw image from pdf file to compute resolution
+	# unfortunately this image can have another orientation than in the pdf...
+	# so we will have to extract it again later using pdftoppm
+	pdfimages -f $page -l $page -j "$FILE_INPUT_PDF" "$curOrigImg" 1>&2	
+	# count number of extracted images 
+	nbImg=$((`ls -1 "$curOrigImg"* 2>/dev/null | wc -l`))
+	if [ $nbImg -ne "1" ]; then
+		[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Expecting exactly 1 image covering the whole page (found $nbImg). Cannot compute dpi value."
+		return 2
+	fi
+	# Get characteristics of the extracted image
+	curImg=`ls -1 "$curOrigImg"* 2>/dev/null`
+	propCurImg=`identify -format "%w %h %[colorspace] %[depth]" "$curImg"`
+	widthCurImg=`echo "$propCurImg" | cut -f1 -d" "`
+	heightCurImg=`echo "$propCurImg" | cut -f2 -d" "`
+	colorspaceCurImg=`echo "$propCurImg" | cut -f3 -d" "`
+	depthCurImg=`echo "$propCurImg" | cut -f4 -d" "`
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Size ${heightCurImg}x${widthCurImg} (in pixel)"	
+
+	# compute the resolution of the image (making the assumption that x & y resolution are equal)
+	# and round it to the nearest integer
+	dpi=`echo "scale=5;sqrt($widthCurImg*72*$heightCurImg*72/$widthPDF/$heightPDF)+0.5" | bc`
+	dpi=`echo "scale=0;$dpi/1" | bc`
+	
+	# save the image characteristics
+	echo "DPI=$dpi" > "$curImgInfo"
+	echo "COLOR_SPACE=$colorspaceCurImg" >> "$curImgInfo"
+	echo "DEPTH=$depthCurImg" >> "$curImgInfo"
+	
+	return 0
+}
+
+
+page=`echo $PAGE_INFO | cut -f1 -d" "`
+[ $VERBOSITY -ge $LOG_INFO ] && echo "Processing page $page / $NUM_PAGES"
+
+# get width / height of PDF page (in pt)
+widthPDF=`echo $PAGE_INFO | cut -f2 -d" "`
+heightPDF=`echo $PAGE_INFO | cut -f3 -d" "`
+
+# create the name of the required temporary files
+curOrigImg="$TMP_FLD/${page}.orig-img"				# original image available in the current PDF page 
+								# (the image file may have a different orientation than in the pdf file)
+curHocr="$TMP_FLD/${page}.hocr"					# hocr file to be generated by the OCR SW for the current page
+curOCRedPDF="$TMP_FLD/${page}.ocred.pdf"			# PDF file containing the image + the OCRed text for the current page
+curOCRedPDFDebug="$TMP_FLD/${page}.ocred.todebug.pdf"		# PDF file containing data required to find out if OCR worked correctly
+curImgInfo="$TMP_FLD/${page}.orig-img-info.txt"			# Detected characteristics of the embedded image
+
+
+# auto-detect the characteristics of the embedded image
+depthCurImg="8"			# default color depth
+colorspaceCurImg="sRGB"		# default color space
+dpi=$DEFAULT_DPI		# default resolution
+
+getImgInfo "$page" "$widthPDF" "$heightPDF" "$curImgInfo"
+ret_code="$?"
+
+# in case the page contains text do not OCR, unless the FORCE_OCR flag is set
+if ([ "$ret_code" -eq "1" ] && [ "$FORCE_OCR" -eq "0" ]); then
+	echo "Page $page: Exiting... (Use the -f option to force OCRing, even though fonts are available in the input file)" && exit $EXIT_BAD_INPUT_FILE
+elif ([ "$ret_code" -eq "1" ] && [ "$FORCE_OCR" -eq "1" ]); then
+	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: OCRing anyway, assuming a default resolution of $dpi dpi"
+# in case the page contains more than one image, warn the user but go on with default parameters
+elif [ "$ret_code" -eq "2" ]; then
+	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Continuing anyway, assuming a default resolution of $dpi dpi"
+else
+	# read the image characteristics from the file
+	dpi=`cat "$curImgInfo" | grep "^DPI=" | cut -f2 -d"="`
+	colorspaceCurImg=`cat "$curImgInfo" | grep "^COLOR_SPACE=" | cut -f2 -d"="`
+	depthCurImg=`cat "$curImgInfo" | grep "^DEPTH=" | cut -f2 -d"="`
+fi
+
+# perform oversampling if the resolution is not sufficient to get good OCR results
+if [ "$dpi" -lt "$OVERSAMPLING_DPI" ]; then
+	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Low image resolution detected ($dpi dpi). Performing oversampling ($OVERSAMPLING_DPI dpi) to try to get better OCR results." 
+	dpi="$OVERSAMPLING_DPI"
+elif [ "$dpi" -lt "200" ]; then
+	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Low image resolution detected ($dpi dpi). If needed, please use the \"-o\" to try to get better OCR results." 
+fi
+	
+# Identify if page image should be saved as ppm (color), pgm (gray) or pbm (b&w)
+ext="ppm" 	# by default (color image) the extension of the extracted image is ppm
+opt=""		# by default (color image) no option as to be passed to pdftoppm
+if [ "$colorspaceCurImg" = "Gray" ] && [ "$depthCurImg" = "1" ]; then		# if monochrome (b&w)
+	ext="pbm"
+	opt="-mono"
+elif [ "$colorspaceCurImg" = "Gray" ]; then					# if gray
+	ext="pgm"
+	opt="-gray"
+fi
+curImgPixmap="$TMP_FLD/$page.$ext"
+curImgPixmapDeskewed="$TMP_FLD/$page.deskewed.$ext"
+curImgPixmapClean="$TMP_FLD/$page.cleaned.$ext"
+
+# extract current page as image with correct orientation and resolution
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracting image as $ext file (${dpi} dpi)"
+! pdftoppm -f $page -l $page -r $dpi $opt "$FILE_INPUT_PDF" > "$curImgPixmap" \
+	&& echo "Could not extract page $page as $ext from \"$FILE_INPUT_PDF\". Exiting..." && exit $EXIT_OTHER_ERROR
+
+# if requested deskew image (without changing its size in pixel)
+widthCurImg=$(($dpi*$widthPDF/72))
+heightCurImg=$(($dpi*$heightPDF/72))
+if [ "$PREPROCESS_DESKEW" -eq "1" ]; then
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Deskewing image"
+	! convert "$curImgPixmap" -deskew 40% -gravity center -extent ${widthCurImg}x${heightCurImg} "$curImgPixmapDeskewed" \
+		&& echo "Could not deskew \"$curImgPixmap\". Exiting..." && exit $EXIT_OTHER_ERROR
+else
+	ln -s `basename "$curImgPixmap"` "$curImgPixmapDeskewed"
+fi
+
+# if requested clean image with unpaper to get better OCR results
+if [ "$PREPROCESS_CLEAN" -eq "1" ]; then
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Cleaning image with unpaper"
+	! unpaper --dpi $dpi --mask-scan-size 100 \
+		--no-deskew --no-grayfilter --no-blackfilter --no-mask-center --no-border-align \
+		"$curImgPixmapDeskewed" "$curImgPixmapClean" 1> /dev/null \
+		&& echo "Could not clean \"$curImgPixmapDeskewed\". Exiting..." && exit $EXIT_OTHER_ERROR
+else
+	ln -s `basename "$curImgPixmapDeskewed"` "$curImgPixmapClean"
+fi
+
+# perform OCR
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Performing OCR"
+! tesseract -l "$LAN" "$curImgPixmapClean" "$curHocr" hocr $TESS_CFG_FILES 1> /dev/null 2> /dev/null \
+	&& echo "Could not OCR file \"$curImgPixmapClean\". Exiting..." && exit $EXIT_OTHER_ERROR
+mv "$curHocr.html" "$curHocr"
+
+# embed text and image to new pdf file
+if [ "$PREPROCESS_CLEANTOPDF" -eq "1" ]; then
+	image4finalPDF="$curImgPixmapClean"
+else
+	image4finalPDF="$curImgPixmapDeskewed"	
+fi
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF"
+! python2 $SRC/hocrTransform.py -r $dpi -i "$image4finalPDF" "$curHocr" "$curOCRedPDF" \
+	&& echo "Could not create PDF file from \"$curHocr\". Exiting..." && exit $EXIT_OTHER_ERROR
+
+# if requested generate special debug PDF page with visible OCR text
+if [ $PDF_NOIMG -eq "1" ] ; then
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF (debug page)"
+	! python2 $SRC/hocrTransform.py -b -r $dpi "$curHocr" "$curOCRedPDFDebug" \
+		&& echo "Could not create PDF file from \"$curHocr\". Exiting..." && exit $EXIT_OTHER_ERROR	
+fi
+
+# delete temporary files created for the current page
+# to avoid using to much disk space in case of PDF files having many pages
+if [ $KEEP_TMP -eq 0 ]; then
+	rm -f "$curOrigImg"*
+	rm -f "$curHocr"
+	rm -f "$curImgPixmap"
+	rm -f "$curImgPixmapDeskewed"
+	rm -f "$curImgPixmapClean"
+	rm -f "$curImgInfo"
+fi
+
+exit 0

tess-cfg/README.md

@@ -1,10 +0,0 @@
-Note
-====
-
-The file(s) located in this folder are tesseract configuration files.
-(Tesseract configuration files enable to tune the behaviour of tesseract)
-
-If needed, these files should be copied to the "tessdata/configs" folder of your tesseract installation.
-
-To request OCRmyPDF.sh to use a configuration file, please use the -C option
-

tess-cfg/no_ligature

@@ -1 +0,0 @@
-tessedit_char_blacklist fifl