update release notes for v2.2-stable

Python does not map the expression to its return code automatically, so
this line returns success regardless of the reportlab version installed.
(I also realized that hasattr is superfluous).

.gitignore

@@ -1,2 +1,3 @@
 tmp/
-log/
+log/
+*.pyc

LICENSE.md

@@ -1,3 +1,5 @@
+Copyright (c) 2013 fritz-hh from Github
+
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights

OCRmyPDF.sh

@@ -1,10 +1,17 @@
 #!/bin/sh
 ##############################################################################
-# Copyright (c) 2013: fritz-hh from Github (https://github.com/fritz-hh)
+# Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
 ##############################################################################
 
-TOOLNAME="OCRmyPDF"
-VERSION="v1.0-stable"
+# Determine real path of this script, following symlinks if present
+! command -v python2 > /dev/null && echo "Please install python v2.x. Exiting..." && exit 1
+BASEPATH="$(dirname $(python2 -c "import os; print os.path.realpath(\"$0\")"))"
+
+# Import required scripts
+. "$BASEPATH/src/config.sh"
+
+# Set variables corresponding to the input parameters
+ARGUMENTS="$@"
 
 START=`date +%s`
 
@@ -15,13 +22,13 @@ Script aimed at generating a searchable PDF file from a PDF file containing only
 (The script performs optical character recognition of each respective page using the
 tesseract engine)
 
-Copyright: fritz from NAS4Free forum
+Copyright: fritz-hh  from Github (https://github.com/fritz-hh)
 Version: $VERSION
 
-Usage: OCRmyPDF.sh  [-h] [-v] [-g] [-k] [-d] [-c] [-i] [-l language] [-C filename] inputfile outputfile
+Usage: OCRmyPDF.sh  [-h] [-v] [-g] [-k] [-d] [-c] [-i] [-o dpi] [-f] [-l language] [-C filename] inputfile outputfile
 
 -h : Display this help message
--v : Increase the verbosity (this option can be used more than once)
+-v : Increase the verbosity (this option can be used more than once) (e.g. -vvv)
 -k : Do not delete the temporary files
 -g : Activate debug mode:
      - Generates a PDF file containing each page twice (once with the image, once without the image
@@ -31,15 +38,20 @@ Usage: OCRmyPDF.sh  [-h] [-v] [-g] [-k] [-d] [-c] [-i] [-l language] [-C filenam
 -d : Deskew each page before performing OCR
 -c : Clean each page before performing OCR
 -i : Incorporate the cleaned image in the final PDF file (by default the original image	
-     image, or the deskewed image if the -d option is set, is incorporated)
+     image, or the deskewed image if the -d option is set)
+-o : If the resolution of an image is lower than dpi value provided as argument, provide the OCR engine with 
+     an oversampled image having the latter dpi value. This can improve the OCR results but can lead to a larger output PDF file.
+     (default: no oversampling performed)
+-f : Force to OCR the whole document, even if some page already contain font data 
+     (which should not be the case for PDF files built from scanned images) 
 -l : Set the language of the PDF file in order to improve OCR results (default "eng")
-     Any language supported by tesseract is supported.
+     Any language supported by tesseract is supported (Tesseract uses 3-character ISO 639-2 language codes)
+     Multiple languages may be specified, separated by '+' characters.
 -C : Pass an additional configuration file to the tesseract OCR engine.
      (this option can be used more than once)
-     Note: The configuration file must be available in the "tessdata/configs" folder
-     of your tesseract installation
+     Note 1: The configuration file must be available in the "tessdata/configs" folder of your tesseract installation
 inputfile  : PDF file to be OCRed
-outputfile : The PDF/A file to be generated 
+outputfile : The PDF/A file that will be generated 
 --------------------------------------------------------------------------------------
 EOF
 }
@@ -63,48 +75,38 @@ absolutePath() {
 }
 
 
-
-# Initialization of constants
-EXIT_BAD_ARGS="1"		# possible exit codes
-EXIT_BAD_INPUT_FILE="2"
-EXIT_MISSING_DEPENDENCY="3"
-EXIT_INVALID_OUPUT_PDFA="4"
-EXIT_OTHER_ERROR="5"
-LOG_ERR="0"			# 0=only error messages
-LOG_INFO="1"			# 1=error messages and some infos
-LOG_DEBUG="2"			# 2=debug level logging
-SRC="./src"				# location of the source folder (except source of external tools like jhove)
-JHOVE="./jhove/bin/JhoveApp.jar"	# java SW for validating the final PDF/A
-JHOVE_CFG="./jhove/conf/jhove.conf"	# location of the jhove config file
-
 # Initialization the configuration parameters with default values
 VERBOSITY="$LOG_ERR"		# default verbosity level
 LAN="eng"			# default language of the PDF file (required to get good OCR results)
-KEEP_TMP="0"			# do not delete the temporary files (default)
+KEEP_TMP="0"			# 0=no, 1=yes (keep the temporary files)
 PREPROCESS_DESKEW="0"		# 0=no, 1=yes (deskew image)
 PREPROCESS_CLEAN="0"		# 0=no, 1=yes (clean image to improve OCR)
 PREPROCESS_CLEANTOPDF="0"	# 0=no, 1=yes (put cleaned image in final PDF)
+OVERSAMPLING_DPI="0"		# 0=do not perform oversampling (dpi value under which oversampling should be performed)
 PDF_NOIMG="0"			# 0=no, 1=yes (generates each PDF page twice, with and without image)
+FORCE_OCR="0"			# 0=do not force, 1=force (force to OCR the whole document, even if some page already contain font data)
 TESS_CFG_FILES=""		# list of additional configuration files to be used by tesseract
 
 # Parse optional command line arguments
-while getopts ":hvgkdcil:C:" opt; do
+while getopts ":hvgkdcio:fl:C:" opt; do
 	case $opt in
 		h) usage ; exit 0 ;;
 		v) VERBOSITY=$(($VERBOSITY+1)) ;;
 		k) KEEP_TMP="1" ;;
-		g) PDF_NOIMG="1"; VERBOSITY="10"; KEEP_TMP="1" ;;
+		g) PDF_NOIMG="1"; VERBOSITY="$LOG_DEBUG"; KEEP_TMP="1" ;;
 		d) PREPROCESS_DESKEW="1" ;;
 		c) PREPROCESS_CLEAN="1" ;;
 		i) PREPROCESS_CLEANTOPDF="1" ;;
+		o) OVERSAMPLING_DPI="$OPTARG" ;;
+		f) FORCE_OCR="1" ;;
 		l) LAN="$OPTARG" ;;
 		C) TESS_CFG_FILES="$OPTARG $TESS_CFG_FILES" ;;
 		\?)
-			echo "Invalid option: -$OPTARG" >&2
+			echo "Invalid option: -$OPTARG"
 			usage
 			exit $EXIT_BAD_ARGS ;;
 		:)
-			echo "Option -$OPTARG requires an argument" >&2
+			echo "Option -$OPTARG requires an argument"
 			usage
 			exit $EXIT_BAD_ARGS ;;
 	esac
@@ -113,238 +115,174 @@ done
 # Remove the optional arguments parsed above.
 shift $((OPTIND-1))
 
-# Check if the number of mandatory parameters
-# provided is as expected
+# Check if the number of mandatory parameters provided is as expected
 if [ "$#" -ne "2" ]; then
-	echo "Exactly two mandatory argument shall be provided ($# arguments provided)" >&2
+	echo "Exactly two mandatory argument shall be provided ($# arguments provided)"
 	usage
 	exit $EXIT_BAD_ARGS
 fi
 
-! absolutePath "$1" > /dev/null && echo "The folder in which the input file should be located does not exist. Exiting..." >&2 && exit $EXIT_BAD_ARGS
+[ ! -f "$1" ] \
+	&& echo "The input file does not exist. Exiting..." && exit $EXIT_BAD_ARGS
 FILE_INPUT_PDF="`absolutePath "$1"`"
-! absolutePath "$2" > /dev/null  && echo "The folder in which the output file should be generated does not exist. Exiting..." >&2 && exit $EXIT_BAD_ARGS
+
+! absolutePath "$2" > /dev/null \
+	&& echo "The folder in which the output file should be generated does not exist. Exiting..." && exit $EXIT_BAD_ARGS
 FILE_OUTPUT_PDFA="`absolutePath "$2"`"
 
 
 
 # set script path as working directory
-cd "`dirname $0`"
+cd "$BASEPATH"
 
-[ $VERBOSITY -ge $LOG_INFO ] && echo "$TOOLNAME version: $VERSION"
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "$TOOLNAME version: $VERSION"
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Arguments: $ARGUMENTS"
 
 # check if the required utilities are installed
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Checking if all dependencies are installed"
-! command -v identify > /dev/null && echo "Please install ImageMagick. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v pdfimages > /dev/null && echo "Please install poppler-utils. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v pdftoppm > /dev/null && echo "Please install poppler-utils. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v pdftk > /dev/null && echo "Please install pdftk. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-[ $PREPROCESS_CLEAN -eq 1 ] && ! command -v unpaper > /dev/null && echo "Please install unpaper. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v tesseract > /dev/null && echo "Please install tesseract and tesseract-data. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v python > /dev/null && echo "Please install python, and the python libraries: reportlab, lxml. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v gs > /dev/null && echo "Please install ghostcript. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v java > /dev/null && echo "Please install java. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
+! command -v identify > /dev/null && echo "Please install ImageMagick. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v parallel > /dev/null && echo "Please install GNU Parallel. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v pdfimages > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v pdftoppm > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v pdffonts > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+[ $PREPROCESS_CLEAN -eq 1 ] && ! command -v unpaper > /dev/null && echo "Please install unpaper. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v tesseract > /dev/null && echo "Please install tesseract and tesseract-data. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! python2 -c 'import lxml' 2>/dev/null && echo "Please install the python library lxml. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! python2 -c 'import sys, reportlab; (getattr(reportlab, "Version", "0.0") >= "3.0") or sys.exit(1)' 2>/dev/null \
+	&& echo "Please install the python library reportlab. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+
+! command -v gs > /dev/null && echo "Please install ghostscript. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v java > /dev/null && echo "Please install java. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+
+
+# ensure the right tesseract version is installed
+# older versions are known to produce malformed hocr output and should not be used
+# Even 3.02.01 fails in few cases (see issue #28). I decided to allow this version anyway because
+# 3.02.02 is not yet available for some widespread linux distributions
+reqtessversion="3.02.01"
+tessversion=`tesseract -v 2>&1 | grep "tesseract" | sed s/[^0-9.]//g`
+tesstooold=$(echo "`echo $tessversion | sed s/[.]//2`-`echo $reqtessversion | sed s/[.]//2` < 0" | bc)
+[ "$tesstooold" -eq "1" ] \
+	&& echo "Please install tesseract ${reqtessversion} or newer (currently installed version is ${tessversion})" && exit $EXIT_MISSING_DEPENDENCY
+
+# ensure the right GNU parallel version is installed
+# older version do not support -q flag (required to escape special characters)
+reqparallelversion="20121122"
+parallelversion=`parallel --minversion 0`
+! parallel --minversion "$reqparallelversion" > /dev/null \
+	&& echo "Please install GNU parallel ${reqparallelversion} or newer (currently installed version is ${parallelversion})" && exit $EXIT_MISSING_DEPENDENCY
+
+# ensure pdftoppm is provided by poppler-utils, not the older xpdf version
+! pdftoppm -v 2>&1 | grep -q 'Poppler' && echo "Please remove xpdf and install poppler-utils. Exiting..." && $EXIT_MISSING_DEPENDENCY
 
 
 
+# Display the version of the tools if log level is LOG_DEBUG
+if [ $VERBOSITY -ge $LOG_DEBUG ]; then
+	echo "--------------------------------"
+	echo "ImageMagick version:"
+	identify --version
+	echo "--------------------------------"
+	echo "GNU Parallel version:"
+	parallel --version
+	echo "--------------------------------"
+	echo "Poppler-utils version:"
+	pdfimages -v
+	pdftoppm -v
+	pdffonts -v
+	echo "--------------------------------"
+	echo "unpaper version:"
+	unpaper --version
+	echo "--------------------------------"
+	echo "tesseract version:"
+	tesseract --version
+	echo "--------------------------------"
+	echo "python2 version:"
+	python2 --version
+	echo "--------------------------------"
+	echo "Ghostscript version:"
+	gs --version
+	echo "--------------------------------"
+	echo "Java version:"
+	java -version
+	echo "--------------------------------"
+fi
 
-# Initialize path to temporary files
-today=$(date +"%Y%m%d_%H%M")
-fld=$(basename "$FILE_INPUT_PDF" | sed 's/[.][^.]*//')
-TMP_FLD="./tmp/$today.filename.$fld"
-FILE_TMP="$TMP_FLD/tmp.txt"						# temporary file with a very short lifetime (may be used for several things)
-FILE_SIZE_PAGES="$TMP_FLD/page-sizes.txt"				# size in pt of the respective page of the input PDF file
-FILE_OUTPUT_PDF_CAT="${TMP_FLD}/ocred.pdf"				# concatenated OCRed PDF files
-FILE_OUTPUT_PDFA_WO_META="${TMP_FLD}/ocred-pdfa-wo-metadata.pdf"	# PDFA file before appending metadata
+
+
+# check if the languages passed to tesseract are all supported
+for currentlan in `echo "$LAN" | sed 's/+/ /g'`; do
+	if ! tesseract --list-langs 2>&1 | grep "^$currentlan\$" > /dev/null; then
+		echo "The language \"$currentlan\" is not supported by tesseract."
+		tesseract --list-langs 2>&1 | tr '\n' ' '; echo
+		echo "Exiting..."
+		exit $EXIT_BAD_ARGS
+	fi
+done
+
+
+
+# Initialize path to temporary files using mktemp
+# Goal: save tmp file in a sub-folder of the $TMPDIR environment variable (or in "/tmp" if unset)
+# Unfortunately, Linux mktemp is not compatible with FreeBSD/OSX mktemp
+# Linux version requires no arg
+# FreeBSD requires '-t prefix' to be used so that $TMPDIR is taken into account
+# But in Linux '-t template' is handled differently than in FreeBSD
+# Therefore different calls must be used for Linux and for FreeBSD
+prefix="$(date +"%Y%m%d_%H%M").filename.$(basename "$FILE_INPUT_PDF" | sed 's/[.][^.]*$//')"	# prefix made of date, time and pdf file name without extension
+TMP_FLD=`mktemp -d 2>/dev/null || mktemp -d -t "${prefix}" 2>/dev/null`				# try Linux syntax first, if it fails try FreeBSD/OSX			
+if [ $? -ne 0 ]; then
+	if [ -z "$TMPDIR" ]; then
+		echo "Could not create folder for temporary files. Please ensure you have sufficient right and \"/tmp\" exists"
+	else
+		echo "Could not create folder for temporary files. Please ensure you have sufficient right and \"$TMPDIR\" exists"
+	fi
+	exit $EXIT_FILE_ACCESS_ERROR
+fi
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Created temporary folder: \"$TMP_FLD\""
+
+FILE_TMP="${TMP_FLD}/tmp.txt"						# temporary file with a very short lifetime (may be used for several things)
+FILE_PAGES_INFO="${TMP_FLD}/pages-info.txt"				# for each page: page #; width in pt; height in pt
 FILE_VALIDATION_LOG="${TMP_FLD}/pdf_validation.log"			# log file containing the results of the validation of the PDF/A file
 
-# Create tmp folder
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Creating temporary folder: \"$TMP_FLD\""
-rm -r -f "${TMP_FLD}"
-mkdir -p "${TMP_FLD}"
 
 
-
-
-# get the size of each pdf page (width / height) in pt (inch*72)
+# get the size of each pdf page (width / height) in pt (i.e. inch/72)
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Input file: Extracting size of each page (in pt)"
 ! identify -format "%w %h\n" "$FILE_INPUT_PDF" > "$FILE_TMP" \
-	&& echo "Could not get size of PDF pages. Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE
-# removing empty lines (last one should be) and prepend page # before each line
-sed '/^$/d' "$FILE_TMP" | awk '{printf "%04d %s\n", NR, $0}' > "$FILE_SIZE_PAGES"
-numpages=`tail -n 1 "$FILE_SIZE_PAGES" | cut -f1 -d" "`
+	&& echo "Could not get size of PDF pages. Exiting..." && exit $EXIT_BAD_INPUT_FILE
+# removing empty lines (last one should be) and add page # before each line
+sed '/^$/d' "$FILE_TMP" | awk '{printf "%04d %s\n", NR, $0}' > "$FILE_PAGES_INFO"
+numpages=`tail -n 1 "$FILE_PAGES_INFO" | cut -f1 -d" "`
 
-# Itterate the pages of the input pdf file
-while read pageSize ; do
+# process each page of the input pdf file
+parallel --gnu -q -k --halt-on-error 1 "$OCR_PAGE" "$FILE_INPUT_PDF" "{}" "$numpages" "$TMP_FLD" \
+	"$VERBOSITY" "$LAN" "$KEEP_TMP" "$PREPROCESS_DESKEW" "$PREPROCESS_CLEAN" "$PREPROCESS_CLEANTOPDF" "$OVERSAMPLING_DPI" \
+	"$PDF_NOIMG" "$TESS_CFG_FILES" "$FORCE_OCR" < "$FILE_PAGES_INFO"
+ret_code="$?"
+[ $ret_code -ne 0 ] && exit $ret_code 
 
-	page=`echo $pageSize | cut -f1 -d" "`
-	[ $VERBOSITY -ge $LOG_INFO ] && echo "Processing page $page / $numpages"
-	
-	# create the name of the required file
-	curOrigImg="$TMP_FLD/${page}_Image"			# original image available in the current PDF page 
-								# (the image file may have a different orientation than in the pdf file)
-	curHocr="$TMP_FLD/$page.hocr"				# hocr file to be generated by the OCR SW for the current page
-	curOCRedPDF="$TMP_FLD/${page}-ocred.pdf"		# PDF file containing the image + the OCRed text for the current page
-	curOCRedPDFDebug="$TMP_FLD/${page}-debug-ocred.pdf"	# PDF file containing data required to find out if OCR worked correctly
-	
-	# get width / height of PDF page (in pt)
-	widthPDF=`echo $pageSize | cut -f2 -d" "`
-	heightPDF=`echo $pageSize | cut -f3 -d" "`
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: size ${heightPDF}x${widthPDF} (h*w in pt)"
-	# extract raw image from pdf file to compute resolution
-	# unfortunatelly this image can have another orientation than in the pdf...
-	# so we will have to extract it again later using pdftoppm
-	pdfimages -f $page -l $page -j "$FILE_INPUT_PDF" "$curOrigImg" 1>&2	
-	# count number of extracted images
-	nbImg=`ls -1 "$curOrigImg"* | wc -l`
-	[ $nbImg -ne "1" ] && echo "Expecting exactly 1 image on page $page (found $nbImg). Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE
-	# Get characteristics of the extracted image
-	curImg=`ls -1 "$curOrigImg"*`
-	propCurImg=`identify -format "%w %h %[colorspace]" "$curImg"`
-	widthCurImg=`echo "$propCurImg" | cut -f1 -d" "`
-	heightCurImg=`echo "$propCurImg" | cut -f2 -d" "`
-	colorspaceCurImg=`echo "$propCurImg" | cut -f3 -d" "`
-	# switch height/width values if the image has not the right orientation
-	# we make here the assumption that vertical/horizontal dpi are equal
-	# we will check that later
-	if [ $((($heightPDF-$widthPDF)*($heightCurImg-$widthCurImg))) -lt 0 ]; then
-		[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracted image has wrong orientation. Inverting image height/width values"
-		tmpval=$heightCurImg
-		heightCurImg=$widthCurImg
-		widthCurImg=$tmpval
-	fi
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: size ${heightCurImg}x${widthCurImg} (h*w pixel)"	
-	# compute the resolution of the image
-	dpi_x=`echo "scale=5;$widthCurImg*72/$widthPDF" | bc`
-	dpi_y=`echo "scale=5;$heightCurImg*72/$heightPDF" | bc`
-	# compute the maximum allowed resolution difference that can be cause by:
-	# - the truncated PDF with/height in pt
-	# - the precision of dpi value
-	epsilon=`echo "scale=5;($widthCurImg*72/$widthPDF^2)+($heightCurImg*72/$heightPDF^2)+0.00002" | bc`	# max inaccuracy due to truncation of PDF size in pt
-	[ `echo "($dpi_x - $dpi_y) < $epsilon " | bc` -eq 0 -o `echo "($dpi_y - $dpi_x) < $epsilon " | bc` -eq 0 ] \
-		&& echo "Resolutions difference ($dpi_x/$dpi_y) higher than expected ($epsilon). Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE
-	dpi=`echo "scale=5;($dpi_x+$dpi_y)/2+0.5" | bc` # adding 0.5 is required for rounding
-	dpi=`echo "scale=0;$dpi/1" | bc`		# round to the nearest integer
-	
-	# Identify if page image should be saved as ppm (color) or pgm (gray)
-	ext="ppm"
-	opt=""		
-	if [ $colorspaceCurImg == "Gray" ]; then
-		ext="pgm"
-		opt="-gray"
-	fi
-	curImgPixmap="$TMP_FLD/$page.$ext"
-	curImgPixmapDeskewed="$TMP_FLD/$page.deskewed.$ext"
-	curImgPixmapClean="$TMP_FLD/$page.cleaned.$ext"
-	
-	# extract current page as image with right orientation and resoltution
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracting image as $ext file (${dpi} dpi)"
-	! pdftoppm -f $page -l $page -r $dpi $opt "$FILE_INPUT_PDF" > "$curImgPixmap" \
-		&& echo "Could not extract page $page as $ext from \"$FILE_INPUT_PDF\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
-
-	# if requested deskew image (without changing its size in pixel)
-	if [ "$PREPROCESS_DESKEW" -eq "1" ]; then
-		[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Deskewing image"
-		! convert "$curImgPixmap" -deskew 40% -gravity center -extent ${widthCurImg}x${heightCurImg} "$curImgPixmapDeskewed" \
-			&& echo "Could not deskew \"$curImgPixmap\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
-	else
-		cp "$curImgPixmap" "$curImgPixmapDeskewed"
-	fi
-
-	# if requested clean image with unpaper to get better OCR results
-	if [ "$PREPROCESS_CLEAN" -eq "1" ]; then
-		[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Cleaning image with unpaper"
-		! unpaper --dpi $dpi --mask-scan-size 100 \
-			--no-deskew --no-grayfilter --no-blackfilter --no-mask-center --no-border-align \
-			"$curImgPixmapDeskewed" "$curImgPixmapClean" 1> /dev/null \
-			&& echo "Could not clean \"$curImgPixmapDeskewed\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
-	else
-		cp "$curImgPixmapDeskewed" "$curImgPixmapClean"
-	fi
-
-	# perform OCR
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Performing OCR"
-	! tesseract -l "$LAN" "$curImgPixmapClean" "$curHocr" hocr $TESS_CFG_FILES 1> /dev/null 2> /dev/null \
-		&& echo "Could not OCR file \"$curImgPixmapClean\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
-	mv "$curHocr.html" "$curHocr"
-
-	# embed text and image to new pdf file
-	if [ "$PREPROCESS_CLEANTOPDF" -eq "1" ]; then
-		image4finalPDF="$curImgPixmapClean"
-	else
-		image4finalPDF="$curImgPixmapDeskewed"	
-	fi
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF"
-	! python $SRC/hocrTransform.py -r $dpi -i "$image4finalPDF" "$curHocr" "$curOCRedPDF" \
-		&& echo "Could not create PDF file from \"$curHocr\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
-	
-	# if requested generate special debug PDF page with visible OCR text
-	if [ $PDF_NOIMG -eq "1" ] ; then
-		[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF (debug page)"
-		! python $SRC/hocrTransform.py -b -r $dpi "$curHocr" "$curOCRedPDFDebug" \
-			&& echo "Could not create PDF file from \"$curHocr\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR	
-	fi
-	
-	# delete temporary files created for the current page
-	# to avoid using to much disk space in case of PDF files having many pages
-	if [ $KEEP_TMP -eq 0 ]; then
-		rm "$curOrigImg"*.*
-		rm "$curHocr"
-		rm "$curImgPixmap"
-		rm "$curImgPixmapDeskewed"
-		rm "$curImgPixmapClean"
-	fi
-	
-done < "$FILE_SIZE_PAGES"
-
-
-
-
-# concatenate all pages
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Concatenating all pages"
-! pdftk "${TMP_FLD}/"*-ocred.pdf cat output "$FILE_OUTPUT_PDF_CAT" \
-	&& echo "Could not concatenate individual PDF pages (\"${TMP_FLD}/*-ocred.pdf\") to one file. Exiting..." >&2 && exit $EXIT_OTHER_ERROR
-
-# convert the pdf file to match PDF/A format
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Converting to PDF/A" 
+# concatenate all pages and convert the pdf file to match PDF/A format
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Concatenating all pages to the final PDF/A file" 
 ! gs -dQUIET -dPDFA -dBATCH -dNOPAUSE -dUseCIEColor \
 	-sProcessColorModel=DeviceCMYK -sDEVICE=pdfwrite -sPDFACompatibilityPolicy=2 \
-	-sOutputFile="$FILE_OUTPUT_PDFA" "$FILE_OUTPUT_PDF_CAT" 1> /dev/null 2> /dev/null \
-	&& echo "Could not convert PDF file \"$FILE_OUTPUT_PDF_CAT\" to PDF/A. Exiting..." >&2 && exit $EXIT_OTHER_ERROR
-
-# # Write metadata
-# # Needs to be done after converting to PDF/A, as gs does not preserve metadata
-# [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Update metadata (creator, producer, and title)" 
-# title=`basename "$FILE_INPUT_PDF" | sed 's/[.][^.]*//' | \
-	# sed 's/_/ /g' | sed 's/-/ /g' | \
-	# sed 's/\([[:lower:]]\)\([[:upper:]]\)/\1 \2/g' | \
-	# sed 's/\([[:alpha:]]\)\([[:digit:]]\)/\1 \2/g' | \
-	# sed 's/\([[:digit:]]\)\([[:alpha:]]\)/\1 \2/g'`	# transform the file name (with extension) into distinct words
-# pdftk "$FILE_OUTPUT_PDFA_WO_META" update_info_utf8 - output "$FILE_OUTPUT_PDFA" << EOF
-# InfoBegin
-# InfoKey: Title
-# InfoValue: $title
-# InfoBegin
-# InfoKey: Creator
-# InfoValue: $TOOLNAME $VERSION
-# InfoBegin
-# InfoKey: Producer
-# InfoValue: ghostcript `gs --version`, pdftk
-# EOF
+	-sOutputFile="$FILE_OUTPUT_PDFA" "${TMP_FLD}/"*ocred*.pdf 1> /dev/null 2> /dev/null \
+	&& echo "Could not concatenate all pages to the final PDF/A file. Exiting..." && exit $EXIT_OTHER_ERROR
 
 # validate generated pdf file (compliance to PDF/A)
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Checking compliance to PDF/A standard" 
-java -jar "$JHOVE" -c "$JHOVE_CFG" -m PDF-hul "$FILE_OUTPUT_PDFA" > "$FILE_VALIDATION_LOG"
+! java -jar "$JHOVE" -c "$JHOVE_CFG" -m PDF-hul "$FILE_OUTPUT_PDFA" 2> /dev/null 1> "$FILE_VALIDATION_LOG" \
+	&& echo "Unexpected error while checking compliance to PDF/A file. Exiting..." && exit $EXIT_OTHER_ERROR
 grep -i "Status|Message" "$FILE_VALIDATION_LOG" # summary of the validation
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "The full validation log is available here: \"$FILE_VALIDATION_LOG\""
 # check the validation results
 pdf_valid=1
-grep -i 'ErrorMessage' "$FILE_VALIDATION_LOG" >&2 && pdf_valid=0
-grep -i 'Status.*not valid' "$FILE_VALIDATION_LOG" >&2 && pdf_valid=0
-grep -i 'Status.*Not well-formed' "$FILE_VALIDATION_LOG" >&2 && pdf_valid=0
-! grep -i 'Profile:.*PDF/A-1' "$FILE_VALIDATION_LOG" > /dev/null && echo "PDF file profile is not PDF/A-1" >&2 && pdf_valid=0
-[ $pdf_valid -ne 1 ] && echo "Output file: The generated PDF/A file is INVALID" >&2
-[ $pdf_valid -ne 0 ] && [ $VERBOSITY -ge $LOG_INFO ] && echo "Output file: The generated PDF/A file is VALID"
+grep -i 'ErrorMessage' "$FILE_VALIDATION_LOG" && pdf_valid=0
+grep -i 'Status.*not valid' "$FILE_VALIDATION_LOG" && pdf_valid=0
+grep -i 'Status.*Not well-formed' "$FILE_VALIDATION_LOG" && pdf_valid=0
+! grep -i 'Profile:.*PDF/A-1' "$FILE_VALIDATION_LOG" > /dev/null && echo "PDF file profile is not PDF/A-1" && pdf_valid=0
+[ $pdf_valid -ne 1 ] && echo "Output file: The generated PDF/A file is INVALID"
+[ $pdf_valid -eq 1 ] && [ $VERBOSITY -ge $LOG_INFO ] && echo "Output file: The generated PDF/A file is VALID"
 
 
 
@@ -360,4 +298,4 @@ END=`date +%s`
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Script took $(($END-$START)) seconds"
 
 
-[ $pdf_valid -ne 1 ] && exit $EXIT_INVALID_OUPUT_PDFA || exit 0
+[ $pdf_valid -ne 1 ] && exit $EXIT_INVALID_OUTPUT_PDFA || exit 0

README.md

@@ -3,17 +3,21 @@ OCRmyPDF
 
 OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched
 
-To get the script usage, call: ./OCmyPDF.sh -h
+To get the script usage, call: sh ./OCRmyPDF.sh -h
 
-Features
+Main features
 --------
 
-- Generate a searchable PDF/A file from a PDF file containing only images
-- Place OCRed text accurately below the image to easy copy / paste
-- Keep the exact resolution of the original embedded images
-- If requested deskew and / or clean the image before performing OCR
-- Validate the generated file against the PDF/A specification using jhove
+- Generates a searchable PDF/A file from a PDF file containing only images
+- Places OCRed text accurately below the image to ease copy / paste
+- Keeps the exact resolution of the original embedded images
+    - or if requested oversamples the images before OCRing so as to get better results 
+- If requested deskews and / or clean the image before performing OCR
+- Validates the generated file against the PDF/A specification using jhove
 - Provides debug mode to enable easy verification of the OCR results
+- Processes several pages in parallel if more than one CPU core is available
+
+For details: please consult the release notes
 
 Motivation
 ----------
@@ -31,12 +35,35 @@ I found many, but none of them were really satisfying.
 ... so I decided to develop my own tool (using various existing scripts as an inspiration)
 
 Install
---------
+-------
 
-Download OCRmyPDF here: https://github.com/fritz-hh/OCRmyPDF/tags
+Download OCRmyPDF here: https://github.com/fritz-hh/OCRmyPDF/releases
 
 Copy the file in onto your linux/unix machine and extract it.
 
 Run: "sh ./OCRmyPDF.sh -h" to get the script usage
 
-If not yet installed, the script will notify you about dependencies that need to be installed
+If not yet installed, the script will notify you about dependencies that need to be installed.
+The script requires specific versions of the dependencies. Older version than the ones mentioned in the release notes are likely not to be compatible to OCRmyPDF.
+
+Support
+-------
+
+In case you detect an issue, please:
+
+- Check if your issue is already known
+- If no problem report exists on github, please create one here: https://github.com/fritz-hh/OCRmyPDF/issues
+- Describe your problem thoroughly
+- Append the console output of the script when running the debug mode (-g option)
+- If possible provide your input PDF file as well as the content of the temporary folder (using a file sharing service like www.file-upload.net)
+
+Press & Media
+-------------
+
+- c't 1-2014, page 59: Detailed presentation of OCRmyPDF v1.0 in the leading german IT magazine c't (http://www.heise.de/ct/inhalt/2014/1/58/) 
+- heise Open Source, 09/2014: Texterkennung mit OCRmyPDF (http://www.heise.de/-2356670)
+
+Disclaimer
+----------
+
+The software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

RELEASE_NOTES.md

@@ -5,6 +5,230 @@ Please always read this file before installing the package
 
 Download software here: https://github.com/fritz-hh/OCRmyPDF/tags
 
+v2.2-stable (2014-09-29):
+=======
+
+New features
+------------
+
+- None
+
+Changes
+-------
+
+- Update to jhove v1.11
+- Request the python library reportlab v3.0 or newer (So that we could remove a patch to the previous version of reportlab leading to issues for some users)
+
+Fixes
+-----
+
+- Fix bug on Mac OS X (resolution of simlink to OCRmyPDF.sh script) (thanks to jbarlow83)
+- Check if the input pdf file exists before to continue
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.2
+- Dependencies:
+   - parallel 20140822
+   - poppler-utils 0.24.5
+   - ImageMagick 6.8.9-4 2014-09-17
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.8
+   - ghostcript (gs): 9.06
+   - java: openjdk version "1.7.0_65"
+
+v2.1-stable (2014-09-20):
+=======
+
+New features
+------------
+
+- None
+
+Changes
+-------
+
+- None
+
+Fixes
+-----
+
+- Allow execution via simlink
+- Add support for tesseract 3.03
+- Add support for newer version of reportlab
+- Lowered minimum version of gnu parallel
+- Various typo
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.1
+- Dependencies:
+   - parallel 20130222
+   - poppler-utils 0.22.2
+   - ImageMagick 6.8.0-7 2013-03-30
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.3
+   - ghostcript (gs): 9.06
+   - java: openjdk version "1.7.0_17"
+
+v2.0-stable (2014-01-25):
+=======
+
+New features
+------------
+
+- Check if the language(s) passed using the -l option is supported by tesseract (fixes #60)
+
+Changes
+-------
+
+- Allow OCRmyPDF to be used with tesseract 3.02.01, even though OCR might fail for few PDF file (see issue #28). Rationale: For some linux distribution, no newer version than tesseract 3.02.01 is available
+
+Fixes
+-----
+
+- More robust algorithm for checking the version of the installed tesseract package
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.1
+- Dependencies:
+   - parallel 20130222
+   - poppler-utils 0.22.2
+   - ImageMagick 6.8.0-7 2013-03-30
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.3
+   - ghostcript (gs): 9.06
+   - java: openjdk version "1.7.0_17"
+
+v2.0-rc2 (2014-01-16):
+=======
+   
+New features
+------------
+
+- None
+
+Changes
+-------
+
+- Size reduction of final PDF file: (fixes #50)
+   - Support for monochrome (Black&White) images (massive size reduction in final PDF: >80%)
+   - Reduced size of grayscale images (by 13% on test PDF file)
+- Preventing fi, fl ligatures does not require anymore to pass an additional config file to tesseract using the -C option (fixes #58)
+- Location of temporary folder according to content of environment variable TMPDIR.
+- Dependency to pdftk removed
+- Check for compatible versions of dependencies: (fixes #51)
+   - parallel and tesseract
+   - python libraries reportlab and lxml
+
+Fixes
+-----
+
+- Improved portability with various shells (dash, bash, tcsh) and OS (FreeBSD, MAC OSX, Linux) (fixes #59)
+- Corrected bug in case the input PDF file contains a space character (fixes #48)
+- Prevent spurious error message in case there is no image in a PDF page
+- Prevent collision of temporary folder names (fixes #57)
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.1
+- Dependencies:
+   - parallel 20130222
+   - poppler-utils 0.22.2
+   - ImageMagick 6.8.0-7 2013-03-30
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.3
+   - ghostcript (gs): 9.06
+   - java: openjdk version "1.7.0_17"
+
+v2.0-rc1 (2014-01-07):
+====
+
+New features
+------------
+
+- Huge performance improvement on machines having multiple CPU/cores (processing of several pages concurrently) (fixes #18)
+- By default prevent from processing a PDF file already containing fonts (i.e. text)(it can be overridden with the -f flag) (fixes #16)
+- Warn if the resolution is too low to get reasonable OCR results (fixes #37)
+- New option (-o) to perform automatic oversampling if the image resolution is too low. This can improve OCR results.
+- Warn if using a tesseract version older than v3.02.02 (as older versions are known to produce invalid output) (fixes #41)
+- Echo version of the installed dependencies (e.g. tesseract) in debug mode in order to ease support (fixes #35)
+- Echo the arguments passed to the script in debug mode to ease support
+
+Changes
+-------
+
+- In debug mode: The debug page is now placed after the respective "normal" page
+- Reduced disk space usage in temporary folder if -d (deskew) or -c (cleanup) options are not selected
+- New file src/config.sh containing various configuration parameters
+- Documentation of the tesseract config file "tess-cfg/no_ligature" improved 
+- Improved consistency of the temporary file names
+
+Fixes
+-----
+
+- Improved robustness:
+   - in case vertical resolution differs from horizontal resolution (fixes #38)
+   - in case a PDF page contains more than one image (fixes #36)
+- Fix a problem occurring if python 3 is the standard interpreter (fixes #33)
+- Fix a problem occurring if the input PDF file contains special characters like "#" (fixes #34)
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.1
+- Dependencies:
+   - parallel 20130222
+   - poppler-utils 0.22.2
+   - ImageMagick 6.8.0-7 2013-03-30
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.3
+   - pdftk 1.45
+   - ghostcript (gs): 9.06
+   - java: openjdk version "1.7.0_17"
+
+v1.1-stable (2014-01-06):
+====
+
+New features
+------------
+
+- N/A
+
+Changes
+-------
+
+- N/A
+
+Fixes
+-----
+
+- Fixed syntax error (bashism) leading to an error message on certain systems (fixes #42)
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.1
+- Dependencies:
+   - poppler-utils 0.22.2
+   - ImageMagick 6.8.0-7 2013-03-30
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.3
+   - pdftk 1.45
+   - ghostcript (gs): 9.06
+   - java: openjdk version "1.7.0_17"
+
 v1.0-stable (2013-05-06):
 ====
 
@@ -39,7 +263,7 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"
 
 v1.0-rc2 (2013-04-29):
@@ -75,7 +299,7 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"
 
 v1.0-rc1 (2013-04-26):
@@ -107,5 +331,5 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"

jhove/README

@@ -3,7 +3,7 @@ Copyright 2003-2012 by JSTOR and the President and Fellows of Harvard College
 JHOVE is made available under the GNU Lesser General Public License (LGPL;
 see the file LICENSE for details)
 
-Rev. 1.9, 2012-12-17
+Rev. 1.11, 2013-09-29
 
 JHOVE (the JSTOR/Harvard Object Validation Environment, pronounced "jhove")
 is an extensible software framework for performing format identification,

jhove/RELEASENOTES

@@ -6,6 +6,85 @@ see the file LICENSE for details)
 
 Versions 1.7 and beyond of JHOVE are no longer under the control of Harvard.
 
+RELEASE NOTES FOR JHOVE 1.11
+
+GENERAL
+
+1. I've added lots of logging code. Calls at the FINE level and lower 
+   don't show up no matter what I do, so I've put them at the INFO level.
+   The level is set in JhoveBase.java.
+
+2. All .bat and _bat.tmpl files now have CR-LF line endings. That is, they
+   do in the gzip and zip archives you download. I'm not sure how
+   SourceForge will treat files that you download individually,
+   but hopefully it will have the sense to keep CR-LF when downloading
+   to a Windows system. 
+
+3. All .bat files now assume JHOVE_HOME is the directory from which they're
+   run. They no longer try to set JAVA_HOME (which was still stuck in
+   Java 1.4 and probably wasn't working for many people), instead assuming
+   that the JAVA command is available on the command line.
+
+4. All javac commands in build.xml files now specify source=1.5 for
+   compatibility with more recent compilers.
+  
+5. gdumpwin.bat is deleted. It's redundant with gdump.bat and has bugs
+   of its own.
+   
+PDF MODULE
+
+1. Fix to PDF module, submitted by willp-bl, may reduce tendency 
+   to run out of heap space on some files.
+
+RELEASE NOTES FOR JHOVE 1.10
+
+GENERAL
+
+1. The amount of logging code has been increased, mostly at the
+   DEBUG level.
+   
+2. Further work on generics in Java code.
+
+3. JhoveView now checks for Java 1.5. Was previously allowing 1.4 even
+   though it wouldn't work.
+
+HTML MODULE
+
+1. XHTML files are processed by the HTML module, which invokes the XML
+   modules. In this case, the XML module doesn't have the parameters
+   specified in the JHOVE configuration file and so won't use local
+   copies of schemas. Starting with this version, the parameters of
+   the HTML module are passed to the XML module when invoking it.
+   However, this doesn't work properly (in either module) for a DTD
+   that invokes additional DTDs by relative URLs. Such DTDs should
+   be edited to use only absolute URLs.
+   
+PDF MODULE
+
+1. Failure to get a page object number wasn't being handled cleanly,
+   resulting in a report of an invalid document without an error message
+   to explain it (SourceForge bug 49). This has been fixed.
+
+2. The PDF module unnecessarily uses huge amounts of memory to build
+   complex structure trees, when it doesn't need to keep the whole 
+   tree in memory to validate it. In the new version, it uses memory
+   more economically. This should result in the successful processing
+   of some PDF files that ran out of memory or took hours to process before.
+   
+3. If an annotation isn't a dictionary object, report that explicitly.
+   This happens with some otherwise good files; I can't find any warrant
+   for it in the PDF spec.
+
+4. Some efficiency improvements to PDF parser. Increased buffer size from 4K 
+   to 64K. Made Parser.collapseObjectVector more efficient. Parser now
+   returns pseudo-objects for array and dictionary end instead of throwing
+   an exception.
+
+5. Minor cleanup of error reporting.
+
+6. If an object uses a compression scheme which JHOVE can't deal with, JHOVE
+   will try to give a specific error message.
+
 RELEASE NOTES FOR JHOVE 1.9
 
 GENERAL

jhove/conf/jhove-byteoffset=true.conf

@@ -3,7 +3,7 @@
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
  xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.4/jhoveConfig.xsd">
+                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
  <jhoveHome>/users/stephen/projects/jhove</jhoveHome>
  <defaultEncoding>utf-8</defaultEncoding>
  <tempDirectory>/var/tmp</tempDirectory>

jhove/conf/jhove-withTextMD.conf

@@ -3,7 +3,7 @@
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
  xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.4/jhoveConfig.xsd">
+                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
  <jhoveHome>/users/stephen/projects/jhove</jhoveHome>
  <defaultEncoding>utf-8</defaultEncoding>
  <tempDirectory>/var/tmp</tempDirectory>

jhove/conf/jhove.conf

@@ -3,7 +3,7 @@
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
  xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.4/jhoveConfig.xsd">
+                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
  <jhoveHome>./jhove/</jhoveHome>
  <defaultEncoding>utf-8</defaultEncoding>
  <tempDirectory>/var/tmp</tempDirectory>

jhove/jhove_bat.tmpl

@@ -35,9 +35,6 @@ REM EXTRA_JARS Extra jar files to add to CLASSPATH
 REM SET JHOVE_HOME="C:\Program Files\jhove"
 SET JHOVE_HOME="[your directory path]\jhove"
 
-SET JAVA_HOME="C:\Program Files\java\j2re1.4.1_02"
-SET JAVA=%JAVA_HOME%\bin\java
-
 SET EXTRA_JARS=
 
 REM NOTE: Nothing below this line should be edited
@@ -60,4 +57,4 @@ IF "%1"=="" GOTO LOOP
 :LOOP
 
 REM Set the CLASSPATH and invoke the Java loader
-%JAVA% -classpath %CP% Jhove %ARGS%
+JAVA -classpath %CP% Jhove %ARGS%

src/config.sh

@@ -0,0 +1,32 @@
+#####################################################################################
+# The following parameters might be changed by the user
+#####################################################################################
+
+DEFAULT_DPI=300				# dpi value used as fall back if the page dpi cannot be determined
+
+#####################################################################################
+# Do NOT change the following parameters
+#####################################################################################
+
+TOOLNAME="OCRmyPDF"
+VERSION="v2.2-stable"
+
+# possible exit codes
+EXIT_BAD_ARGS="1"
+EXIT_BAD_INPUT_FILE="2"
+EXIT_MISSING_DEPENDENCY="3"
+EXIT_INVALID_OUTPUT_PDFA="4"
+EXIT_FILE_ACCESS_ERROR="5"
+EXIT_OTHER_ERROR="15"
+
+# possible log levels
+LOG_ERR="0"				# only error messages
+LOG_WARN="1"				# error messages and warnings
+LOG_INFO="2"				# error messages, warnings and some infos
+LOG_DEBUG="3"				# debug level logging
+
+# various paths
+SRC="./src"				# location of the source folder (except source of external tools like jhove)
+OCR_PAGE="$SRC/ocrPage.sh"		# path to the script aimed at OCRing one page
+JHOVE="./jhove/bin/JhoveApp.jar"	# java SW for validating the final PDF/A
+JHOVE_CFG="./jhove/conf/jhove.conf"	# location of the jhove config file

src/hocrTransform.py

@@ -1,5 +1,7 @@
+#!/usr/local/bin/python2
+# coding: utf-8
 ##############################################################################
-# Copyright (c) 2013: fritz-hh from Github (https://github.com/fritz-hh)
+# Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
 #
 # Copyright (c) 2010: Jonathan Brinley from Github (https://github.com/jbrinley/HocrConverter)
 # Initial version by Jonathan Brinley, jonathanbrinley@gmail.com
@@ -7,9 +9,11 @@
 from reportlab.pdfgen.canvas import Canvas
 from reportlab.lib.units import inch
 from lxml import etree as ElementTree
-import Image, re, sys
+from PIL import Image
+import re, sys
 import argparse
 
+
 class hocrTransform():
 	"""
 	A class for converting documents from the hOCR format.
@@ -22,23 +26,24 @@ class hocrTransform():
 
 		self.hocr = ElementTree.ElementTree()
 		self.hocr.parse(hocrFileName)
- 
+
 		# if the hOCR file has a namespace, ElementTree requires its use to find elements
 		matches = re.match('({.*})html', self.hocr.getroot().tag)
 		self.xmlns = ''
 		if matches:
 			self.xmlns = matches.group(1)
-			
+
 		# get dimension in pt (not pixel!!!!) of the OCRed image
+		self.width, self.height = None, None
 		for div in self.hocr.findall(".//%sdiv[@class='ocr_page']"%(self.xmlns)):
 			coords = self.element_coordinates(div)
 			self.width = self.px2pt(coords[2]-coords[0])
 			self.height = self.px2pt(coords[3]-coords[1])
 			break # there shouldn't be more than one, and if there is, we don't want it
-			
+
 		# no width and heigh definition in the ocr_image element of the hocr file
 		if self.width is None:
-			print "No page dimension found in the hocr file"
+			print("No page dimension found in the hocr file")
 			sys.exit(1)
 
 	def __str__(self):
@@ -52,7 +57,7 @@ class hocrTransform():
 			return self._get_element_text(body).encode('utf-8') # XML gives unicode
 		else:
 			return ''
-  
+
 	def _get_element_text(self, element):
 		"""
 		Return the textual content of the element and its children
@@ -65,7 +70,7 @@ class hocrTransform():
 		if element.tail is not None:
 			text = text + element.tail
 		return text
-    
+
 	def element_coordinates(self, element):
 		"""
 		Returns a tuple containing the coordinates of the bounding box around
@@ -78,13 +83,24 @@ class hocrTransform():
 				coords = matches.group(1).split()
 				out = (int(coords[0]),int(coords[1]),int(coords[2]),int(coords[3]))
 		return out
-    
+
 	def px2pt(self, pxl):
 		"""
 		Returns the length in pt given length in pxl
 		"""
 		return float(pxl)/self.dpi*inch
-			
+
+	def replace_unsupported_chars(self, str):
+		"""
+		Given an input string, returns the corresponding string that:
+		- is available in the helvetica facetype
+		- does not contain any ligature (to allow easy search in the PDF file)
+		"""		
+		# The 'u' before the character to replace indicates that it is a unicode character
+		str=str.replace(u"fl","fl")
+		str=str.replace(u"fi","fi")
+		return str
+		
 	def to_pdf(self, outFileName, imageFileName, showBoundingboxes, fontname="Helvetica"):
 		"""
 		Creates a PDF file with an image superimposed on top of the text.
@@ -95,13 +111,13 @@ class hocrTransform():
 		"""
 		# create the PDF file
 		pdf = Canvas(outFileName, pagesize=(self.width, self.height), pageCompression=1) # page size in points (1/72 in.)
-		
+
 		# draw bounding box for each paragraph
 		pdf.setStrokeColorRGB(0,1,1)	# light blue for bounding box of paragraph
 		pdf.setFillColorRGB(0,1,1)	# light blue for bounding box of paragraph
 		pdf.setLineWidth(0)		# no line for bounding box
 		for elem in self.hocr.findall(".//%sp[@class='%s']" % (self.xmlns, "ocr_par")):
-		
+
 			elemtxt=self._get_element_text(elem).rstrip()
 			if len(elemtxt) == 0:
 				continue
@@ -111,12 +127,12 @@ class hocrTransform():
 			y1=self.px2pt(coords[1])
 			x2=self.px2pt(coords[2])
 			y2=self.px2pt(coords[3])
-			
+
 			# draw the bbox border
 			if showBoundingboxes == True:
-				pdf.rect(x1, self.height-y2, x2-x1, y2-y1, fill=1)		
-		
-		
+				pdf.rect(x1, self.height-y2, x2-x1, y2-y1, fill=1)
+
+
 		# check if element with class 'ocrx_word' are available
 		# otherwise use 'ocr_line' as fallback
 		elemclass="ocr_line"
@@ -131,6 +147,9 @@ class hocrTransform():
 		for elem in self.hocr.findall(".//%sspan[@class='%s']" % (self.xmlns, elemclass)):
 
 			elemtxt=self._get_element_text(elem).rstrip()
+			
+			elemtxt=self.replace_unsupported_chars(elemtxt)
+			
 			if len(elemtxt) == 0:
 				continue
 
@@ -139,7 +158,7 @@ class hocrTransform():
 			y1=self.px2pt(coords[1])
 			x2=self.px2pt(coords[2])
 			y2=self.px2pt(coords[3])
-			
+
 			# draw the bbox border
 			if showBoundingboxes == True:
 				pdf.rect(x1, self.height-y2, x2-x1, y2-y1, fill=0)
@@ -150,7 +169,7 @@ class hocrTransform():
 
 			# set cursor to bottom left corner of bbox (adjust for dpi)
 			text.setTextOrigin(x1, self.height-y2)
-       
+
 			# scale the width of the text to fill the width of the bbox
 			text.setHorizScale(100*(x2-x1)/pdf.stringWidth(elemtxt, fontname, fontsize))
 
@@ -160,13 +179,14 @@ class hocrTransform():
 
 		# put the image on the page, scaled to fill the page
 		if imageFileName != None:
-			im = Image.open(imageFileName)		
+			im = Image.open(imageFileName)
 			pdf.drawInlineImage(im, 0, 0, width=self.width, height=self.height)
-			
+
 		# finish up the page and save it
 		pdf.showPage()
 		pdf.save()
-  
+
+
 if __name__ == "__main__":
 	parser = argparse.ArgumentParser(description='Convert hocr file to PDF')
 	parser.add_argument('-b', '--boundingboxes', action="store_true", default=False, help='Show bounding boxes borders')
@@ -179,5 +199,5 @@ if __name__ == "__main__":
 	hocr = hocrTransform(args.hocrfile, args.resolution)
 	hocr.to_pdf(args.outputfile, args.image, args.boundingboxes)
 
-	
-	
+
+

src/ocrPage.sh

@@ -0,0 +1,227 @@
+#!/bin/sh
+##############################################################################
+# Script aimed at OCRing a single page of a PDF file
+#
+# Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
+##############################################################################
+
+. "./src/config.sh"
+
+
+# Initialization of variables passed by arguments
+FILE_INPUT_PDF="$1"			# PDF file containing the page to be OCRed
+PAGE_INFO="$2"				# Various characteristics of the page to be OCRed
+NUM_PAGES="$3"				# Total number of page of the PDF file (required for logging)
+TMP_FLD="$4"				# Folder where the temporary files should be placed
+VERBOSITY="$5"				# Requested verbosity
+LAN="$6"				# Language of the file to be OCRed
+KEEP_TMP="$7"				# Keep the temporary files after processing (helpful for debugging)
+PREPROCESS_DESKEW="$8"			# Deskew the page to be OCRed
+PREPROCESS_CLEAN="$9"			# Clean the page to be OCRed
+PREPROCESS_CLEANTOPDF="${10}"		# Put the cleaned paged in the OCRed PDF
+OVERSAMPLING_DPI="${11}"		# Oversampling resolution in dpi
+PDF_NOIMG="${12}"			# Request to generate also a PDF page containing only the OCRed text but no image (helpful for debugging) 
+TESS_CFG_FILES="${13}"			# Specific configuration files to be used by Tesseract during OCRing
+FORCE_OCR="${14}"			# Force to OCR, even if the page already contains fonts
+
+
+
+################################## 
+# Detect the characteristics of the embedded image for 
+# the page number provided as parameter
+#
+# Param 1: page number
+# Param 2: PDF page width in pt
+# Param 3: PDF page height in pt
+# Param 4: temporary file path (Path of the file in which the output should be written)
+# Output:  A file containing the characteristics of the embedded image. File structure:
+#          DPI=<dpi>
+#          COLOR_SPACE=<colorspace>
+#          DEPTH=<colordepth>
+# Returns:
+#       - 0: if no error occurs
+#       - 1: in case the page already contains fonts (which should be the case for PDF generated from scanned pages) 
+#       - 2: in case the page contains more than one image
+##################################
+getImgInfo() {
+	local page widthPDF heightPDF curImgInfo nbImg curImg propCurImg widthCurImg heightCurImg colorspaceCurImg depthCurImg dpi
+
+	# page number
+	page="$1"
+	# width / height of PDF page (in pt)
+	widthPDF="$2"
+	heightPDF="$3"
+	# path of the file in which the output should be written
+	curImgInfo="$4"
+
+	
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Size ${heightPDF}x${widthPDF} (h*w in pt)"
+	
+	
+	# check if the page already contains fonts (which should not be the case for PDF based on scanned files
+	[ `pdffonts -f $page -l $page "${FILE_INPUT_PDF}" | wc -l` -gt 2 ] && echo "Page $page: Page already contains font data !!!" && return 1
+
+	
+	# extract raw image from pdf file to compute resolution
+	# unfortunately this image can have another orientation than in the pdf...
+	# so we will have to extract it again later using pdftoppm
+	pdfimages -f $page -l $page -j "$FILE_INPUT_PDF" "$curOrigImg" 1>&2	
+	# count number of extracted images 
+	nbImg=$((`ls -1 "$curOrigImg"* 2>/dev/null | wc -l`))
+	if [ $nbImg -ne "1" ]; then
+		[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Expecting exactly 1 image covering the whole page (found $nbImg). Cannot compute dpi value."
+		return 2
+	fi
+	# Get characteristics of the extracted image
+	curImg=`ls -1 "$curOrigImg"* 2>/dev/null`
+	propCurImg=`identify -format "%w %h %[colorspace] %[depth]" "$curImg"`
+	widthCurImg=`echo "$propCurImg" | cut -f1 -d" "`
+	heightCurImg=`echo "$propCurImg" | cut -f2 -d" "`
+	colorspaceCurImg=`echo "$propCurImg" | cut -f3 -d" "`
+	depthCurImg=`echo "$propCurImg" | cut -f4 -d" "`
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Size ${heightCurImg}x${widthCurImg} (in pixel)"	
+
+	# compute the resolution of the image (making the assumption that x & y resolution are equal)
+	# and round it to the nearest integer
+	dpi=`echo "scale=5;sqrt($widthCurImg*72*$heightCurImg*72/$widthPDF/$heightPDF)+0.5" | bc`
+	dpi=`echo "scale=0;$dpi/1" | bc`
+	
+	# save the image characteristics
+	echo "DPI=$dpi" > "$curImgInfo"
+	echo "COLOR_SPACE=$colorspaceCurImg" >> "$curImgInfo"
+	echo "DEPTH=$depthCurImg" >> "$curImgInfo"
+	
+	return 0
+}
+
+
+page=`echo $PAGE_INFO | cut -f1 -d" "`
+[ $VERBOSITY -ge $LOG_INFO ] && echo "Processing page $page / $NUM_PAGES"
+
+# get width / height of PDF page (in pt)
+widthPDF=`echo $PAGE_INFO | cut -f2 -d" "`
+heightPDF=`echo $PAGE_INFO | cut -f3 -d" "`
+
+# create the name of the required temporary files
+curOrigImg="$TMP_FLD/${page}.orig-img"				# original image available in the current PDF page 
+								# (the image file may have a different orientation than in the pdf file)
+curHocr="$TMP_FLD/${page}.hocr"					# hocr file to be generated by the OCR SW for the current page
+curOCRedPDF="$TMP_FLD/${page}.ocred.pdf"			# PDF file containing the image + the OCRed text for the current page
+curOCRedPDFDebug="$TMP_FLD/${page}.ocred.todebug.pdf"		# PDF file containing data required to find out if OCR worked correctly
+curImgInfo="$TMP_FLD/${page}.orig-img-info.txt"			# Detected characteristics of the embedded image
+
+
+# auto-detect the characteristics of the embedded image
+depthCurImg="8"			# default color depth
+colorspaceCurImg="sRGB"		# default color space
+dpi=$DEFAULT_DPI		# default resolution
+
+getImgInfo "$page" "$widthPDF" "$heightPDF" "$curImgInfo"
+ret_code="$?"
+
+# in case the page contains text do not OCR, unless the FORCE_OCR flag is set
+if ([ "$ret_code" -eq "1" ] && [ "$FORCE_OCR" -eq "0" ]); then
+	echo "Page $page: Exiting... (Use the -f option to force OCRing, even though fonts are available in the input file)" && exit $EXIT_BAD_INPUT_FILE
+elif ([ "$ret_code" -eq "1" ] && [ "$FORCE_OCR" -eq "1" ]); then
+	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: OCRing anyway, assuming a default resolution of $dpi dpi"
+# in case the page contains more than one image, warn the user but go on with default parameters
+elif [ "$ret_code" -eq "2" ]; then
+	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Continuing anyway, assuming a default resolution of $dpi dpi"
+else
+	# read the image characteristics from the file
+	dpi=`cat "$curImgInfo" | grep "^DPI=" | cut -f2 -d"="`
+	colorspaceCurImg=`cat "$curImgInfo" | grep "^COLOR_SPACE=" | cut -f2 -d"="`
+	depthCurImg=`cat "$curImgInfo" | grep "^DEPTH=" | cut -f2 -d"="`
+fi
+
+# perform oversampling if the resolution is not sufficient to get good OCR results
+if [ "$dpi" -lt "$OVERSAMPLING_DPI" ]; then
+	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Low image resolution detected ($dpi dpi). Performing oversampling ($OVERSAMPLING_DPI dpi) to try to get better OCR results." 
+	dpi="$OVERSAMPLING_DPI"
+elif [ "$dpi" -lt "200" ]; then
+	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Low image resolution detected ($dpi dpi). If needed, please use the \"-o\" to try to get better OCR results." 
+fi
+	
+# Identify if page image should be saved as ppm (color), pgm (gray) or pbm (b&w)
+ext="ppm" 	# by default (color image) the extension of the extracted image is ppm
+opt=""		# by default (color image) no option as to be passed to pdftoppm
+if [ "$colorspaceCurImg" = "Gray" ] && [ "$depthCurImg" = "1" ]; then		# if monochrome (b&w)
+	ext="pbm"
+	opt="-mono"
+elif [ "$colorspaceCurImg" = "Gray" ]; then					# if gray
+	ext="pgm"
+	opt="-gray"
+fi
+curImgPixmap="$TMP_FLD/$page.$ext"
+curImgPixmapDeskewed="$TMP_FLD/$page.deskewed.$ext"
+curImgPixmapClean="$TMP_FLD/$page.cleaned.$ext"
+
+# extract current page as image with correct orientation and resolution
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracting image as $ext file (${dpi} dpi)"
+! pdftoppm -f $page -l $page -r $dpi $opt "$FILE_INPUT_PDF" > "$curImgPixmap" \
+	&& echo "Could not extract page $page as $ext from \"$FILE_INPUT_PDF\". Exiting..." && exit $EXIT_OTHER_ERROR
+
+# if requested deskew image (without changing its size in pixel)
+widthCurImg=$(($dpi*$widthPDF/72))
+heightCurImg=$(($dpi*$heightPDF/72))
+if [ "$PREPROCESS_DESKEW" -eq "1" ]; then
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Deskewing image"
+	! convert "$curImgPixmap" -deskew 40% -gravity center -extent ${widthCurImg}x${heightCurImg} "$curImgPixmapDeskewed" \
+		&& echo "Could not deskew \"$curImgPixmap\". Exiting..." && exit $EXIT_OTHER_ERROR
+else
+	ln -s `basename "$curImgPixmap"` "$curImgPixmapDeskewed"
+fi
+
+# if requested clean image with unpaper to get better OCR results
+if [ "$PREPROCESS_CLEAN" -eq "1" ]; then
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Cleaning image with unpaper"
+	! unpaper --dpi $dpi --mask-scan-size 100 \
+		--no-deskew --no-grayfilter --no-blackfilter --no-mask-center --no-border-align \
+		"$curImgPixmapDeskewed" "$curImgPixmapClean" 1> /dev/null \
+		&& echo "Could not clean \"$curImgPixmapDeskewed\". Exiting..." && exit $EXIT_OTHER_ERROR
+else
+	ln -s `basename "$curImgPixmapDeskewed"` "$curImgPixmapClean"
+fi
+
+# perform OCR
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Performing OCR"
+! tesseract -l "$LAN" "$curImgPixmapClean" "$curHocr" hocr $TESS_CFG_FILES 1> /dev/null 2> /dev/null \
+	&& echo "Could not OCR file \"$curImgPixmapClean\". Exiting..." && exit $EXIT_OTHER_ERROR
+# Tesseract names the output files differently in some distributions.
+if [ -e "$curHocr.html" ]; then
+	mv "$curHocr.html" "$curHocr"
+elif [ -e "$curHocr.hocr" ]; then
+	mv "$curHocr.hocr" "$curHocr"
+elif [ ! -e "$curHocr" ]; then
+	echo "\"$curHocr[.html|.hocr]\" not found. Exiting..." && exit $EXIT_OTHER_ERROR
+fi
+
+# embed text and image to new pdf file
+if [ "$PREPROCESS_CLEANTOPDF" -eq "1" ]; then
+	image4finalPDF="$curImgPixmapClean"
+else
+	image4finalPDF="$curImgPixmapDeskewed"	
+fi
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF"
+! python2 $SRC/hocrTransform.py -r $dpi -i "$image4finalPDF" "$curHocr" "$curOCRedPDF" \
+	&& echo "Could not create PDF file from \"$curHocr\". Exiting..." && exit $EXIT_OTHER_ERROR
+
+# if requested generate special debug PDF page with visible OCR text
+if [ $PDF_NOIMG -eq "1" ] ; then
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF (debug page)"
+	! python2 $SRC/hocrTransform.py -b -r $dpi "$curHocr" "$curOCRedPDFDebug" \
+		&& echo "Could not create PDF file from \"$curHocr\". Exiting..." && exit $EXIT_OTHER_ERROR	
+fi
+
+# delete temporary files created for the current page
+# to avoid using to much disk space in case of PDF files having many pages
+if [ $KEEP_TMP -eq 0 ]; then
+	rm -f "$curOrigImg"*
+	rm -f "$curHocr"
+	rm -f "$curImgPixmap"
+	rm -f "$curImgPixmapDeskewed"
+	rm -f "$curImgPixmapClean"
+	rm -f "$curImgInfo"
+fi
+
+exit 0

tess-cfg/README.md

@@ -1,10 +0,0 @@
-Note
-====
-
-The file(s) located in this folder are tesseract configuration files.
-(Tesseract configuration files enable to tune the behaviour of tesseract)
-
-If needed, these files should be copied to the "tessdata/configs" folder of your tesseract installation.
-
-To request OCRmyPDF.sh to use a configuration file, please use the -C option
-

tess-cfg/no_ligature

@@ -1 +0,0 @@
-tessedit_char_blacklist fifl