Release notes updated for v1.1-stable

.gitignore

@@ -1,3 +1,2 @@
 tmp/
 log/
-*.pyc

COPYRIGHT.md

@@ -1,5 +1,3 @@
-Copyright (c) 2013 fritz-hh from Github
-
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights

OCRmyPDF.sh

@@ -1,17 +1,10 @@
 #!/bin/sh
 ##############################################################################
-# Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
+# Copyright (c) 2013: fritz-hh from Github (https://github.com/fritz-hh)
 ##############################################################################
 
-# Determine real path of this script, following symlinks if present
+TOOLNAME="OCRmyPDF"
-! command -v python2 > /dev/null && echo "Please install python v2.x. Exiting..." && exit 1
+VERSION="v1.1-stable"
-BASEPATH="$(dirname $(python2 -c "import os; print os.path.realpath(\"$0\")"))"
-
-# Import required scripts
-. "$BASEPATH/src/config.sh"
-
-# Set variables corresponding to the input parameters
-ARGUMENTS="$@"
 
 START=`date +%s`
 
@@ -22,13 +15,13 @@ Script aimed at generating a searchable PDF file from a PDF file containing only
 (The script performs optical character recognition of each respective page using the
 tesseract engine)
 
-Copyright: fritz-hh  from Github (https://github.com/fritz-hh)
+Copyright: fritz from NAS4Free forum
 Version: $VERSION
 
-Usage: OCRmyPDF.sh  [-h] [-v] [-g] [-k] [-d] [-c] [-i] [-o dpi] [-f] [-l language] [-C filename] inputfile outputfile
+Usage: OCRmyPDF.sh  [-h] [-v] [-g] [-k] [-d] [-c] [-i] [-l language] [-C filename] inputfile outputfile
 
 -h : Display this help message
--v : Increase the verbosity (this option can be used more than once) (e.g. -vvv)
+-v : Increase the verbosity (this option can be used more than once)
 -k : Do not delete the temporary files
 -g : Activate debug mode:
      - Generates a PDF file containing each page twice (once with the image, once without the image
@@ -38,20 +31,15 @@ Usage: OCRmyPDF.sh  [-h] [-v] [-g] [-k] [-d] [-c] [-i] [-o dpi] [-f] [-l languag
 -d : Deskew each page before performing OCR
 -c : Clean each page before performing OCR
 -i : Incorporate the cleaned image in the final PDF file (by default the original image	
-     image, or the deskewed image if the -d option is set)
+     image, or the deskewed image if the -d option is set, is incorporated)
--o : If the resolution of an image is lower than dpi value provided as argument, provide the OCR engine with 
-     an oversampled image having the latter dpi value. This can improve the OCR results but can lead to a larger output PDF file.
-     (default: no oversampling performed)
--f : Force to OCR the whole document, even if some page already contain font data 
-     (which should not be the case for PDF files built from scanned images) 
 -l : Set the language of the PDF file in order to improve OCR results (default "eng")
-     Any language supported by tesseract is supported (Tesseract uses 3-character ISO 639-2 language codes)
+     Any language supported by tesseract is supported.
-     Multiple languages may be specified, separated by '+' characters.
 -C : Pass an additional configuration file to the tesseract OCR engine.
      (this option can be used more than once)
-     Note 1: The configuration file must be available in the "tessdata/configs" folder of your tesseract installation
+     Note: The configuration file must be available in the "tessdata/configs" folder
+     of your tesseract installation
 inputfile  : PDF file to be OCRed
-outputfile : The PDF/A file that will be generated 
+outputfile : The PDF/A file to be generated 
 --------------------------------------------------------------------------------------
 EOF
 }
@@ -75,38 +63,48 @@ absolutePath() {
 }
 
 
+
+# Initialization of constants
+EXIT_BAD_ARGS="1"		# possible exit codes
+EXIT_BAD_INPUT_FILE="2"
+EXIT_MISSING_DEPENDENCY="3"
+EXIT_INVALID_OUPUT_PDFA="4"
+EXIT_OTHER_ERROR="5"
+LOG_ERR="0"			# 0=only error messages
+LOG_INFO="1"			# 1=error messages and some infos
+LOG_DEBUG="2"			# 2=debug level logging
+SRC="./src"				# location of the source folder (except source of external tools like jhove)
+JHOVE="./jhove/bin/JhoveApp.jar"	# java SW for validating the final PDF/A
+JHOVE_CFG="./jhove/conf/jhove.conf"	# location of the jhove config file
+
 # Initialization the configuration parameters with default values
 VERBOSITY="$LOG_ERR"		# default verbosity level
 LAN="eng"			# default language of the PDF file (required to get good OCR results)
-KEEP_TMP="0"			# 0=no, 1=yes (keep the temporary files)
+KEEP_TMP="0"			# do not delete the temporary files (default)
 PREPROCESS_DESKEW="0"		# 0=no, 1=yes (deskew image)
 PREPROCESS_CLEAN="0"		# 0=no, 1=yes (clean image to improve OCR)
 PREPROCESS_CLEANTOPDF="0"	# 0=no, 1=yes (put cleaned image in final PDF)
-OVERSAMPLING_DPI="0"		# 0=do not perform oversampling (dpi value under which oversampling should be performed)
 PDF_NOIMG="0"			# 0=no, 1=yes (generates each PDF page twice, with and without image)
-FORCE_OCR="0"			# 0=do not force, 1=force (force to OCR the whole document, even if some page already contain font data)
 TESS_CFG_FILES=""		# list of additional configuration files to be used by tesseract
 
 # Parse optional command line arguments
-while getopts ":hvgkdcio:fl:C:" opt; do
+while getopts ":hvgkdcil:C:" opt; do
 	case $opt in
 		h) usage ; exit 0 ;;
 		v) VERBOSITY=$(($VERBOSITY+1)) ;;
 		k) KEEP_TMP="1" ;;
-		g) PDF_NOIMG="1"; VERBOSITY="$LOG_DEBUG"; KEEP_TMP="1" ;;
+		g) PDF_NOIMG="1"; VERBOSITY="10"; KEEP_TMP="1" ;;
 		d) PREPROCESS_DESKEW="1" ;;
 		c) PREPROCESS_CLEAN="1" ;;
 		i) PREPROCESS_CLEANTOPDF="1" ;;
-		o) OVERSAMPLING_DPI="$OPTARG" ;;
-		f) FORCE_OCR="1" ;;
 		l) LAN="$OPTARG" ;;
 		C) TESS_CFG_FILES="$OPTARG $TESS_CFG_FILES" ;;
 		\?)
-			echo "Invalid option: -$OPTARG"
+			echo "Invalid option: -$OPTARG" >&2
 			usage
 			exit $EXIT_BAD_ARGS ;;
 		:)
-			echo "Option -$OPTARG requires an argument"
+			echo "Option -$OPTARG requires an argument" >&2
 			usage
 			exit $EXIT_BAD_ARGS ;;
 	esac
@@ -115,174 +113,238 @@ done
 # Remove the optional arguments parsed above.
 shift $((OPTIND-1))
 
-# Check if the number of mandatory parameters provided is as expected
+# Check if the number of mandatory parameters
+# provided is as expected
 if [ "$#" -ne "2" ]; then
-	echo "Exactly two mandatory argument shall be provided ($# arguments provided)"
+	echo "Exactly two mandatory argument shall be provided ($# arguments provided)" >&2
 	usage
 	exit $EXIT_BAD_ARGS
 fi
 
-[ ! -f "$1" ] \
+! absolutePath "$1" > /dev/null && echo "The folder in which the input file should be located does not exist. Exiting..." >&2 && exit $EXIT_BAD_ARGS
-	&& echo "The input file does not exist. Exiting..." && exit $EXIT_BAD_ARGS
 FILE_INPUT_PDF="`absolutePath "$1"`"
-
+! absolutePath "$2" > /dev/null  && echo "The folder in which the output file should be generated does not exist. Exiting..." >&2 && exit $EXIT_BAD_ARGS
-! absolutePath "$2" > /dev/null \
-	&& echo "The folder in which the output file should be generated does not exist. Exiting..." && exit $EXIT_BAD_ARGS
 FILE_OUTPUT_PDFA="`absolutePath "$2"`"
 
 
 
 # set script path as working directory
-cd "$BASEPATH"
+cd "`dirname $0`"
 
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "$TOOLNAME version: $VERSION"
+[ $VERBOSITY -ge $LOG_INFO ] && echo "$TOOLNAME version: $VERSION"
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Arguments: $ARGUMENTS"
 
 # check if the required utilities are installed
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Checking if all dependencies are installed"
-! command -v identify > /dev/null && echo "Please install ImageMagick. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v identify > /dev/null && echo "Please install ImageMagick. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v parallel > /dev/null && echo "Please install GNU Parallel. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v pdfimages > /dev/null && echo "Please install poppler-utils. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v pdfimages > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v pdftoppm > /dev/null && echo "Please install poppler-utils. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v pdftoppm > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v pdftk > /dev/null && echo "Please install pdftk. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v pdffonts > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+[ $PREPROCESS_CLEAN -eq 1 ] && ! command -v unpaper > /dev/null && echo "Please install unpaper. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-[ $PREPROCESS_CLEAN -eq 1 ] && ! command -v unpaper > /dev/null && echo "Please install unpaper. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v tesseract > /dev/null && echo "Please install tesseract and tesseract-data. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! command -v tesseract > /dev/null && echo "Please install tesseract and tesseract-data. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v python > /dev/null && echo "Please install python, and the python libraries: reportlab, lxml. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! python2 -c 'import lxml' 2>/dev/null && echo "Please install the python library lxml. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v gs > /dev/null && echo "Please install ghostcript. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-! python2 -c 'import sys, reportlab; (getattr(reportlab, "Version", "0.0") >= "3.0") or sys.exit(1)' 2>/dev/null \
+! command -v java > /dev/null && echo "Please install java. Exiting..." >&2 && exit $EXIT_MISSING_DEPENDENCY
-	&& echo "Please install the python library reportlab. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
-
-! command -v gs > /dev/null && echo "Please install ghostscript. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
-! command -v java > /dev/null && echo "Please install java. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
-
-
-# ensure the right tesseract version is installed
-# older versions are known to produce malformed hocr output and should not be used
-# Even 3.02.01 fails in few cases (see issue #28). I decided to allow this version anyway because
-# 3.02.02 is not yet available for some widespread linux distributions
-reqtessversion="3.02.01"
-tessversion=`tesseract -v 2>&1 | grep "tesseract" | sed s/[^0-9.]//g`
-tesstooold=$(echo "`echo $tessversion | sed s/[.]//2`-`echo $reqtessversion | sed s/[.]//2` < 0" | bc)
-[ "$tesstooold" -eq "1" ] \
-	&& echo "Please install tesseract ${reqtessversion} or newer (currently installed version is ${tessversion})" && exit $EXIT_MISSING_DEPENDENCY
-
-# ensure the right GNU parallel version is installed
-# older version do not support -q flag (required to escape special characters)
-reqparallelversion="20121122"
-parallelversion=`parallel --minversion 0`
-! parallel --minversion "$reqparallelversion" > /dev/null \
-	&& echo "Please install GNU parallel ${reqparallelversion} or newer (currently installed version is ${parallelversion})" && exit $EXIT_MISSING_DEPENDENCY
-
-# ensure pdftoppm is provided by poppler-utils, not the older xpdf version
-! pdftoppm -v 2>&1 | grep -q 'Poppler' && echo "Please remove xpdf and install poppler-utils. Exiting..." && $EXIT_MISSING_DEPENDENCY
 
 
 
-# Display the version of the tools if log level is LOG_DEBUG
-if [ $VERBOSITY -ge $LOG_DEBUG ]; then
-	echo "--------------------------------"
-	echo "ImageMagick version:"
-	identify --version
-	echo "--------------------------------"
-	echo "GNU Parallel version:"
-	parallel --version
-	echo "--------------------------------"
-	echo "Poppler-utils version:"
-	pdfimages -v
-	pdftoppm -v
-	pdffonts -v
-	echo "--------------------------------"
-	echo "unpaper version:"
-	unpaper --version
-	echo "--------------------------------"
-	echo "tesseract version:"
-	tesseract --version
-	echo "--------------------------------"
-	echo "python2 version:"
-	python2 --version
-	echo "--------------------------------"
-	echo "Ghostscript version:"
-	gs --version
-	echo "--------------------------------"
-	echo "Java version:"
-	java -version
-	echo "--------------------------------"
-fi
 
-
+# Initialize path to temporary files
-
+today=$(date +"%Y%m%d_%H%M")
-# check if the languages passed to tesseract are all supported
+fld=$(basename "$FILE_INPUT_PDF" | sed 's/[.][^.]*//')
-for currentlan in `echo "$LAN" | sed 's/+/ /g'`; do
+TMP_FLD="./tmp/$today.filename.$fld"
-	if ! tesseract --list-langs 2>&1 | grep "^$currentlan\$" > /dev/null; then
+FILE_TMP="$TMP_FLD/tmp.txt"						# temporary file with a very short lifetime (may be used for several things)
-		echo "The language \"$currentlan\" is not supported by tesseract."
+FILE_SIZE_PAGES="$TMP_FLD/page-sizes.txt"				# size in pt of the respective page of the input PDF file
-		tesseract --list-langs 2>&1 | tr '\n' ' '; echo
+FILE_OUTPUT_PDF_CAT="${TMP_FLD}/ocred.pdf"				# concatenated OCRed PDF files
-		echo "Exiting..."
+FILE_OUTPUT_PDFA_WO_META="${TMP_FLD}/ocred-pdfa-wo-metadata.pdf"	# PDFA file before appending metadata
-		exit $EXIT_BAD_ARGS
-	fi
-done
-
-
-
-# Initialize path to temporary files using mktemp
-# Goal: save tmp file in a sub-folder of the $TMPDIR environment variable (or in "/tmp" if unset)
-# Unfortunately, Linux mktemp is not compatible with FreeBSD/OSX mktemp
-# Linux version requires no arg
-# FreeBSD requires '-t prefix' to be used so that $TMPDIR is taken into account
-# But in Linux '-t template' is handled differently than in FreeBSD
-# Therefore different calls must be used for Linux and for FreeBSD
-prefix="$(date +"%Y%m%d_%H%M").filename.$(basename "$FILE_INPUT_PDF" | sed 's/[.][^.]*$//')"	# prefix made of date, time and pdf file name without extension
-TMP_FLD=`mktemp -d 2>/dev/null || mktemp -d -t "${prefix}" 2>/dev/null`				# try Linux syntax first, if it fails try FreeBSD/OSX			
-if [ $? -ne 0 ]; then
-	if [ -z "$TMPDIR" ]; then
-		echo "Could not create folder for temporary files. Please ensure you have sufficient right and \"/tmp\" exists"
-	else
-		echo "Could not create folder for temporary files. Please ensure you have sufficient right and \"$TMPDIR\" exists"
-	fi
-	exit $EXIT_FILE_ACCESS_ERROR
-fi
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Created temporary folder: \"$TMP_FLD\""
-
-FILE_TMP="${TMP_FLD}/tmp.txt"						# temporary file with a very short lifetime (may be used for several things)
-FILE_PAGES_INFO="${TMP_FLD}/pages-info.txt"				# for each page: page #; width in pt; height in pt
 FILE_VALIDATION_LOG="${TMP_FLD}/pdf_validation.log"			# log file containing the results of the validation of the PDF/A file
 
+# Create tmp folder
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Creating temporary folder: \"$TMP_FLD\""
+rm -r -f "${TMP_FLD}"
+mkdir -p "${TMP_FLD}"
 
 
-# get the size of each pdf page (width / height) in pt (i.e. inch/72)
+
+
+# get the size of each pdf page (width / height) in pt (inch*72)
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Input file: Extracting size of each page (in pt)"
 ! identify -format "%w %h\n" "$FILE_INPUT_PDF" > "$FILE_TMP" \
-	&& echo "Could not get size of PDF pages. Exiting..." && exit $EXIT_BAD_INPUT_FILE
+	&& echo "Could not get size of PDF pages. Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE
-# removing empty lines (last one should be) and add page # before each line
+# removing empty lines (last one should be) and prepend page # before each line
-sed '/^$/d' "$FILE_TMP" | awk '{printf "%04d %s\n", NR, $0}' > "$FILE_PAGES_INFO"
+sed '/^$/d' "$FILE_TMP" | awk '{printf "%04d %s\n", NR, $0}' > "$FILE_SIZE_PAGES"
-numpages=`tail -n 1 "$FILE_PAGES_INFO" | cut -f1 -d" "`
+numpages=`tail -n 1 "$FILE_SIZE_PAGES" | cut -f1 -d" "`
 
-# process each page of the input pdf file
+# Itterate the pages of the input pdf file
-parallel --gnu -q -k --halt-on-error 1 "$OCR_PAGE" "$FILE_INPUT_PDF" "{}" "$numpages" "$TMP_FLD" \
+while read pageSize ; do
-	"$VERBOSITY" "$LAN" "$KEEP_TMP" "$PREPROCESS_DESKEW" "$PREPROCESS_CLEAN" "$PREPROCESS_CLEANTOPDF" "$OVERSAMPLING_DPI" \
-	"$PDF_NOIMG" "$TESS_CFG_FILES" "$FORCE_OCR" < "$FILE_PAGES_INFO"
-ret_code="$?"
-[ $ret_code -ne 0 ] && exit $ret_code 
 
-# concatenate all pages and convert the pdf file to match PDF/A format
+	page=`echo $pageSize | cut -f1 -d" "`
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Concatenating all pages to the final PDF/A file" 
+	[ $VERBOSITY -ge $LOG_INFO ] && echo "Processing page $page / $numpages"
+	
+	# create the name of the required file
+	curOrigImg="$TMP_FLD/${page}_Image"			# original image available in the current PDF page 
+								# (the image file may have a different orientation than in the pdf file)
+	curHocr="$TMP_FLD/$page.hocr"				# hocr file to be generated by the OCR SW for the current page
+	curOCRedPDF="$TMP_FLD/${page}-ocred.pdf"		# PDF file containing the image + the OCRed text for the current page
+	curOCRedPDFDebug="$TMP_FLD/${page}-debug-ocred.pdf"	# PDF file containing data required to find out if OCR worked correctly
+	
+	# get width / height of PDF page (in pt)
+	widthPDF=`echo $pageSize | cut -f2 -d" "`
+	heightPDF=`echo $pageSize | cut -f3 -d" "`
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: size ${heightPDF}x${widthPDF} (h*w in pt)"
+	# extract raw image from pdf file to compute resolution
+	# unfortunatelly this image can have another orientation than in the pdf...
+	# so we will have to extract it again later using pdftoppm
+	pdfimages -f $page -l $page -j "$FILE_INPUT_PDF" "$curOrigImg" 1>&2	
+	# count number of extracted images
+	nbImg=`ls -1 "$curOrigImg"* | wc -l`
+	[ $nbImg -ne "1" ] && echo "Expecting exactly 1 image on page $page (found $nbImg). Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE
+	# Get characteristics of the extracted image
+	curImg=`ls -1 "$curOrigImg"*`
+	propCurImg=`identify -format "%w %h %[colorspace]" "$curImg"`
+	widthCurImg=`echo "$propCurImg" | cut -f1 -d" "`
+	heightCurImg=`echo "$propCurImg" | cut -f2 -d" "`
+	colorspaceCurImg=`echo "$propCurImg" | cut -f3 -d" "`
+	# switch height/width values if the image has not the right orientation
+	# we make here the assumption that vertical/horizontal dpi are equal
+	# we will check that later
+	if [ $((($heightPDF-$widthPDF)*($heightCurImg-$widthCurImg))) -lt 0 ]; then
+		[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracted image has wrong orientation. Inverting image height/width values"
+		tmpval=$heightCurImg
+		heightCurImg=$widthCurImg
+		widthCurImg=$tmpval
+	fi
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: size ${heightCurImg}x${widthCurImg} (h*w pixel)"	
+	# compute the resolution of the image
+	dpi_x=`echo "scale=5;$widthCurImg*72/$widthPDF" | bc`
+	dpi_y=`echo "scale=5;$heightCurImg*72/$heightPDF" | bc`
+	# compute the maximum allowed resolution difference that can be cause by:
+	# - the truncated PDF with/height in pt
+	# - the precision of dpi value
+	epsilon=`echo "scale=5;($widthCurImg*72/$widthPDF^2)+($heightCurImg*72/$heightPDF^2)+0.00002" | bc`	# max inaccuracy due to truncation of PDF size in pt
+	[ `echo "($dpi_x - $dpi_y) < $epsilon " | bc` -eq 0 -o `echo "($dpi_y - $dpi_x) < $epsilon " | bc` -eq 0 ] \
+		&& echo "Resolutions difference ($dpi_x/$dpi_y) higher than expected ($epsilon). Exiting..." >&2 && exit $EXIT_BAD_INPUT_FILE
+	dpi=`echo "scale=5;($dpi_x+$dpi_y)/2+0.5" | bc` # adding 0.5 is required for rounding
+	dpi=`echo "scale=0;$dpi/1" | bc`		# round to the nearest integer
+	
+	# Identify if page image should be saved as ppm (color) or pgm (gray)
+	ext="ppm"
+	opt=""		
+	if [ $colorspaceCurImg = "Gray" ]; then
+		ext="pgm"
+		opt="-gray"
+	fi
+	curImgPixmap="$TMP_FLD/$page.$ext"
+	curImgPixmapDeskewed="$TMP_FLD/$page.deskewed.$ext"
+	curImgPixmapClean="$TMP_FLD/$page.cleaned.$ext"
+	
+	# extract current page as image with right orientation and resoltution
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracting image as $ext file (${dpi} dpi)"
+	! pdftoppm -f $page -l $page -r $dpi $opt "$FILE_INPUT_PDF" > "$curImgPixmap" \
+		&& echo "Could not extract page $page as $ext from \"$FILE_INPUT_PDF\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
+
+	# if requested deskew image (without changing its size in pixel)
+	if [ "$PREPROCESS_DESKEW" -eq "1" ]; then
+		[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Deskewing image"
+		! convert "$curImgPixmap" -deskew 40% -gravity center -extent ${widthCurImg}x${heightCurImg} "$curImgPixmapDeskewed" \
+			&& echo "Could not deskew \"$curImgPixmap\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
+	else
+		cp "$curImgPixmap" "$curImgPixmapDeskewed"
+	fi
+
+	# if requested clean image with unpaper to get better OCR results
+	if [ "$PREPROCESS_CLEAN" -eq "1" ]; then
+		[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Cleaning image with unpaper"
+		! unpaper --dpi $dpi --mask-scan-size 100 \
+			--no-deskew --no-grayfilter --no-blackfilter --no-mask-center --no-border-align \
+			"$curImgPixmapDeskewed" "$curImgPixmapClean" 1> /dev/null \
+			&& echo "Could not clean \"$curImgPixmapDeskewed\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
+	else
+		cp "$curImgPixmapDeskewed" "$curImgPixmapClean"
+	fi
+
+	# perform OCR
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Performing OCR"
+	! tesseract -l "$LAN" "$curImgPixmapClean" "$curHocr" hocr $TESS_CFG_FILES 1> /dev/null 2> /dev/null \
+		&& echo "Could not OCR file \"$curImgPixmapClean\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
+	mv "$curHocr.html" "$curHocr"
+
+	# embed text and image to new pdf file
+	if [ "$PREPROCESS_CLEANTOPDF" -eq "1" ]; then
+		image4finalPDF="$curImgPixmapClean"
+	else
+		image4finalPDF="$curImgPixmapDeskewed"	
+	fi
+	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF"
+	! python $SRC/hocrTransform.py -r $dpi -i "$image4finalPDF" "$curHocr" "$curOCRedPDF" \
+		&& echo "Could not create PDF file from \"$curHocr\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR
+	
+	# if requested generate special debug PDF page with visible OCR text
+	if [ $PDF_NOIMG -eq "1" ] ; then
+		[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF (debug page)"
+		! python $SRC/hocrTransform.py -b -r $dpi "$curHocr" "$curOCRedPDFDebug" \
+			&& echo "Could not create PDF file from \"$curHocr\". Exiting..." >&2 && exit $EXIT_OTHER_ERROR	
+	fi
+	
+	# delete temporary files created for the current page
+	# to avoid using to much disk space in case of PDF files having many pages
+	if [ $KEEP_TMP -eq 0 ]; then
+		rm "$curOrigImg"*.*
+		rm "$curHocr"
+		rm "$curImgPixmap"
+		rm "$curImgPixmapDeskewed"
+		rm "$curImgPixmapClean"
+	fi
+	
+done < "$FILE_SIZE_PAGES"
+
+
+
+
+# concatenate all pages
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Concatenating all pages"
+! pdftk "${TMP_FLD}/"*-ocred.pdf cat output "$FILE_OUTPUT_PDF_CAT" \
+	&& echo "Could not concatenate individual PDF pages (\"${TMP_FLD}/*-ocred.pdf\") to one file. Exiting..." >&2 && exit $EXIT_OTHER_ERROR
+
+# convert the pdf file to match PDF/A format
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Converting to PDF/A" 
 ! gs -dQUIET -dPDFA -dBATCH -dNOPAUSE -dUseCIEColor \
 	-sProcessColorModel=DeviceCMYK -sDEVICE=pdfwrite -sPDFACompatibilityPolicy=2 \
-	-sOutputFile="$FILE_OUTPUT_PDFA" "${TMP_FLD}/"*ocred*.pdf 1> /dev/null 2> /dev/null \
+	-sOutputFile="$FILE_OUTPUT_PDFA" "$FILE_OUTPUT_PDF_CAT" 1> /dev/null 2> /dev/null \
-	&& echo "Could not concatenate all pages to the final PDF/A file. Exiting..." && exit $EXIT_OTHER_ERROR
+	&& echo "Could not convert PDF file \"$FILE_OUTPUT_PDF_CAT\" to PDF/A. Exiting..." >&2 && exit $EXIT_OTHER_ERROR
+
+# # Write metadata
+# # Needs to be done after converting to PDF/A, as gs does not preserve metadata
+# [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Update metadata (creator, producer, and title)" 
+# title=`basename "$FILE_INPUT_PDF" | sed 's/[.][^.]*//' | \
+	# sed 's/_/ /g' | sed 's/-/ /g' | \
+	# sed 's/\([[:lower:]]\)\([[:upper:]]\)/\1 \2/g' | \
+	# sed 's/\([[:alpha:]]\)\([[:digit:]]\)/\1 \2/g' | \
+	# sed 's/\([[:digit:]]\)\([[:alpha:]]\)/\1 \2/g'`	# transform the file name (with extension) into distinct words
+# pdftk "$FILE_OUTPUT_PDFA_WO_META" update_info_utf8 - output "$FILE_OUTPUT_PDFA" << EOF
+# InfoBegin
+# InfoKey: Title
+# InfoValue: $title
+# InfoBegin
+# InfoKey: Creator
+# InfoValue: $TOOLNAME $VERSION
+# InfoBegin
+# InfoKey: Producer
+# InfoValue: ghostcript `gs --version`, pdftk
+# EOF
 
 # validate generated pdf file (compliance to PDF/A)
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Checking compliance to PDF/A standard" 
-! java -jar "$JHOVE" -c "$JHOVE_CFG" -m PDF-hul "$FILE_OUTPUT_PDFA" 2> /dev/null 1> "$FILE_VALIDATION_LOG" \
+java -jar "$JHOVE" -c "$JHOVE_CFG" -m PDF-hul "$FILE_OUTPUT_PDFA" > "$FILE_VALIDATION_LOG"
-	&& echo "Unexpected error while checking compliance to PDF/A file. Exiting..." && exit $EXIT_OTHER_ERROR
 grep -i "Status|Message" "$FILE_VALIDATION_LOG" # summary of the validation
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "The full validation log is available here: \"$FILE_VALIDATION_LOG\""
 # check the validation results
 pdf_valid=1
-grep -i 'ErrorMessage' "$FILE_VALIDATION_LOG" && pdf_valid=0
+grep -i 'ErrorMessage' "$FILE_VALIDATION_LOG" >&2 && pdf_valid=0
-grep -i 'Status.*not valid' "$FILE_VALIDATION_LOG" && pdf_valid=0
+grep -i 'Status.*not valid' "$FILE_VALIDATION_LOG" >&2 && pdf_valid=0
-grep -i 'Status.*Not well-formed' "$FILE_VALIDATION_LOG" && pdf_valid=0
+grep -i 'Status.*Not well-formed' "$FILE_VALIDATION_LOG" >&2 && pdf_valid=0
-! grep -i 'Profile:.*PDF/A-1' "$FILE_VALIDATION_LOG" > /dev/null && echo "PDF file profile is not PDF/A-1" && pdf_valid=0
+! grep -i 'Profile:.*PDF/A-1' "$FILE_VALIDATION_LOG" > /dev/null && echo "PDF file profile is not PDF/A-1" >&2 && pdf_valid=0
-[ $pdf_valid -ne 1 ] && echo "Output file: The generated PDF/A file is INVALID"
+[ $pdf_valid -ne 1 ] && echo "Output file: The generated PDF/A file is INVALID" >&2
-[ $pdf_valid -eq 1 ] && [ $VERBOSITY -ge $LOG_INFO ] && echo "Output file: The generated PDF/A file is VALID"
+[ $pdf_valid -ne 0 ] && [ $VERBOSITY -ge $LOG_INFO ] && echo "Output file: The generated PDF/A file is VALID"
 
 
 
@@ -298,4 +360,4 @@ END=`date +%s`
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Script took $(($END-$START)) seconds"
 
 
-[ $pdf_valid -ne 1 ] && exit $EXIT_INVALID_OUTPUT_PDFA || exit 0
+[ $pdf_valid -ne 1 ] && exit $EXIT_INVALID_OUPUT_PDFA || exit 0

README.md

@@ -5,19 +5,15 @@ OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be search
 
 To get the script usage, call: sh ./OCRmyPDF.sh -h
 
-Main features
+Features
 --------
 
-- Generates a searchable PDF/A file from a PDF file containing only images
+- Generate a searchable PDF/A file from a PDF file containing only images
-- Places OCRed text accurately below the image to ease copy / paste
+- Place OCRed text accurately below the image to easy copy / paste
-- Keeps the exact resolution of the original embedded images
+- Keep the exact resolution of the original embedded images
-    - or if requested oversamples the images before OCRing so as to get better results 
+- If requested deskew and / or clean the image before performing OCR
-- If requested deskews and / or clean the image before performing OCR
+- Validate the generated file against the PDF/A specification using jhove
-- Validates the generated file against the PDF/A specification using jhove
 - Provides debug mode to enable easy verification of the OCR results
-- Processes several pages in parallel if more than one CPU core is available
-
-For details: please consult the release notes
 
 Motivation
 ----------
@@ -35,35 +31,12 @@ I found many, but none of them were really satisfying.
 ... so I decided to develop my own tool (using various existing scripts as an inspiration)
 
 Install
--------
+--------
 
-Download OCRmyPDF here: https://github.com/fritz-hh/OCRmyPDF/releases
+Download OCRmyPDF here: https://github.com/fritz-hh/OCRmyPDF/tags
 
 Copy the file in onto your linux/unix machine and extract it.
 
 Run: "sh ./OCRmyPDF.sh -h" to get the script usage
 
-If not yet installed, the script will notify you about dependencies that need to be installed.
+If not yet installed, the script will notify you about dependencies that need to be installed
-The script requires specific versions of the dependencies. Older version than the ones mentioned in the release notes are likely not to be compatible to OCRmyPDF.
-
-Support
--------
-
-In case you detect an issue, please:
-
-- Check if your issue is already known
-- If no problem report exists on github, please create one here: https://github.com/fritz-hh/OCRmyPDF/issues
-- Describe your problem thoroughly
-- Append the console output of the script when running the debug mode (-g option)
-- If possible provide your input PDF file as well as the content of the temporary folder (using a file sharing service like www.file-upload.net)
-
-Press & Media
--------------
-
-- c't 1-2014, page 59: Detailed presentation of OCRmyPDF v1.0 in the leading german IT magazine c't (http://www.heise.de/ct/inhalt/2014/1/58/) 
-- heise Open Source, 09/2014: Texterkennung mit OCRmyPDF (http://www.heise.de/-2356670)
-
-Disclaimer
-----------
-
-The software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

RELEASE_NOTES.md

@@ -5,198 +5,6 @@ Please always read this file before installing the package
 
 Download software here: https://github.com/fritz-hh/OCRmyPDF/tags
 
-v2.2-stable (2014-09-29):
-=======
-
-New features
-------------
-
-- None
-
-Changes
--------
-
-- Update to jhove v1.11
-- Request the python library reportlab v3.0 or newer (So that we could remove a patch to the previous version of reportlab leading to issues for some users)
-
-Fixes
------
-
-- Fix bug on Mac OS X (resolution of simlink to OCRmyPDF.sh script) (thanks to jbarlow83)
-- Check if the input pdf file exists before to continue
-
-Tested with
------------
-
-- Operating system: FreeBSD 9.2
-- Dependencies:
-   - parallel 20140822
-   - poppler-utils 0.24.5
-   - ImageMagick 6.8.9-4 2014-09-17
-   - Unpaper 0.3
-   - tesseract 3.02.02
-   - Python 2.7.8
-   - ghostcript (gs): 9.06
-   - java: openjdk version "1.7.0_65"
-
-v2.1-stable (2014-09-20):
-=======
-
-New features
-------------
-
-- None
-
-Changes
--------
-
-- None
-
-Fixes
------
-
-- Allow execution via simlink
-- Add support for tesseract 3.03
-- Add support for newer version of reportlab
-- Lowered minimum version of gnu parallel
-- Various typo
-
-Tested with
------------
-
-- Operating system: FreeBSD 9.1
-- Dependencies:
-   - parallel 20130222
-   - poppler-utils 0.22.2
-   - ImageMagick 6.8.0-7 2013-03-30
-   - Unpaper 0.3
-   - tesseract 3.02.02
-   - Python 2.7.3
-   - ghostcript (gs): 9.06
-   - java: openjdk version "1.7.0_17"
-
-v2.0-stable (2014-01-25):
-=======
-
-New features
-------------
-
-- Check if the language(s) passed using the -l option is supported by tesseract (fixes #60)
-
-Changes
--------
-
-- Allow OCRmyPDF to be used with tesseract 3.02.01, even though OCR might fail for few PDF file (see issue #28). Rationale: For some linux distribution, no newer version than tesseract 3.02.01 is available
-
-Fixes
------
-
-- More robust algorithm for checking the version of the installed tesseract package
-
-Tested with
------------
-
-- Operating system: FreeBSD 9.1
-- Dependencies:
-   - parallel 20130222
-   - poppler-utils 0.22.2
-   - ImageMagick 6.8.0-7 2013-03-30
-   - Unpaper 0.3
-   - tesseract 3.02.02
-   - Python 2.7.3
-   - ghostcript (gs): 9.06
-   - java: openjdk version "1.7.0_17"
-
-v2.0-rc2 (2014-01-16):
-=======
-   
-New features
-------------
-
-- None
-
-Changes
--------
-
-- Size reduction of final PDF file: (fixes #50)
-   - Support for monochrome (Black&White) images (massive size reduction in final PDF: >80%)
-   - Reduced size of grayscale images (by 13% on test PDF file)
-- Preventing fi, fl ligatures does not require anymore to pass an additional config file to tesseract using the -C option (fixes #58)
-- Location of temporary folder according to content of environment variable TMPDIR.
-- Dependency to pdftk removed
-- Check for compatible versions of dependencies: (fixes #51)
-   - parallel and tesseract
-   - python libraries reportlab and lxml
-
-Fixes
------
-
-- Improved portability with various shells (dash, bash, tcsh) and OS (FreeBSD, MAC OSX, Linux) (fixes #59)
-- Corrected bug in case the input PDF file contains a space character (fixes #48)
-- Prevent spurious error message in case there is no image in a PDF page
-- Prevent collision of temporary folder names (fixes #57)
-
-Tested with
------------
-
-- Operating system: FreeBSD 9.1
-- Dependencies:
-   - parallel 20130222
-   - poppler-utils 0.22.2
-   - ImageMagick 6.8.0-7 2013-03-30
-   - Unpaper 0.3
-   - tesseract 3.02.02
-   - Python 2.7.3
-   - ghostcript (gs): 9.06
-   - java: openjdk version "1.7.0_17"
-
-v2.0-rc1 (2014-01-07):
-====
-
-New features
-------------
-
-- Huge performance improvement on machines having multiple CPU/cores (processing of several pages concurrently) (fixes #18)
-- By default prevent from processing a PDF file already containing fonts (i.e. text)(it can be overridden with the -f flag) (fixes #16)
-- Warn if the resolution is too low to get reasonable OCR results (fixes #37)
-- New option (-o) to perform automatic oversampling if the image resolution is too low. This can improve OCR results.
-- Warn if using a tesseract version older than v3.02.02 (as older versions are known to produce invalid output) (fixes #41)
-- Echo version of the installed dependencies (e.g. tesseract) in debug mode in order to ease support (fixes #35)
-- Echo the arguments passed to the script in debug mode to ease support
-
-Changes
--------
-
-- In debug mode: The debug page is now placed after the respective "normal" page
-- Reduced disk space usage in temporary folder if -d (deskew) or -c (cleanup) options are not selected
-- New file src/config.sh containing various configuration parameters
-- Documentation of the tesseract config file "tess-cfg/no_ligature" improved 
-- Improved consistency of the temporary file names
-
-Fixes
------
-
-- Improved robustness:
-   - in case vertical resolution differs from horizontal resolution (fixes #38)
-   - in case a PDF page contains more than one image (fixes #36)
-- Fix a problem occurring if python 3 is the standard interpreter (fixes #33)
-- Fix a problem occurring if the input PDF file contains special characters like "#" (fixes #34)
-
-Tested with
------------
-
-- Operating system: FreeBSD 9.1
-- Dependencies:
-   - parallel 20130222
-   - poppler-utils 0.22.2
-   - ImageMagick 6.8.0-7 2013-03-30
-   - Unpaper 0.3
-   - tesseract 3.02.02
-   - Python 2.7.3
-   - pdftk 1.45
-   - ghostcript (gs): 9.06
-   - java: openjdk version "1.7.0_17"
-
 v1.1-stable (2014-01-06):
 ====
 
@@ -226,7 +34,7 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghostcript (gs): 9.06
+   - ghoscript (gs): 9.06
    - java: openjdk version "1.7.0_17"
    
 v1.0-stable (2013-05-06):
@@ -263,7 +71,7 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghostcript (gs): 9.06
+   - ghoscript (gs): 9.06
    - java: openjdk version "1.7.0_17"
 
 v1.0-rc2 (2013-04-29):
@@ -299,7 +107,7 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghostcript (gs): 9.06
+   - ghoscript (gs): 9.06
    - java: openjdk version "1.7.0_17"
 
 v1.0-rc1 (2013-04-26):
@@ -331,5 +139,5 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghostcript (gs): 9.06
+   - ghoscript (gs): 9.06
    - java: openjdk version "1.7.0_17"

jhove/README

@@ -3,7 +3,7 @@ Copyright 2003-2012 by JSTOR and the President and Fellows of Harvard College
 JHOVE is made available under the GNU Lesser General Public License (LGPL;
 see the file LICENSE for details)
 
-Rev. 1.11, 2013-09-29
+Rev. 1.9, 2012-12-17
 
 JHOVE (the JSTOR/Harvard Object Validation Environment, pronounced "jhove")
 is an extensible software framework for performing format identification,

jhove/RELEASENOTES

@@ -6,85 +6,6 @@ see the file LICENSE for details)
 
 Versions 1.7 and beyond of JHOVE are no longer under the control of Harvard.
 
-RELEASE NOTES FOR JHOVE 1.11
-
-GENERAL
-
-1. I've added lots of logging code. Calls at the FINE level and lower 
-   don't show up no matter what I do, so I've put them at the INFO level.
-   The level is set in JhoveBase.java.
-
-2. All .bat and _bat.tmpl files now have CR-LF line endings. That is, they
-   do in the gzip and zip archives you download. I'm not sure how
-   SourceForge will treat files that you download individually,
-   but hopefully it will have the sense to keep CR-LF when downloading
-   to a Windows system. 
-
-3. All .bat files now assume JHOVE_HOME is the directory from which they're
-   run. They no longer try to set JAVA_HOME (which was still stuck in
-   Java 1.4 and probably wasn't working for many people), instead assuming
-   that the JAVA command is available on the command line.
-
-4. All javac commands in build.xml files now specify source=1.5 for
-   compatibility with more recent compilers.
-  
-5. gdumpwin.bat is deleted. It's redundant with gdump.bat and has bugs
-   of its own.
-   
-PDF MODULE
-
-1. Fix to PDF module, submitted by willp-bl, may reduce tendency 
-   to run out of heap space on some files.
-
-RELEASE NOTES FOR JHOVE 1.10
-
-GENERAL
-
-1. The amount of logging code has been increased, mostly at the
-   DEBUG level.
-   
-2. Further work on generics in Java code.
-
-3. JhoveView now checks for Java 1.5. Was previously allowing 1.4 even
-   though it wouldn't work.
-
-HTML MODULE
-
-1. XHTML files are processed by the HTML module, which invokes the XML
-   modules. In this case, the XML module doesn't have the parameters
-   specified in the JHOVE configuration file and so won't use local
-   copies of schemas. Starting with this version, the parameters of
-   the HTML module are passed to the XML module when invoking it.
-   However, this doesn't work properly (in either module) for a DTD
-   that invokes additional DTDs by relative URLs. Such DTDs should
-   be edited to use only absolute URLs.
-   
-PDF MODULE
-
-1. Failure to get a page object number wasn't being handled cleanly,
-   resulting in a report of an invalid document without an error message
-   to explain it (SourceForge bug 49). This has been fixed.
-
-2. The PDF module unnecessarily uses huge amounts of memory to build
-   complex structure trees, when it doesn't need to keep the whole 
-   tree in memory to validate it. In the new version, it uses memory
-   more economically. This should result in the successful processing
-   of some PDF files that ran out of memory or took hours to process before.
-   
-3. If an annotation isn't a dictionary object, report that explicitly.
-   This happens with some otherwise good files; I can't find any warrant
-   for it in the PDF spec.
-
-4. Some efficiency improvements to PDF parser. Increased buffer size from 4K 
-   to 64K. Made Parser.collapseObjectVector more efficient. Parser now
-   returns pseudo-objects for array and dictionary end instead of throwing
-   an exception.
-
-5. Minor cleanup of error reporting.
-
-6. If an object uses a compression scheme which JHOVE can't deal with, JHOVE
-   will try to give a specific error message.
-
 RELEASE NOTES FOR JHOVE 1.9
 
 GENERAL

jhove/conf/jhove-byteoffset=true.conf

@@ -3,7 +3,7 @@
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
  xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
+                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.4/jhoveConfig.xsd">
  <jhoveHome>/users/stephen/projects/jhove</jhoveHome>
  <defaultEncoding>utf-8</defaultEncoding>
  <tempDirectory>/var/tmp</tempDirectory>

jhove/conf/jhove-withTextMD.conf

@@ -3,7 +3,7 @@
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
  xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
+                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.4/jhoveConfig.xsd">
  <jhoveHome>/users/stephen/projects/jhove</jhoveHome>
  <defaultEncoding>utf-8</defaultEncoding>
  <tempDirectory>/var/tmp</tempDirectory>

jhove/conf/jhove.conf

@@ -3,7 +3,7 @@
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
  xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
+                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.4/jhoveConfig.xsd">
  <jhoveHome>./jhove/</jhoveHome>
  <defaultEncoding>utf-8</defaultEncoding>
  <tempDirectory>/var/tmp</tempDirectory>

jhove/jhove_bat.tmpl

@@ -35,6 +35,9 @@ REM EXTRA_JARS Extra jar files to add to CLASSPATH
 REM SET JHOVE_HOME="C:\Program Files\jhove"
 SET JHOVE_HOME="[your directory path]\jhove"
 
+SET JAVA_HOME="C:\Program Files\java\j2re1.4.1_02"
+SET JAVA=%JAVA_HOME%\bin\java
+
 SET EXTRA_JARS=
 
 REM NOTE: Nothing below this line should be edited
@@ -57,4 +60,4 @@ IF "%1"=="" GOTO LOOP
 :LOOP
 
 REM Set the CLASSPATH and invoke the Java loader
-JAVA -classpath %CP% Jhove %ARGS%
+%JAVA% -classpath %CP% Jhove %ARGS%

src/config.sh

@@ -1,32 +0,0 @@
-#####################################################################################
-# The following parameters might be changed by the user
-#####################################################################################
-
-DEFAULT_DPI=300				# dpi value used as fall back if the page dpi cannot be determined
-
-#####################################################################################
-# Do NOT change the following parameters
-#####################################################################################
-
-TOOLNAME="OCRmyPDF"
-VERSION="v2.2-stable"
-
-# possible exit codes
-EXIT_BAD_ARGS="1"
-EXIT_BAD_INPUT_FILE="2"
-EXIT_MISSING_DEPENDENCY="3"
-EXIT_INVALID_OUTPUT_PDFA="4"
-EXIT_FILE_ACCESS_ERROR="5"
-EXIT_OTHER_ERROR="15"
-
-# possible log levels
-LOG_ERR="0"				# only error messages
-LOG_WARN="1"				# error messages and warnings
-LOG_INFO="2"				# error messages, warnings and some infos
-LOG_DEBUG="3"				# debug level logging
-
-# various paths
-SRC="./src"				# location of the source folder (except source of external tools like jhove)
-OCR_PAGE="$SRC/ocrPage.sh"		# path to the script aimed at OCRing one page
-JHOVE="./jhove/bin/JhoveApp.jar"	# java SW for validating the final PDF/A
-JHOVE_CFG="./jhove/conf/jhove.conf"	# location of the jhove config file

src/hocrTransform.py

@@ -1,7 +1,5 @@
-#!/usr/local/bin/python2
-# coding: utf-8
 ##############################################################################
-# Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
+# Copyright (c) 2013: fritz-hh from Github (https://github.com/fritz-hh)
 #
 # Copyright (c) 2010: Jonathan Brinley from Github (https://github.com/jbrinley/HocrConverter)
 # Initial version by Jonathan Brinley, jonathanbrinley@gmail.com
@@ -9,11 +7,9 @@
 from reportlab.pdfgen.canvas import Canvas
 from reportlab.lib.units import inch
 from lxml import etree as ElementTree
-from PIL import Image
+import Image, re, sys
-import re, sys
 import argparse
 
-
 class hocrTransform():
 	"""
 	A class for converting documents from the hOCR format.
@@ -34,7 +30,6 @@ class hocrTransform():
 			self.xmlns = matches.group(1)
 			
 		# get dimension in pt (not pixel!!!!) of the OCRed image
-		self.width, self.height = None, None
 		for div in self.hocr.findall(".//%sdiv[@class='ocr_page']"%(self.xmlns)):
 			coords = self.element_coordinates(div)
 			self.width = self.px2pt(coords[2]-coords[0])
@@ -43,7 +38,7 @@ class hocrTransform():
 			
 		# no width and heigh definition in the ocr_image element of the hocr file
 		if self.width is None:
-			print("No page dimension found in the hocr file")
+			print "No page dimension found in the hocr file"
 			sys.exit(1)
 
 	def __str__(self):
@@ -90,17 +85,6 @@ class hocrTransform():
 		"""
 		return float(pxl)/self.dpi*inch
 			
-	def replace_unsupported_chars(self, str):
-		"""
-		Given an input string, returns the corresponding string that:
-		- is available in the helvetica facetype
-		- does not contain any ligature (to allow easy search in the PDF file)
-		"""		
-		# The 'u' before the character to replace indicates that it is a unicode character
-		str=str.replace(u"fl","fl")
-		str=str.replace(u"fi","fi")
-		return str
-		
 	def to_pdf(self, outFileName, imageFileName, showBoundingboxes, fontname="Helvetica"):
 		"""
 		Creates a PDF file with an image superimposed on top of the text.
@@ -147,9 +131,6 @@ class hocrTransform():
 		for elem in self.hocr.findall(".//%sspan[@class='%s']" % (self.xmlns, elemclass)):
 
 			elemtxt=self._get_element_text(elem).rstrip()
-			
-			elemtxt=self.replace_unsupported_chars(elemtxt)
-			
 			if len(elemtxt) == 0:
 				continue
 
@@ -186,7 +167,6 @@ class hocrTransform():
 		pdf.showPage()
 		pdf.save()
   
-
 if __name__ == "__main__":
 	parser = argparse.ArgumentParser(description='Convert hocr file to PDF')
 	parser.add_argument('-b', '--boundingboxes', action="store_true", default=False, help='Show bounding boxes borders')

src/ocrPage.sh

@@ -1,227 +0,0 @@
-#!/bin/sh
-##############################################################################
-# Script aimed at OCRing a single page of a PDF file
-#
-# Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
-##############################################################################
-
-. "./src/config.sh"
-
-
-# Initialization of variables passed by arguments
-FILE_INPUT_PDF="$1"			# PDF file containing the page to be OCRed
-PAGE_INFO="$2"				# Various characteristics of the page to be OCRed
-NUM_PAGES="$3"				# Total number of page of the PDF file (required for logging)
-TMP_FLD="$4"				# Folder where the temporary files should be placed
-VERBOSITY="$5"				# Requested verbosity
-LAN="$6"				# Language of the file to be OCRed
-KEEP_TMP="$7"				# Keep the temporary files after processing (helpful for debugging)
-PREPROCESS_DESKEW="$8"			# Deskew the page to be OCRed
-PREPROCESS_CLEAN="$9"			# Clean the page to be OCRed
-PREPROCESS_CLEANTOPDF="${10}"		# Put the cleaned paged in the OCRed PDF
-OVERSAMPLING_DPI="${11}"		# Oversampling resolution in dpi
-PDF_NOIMG="${12}"			# Request to generate also a PDF page containing only the OCRed text but no image (helpful for debugging) 
-TESS_CFG_FILES="${13}"			# Specific configuration files to be used by Tesseract during OCRing
-FORCE_OCR="${14}"			# Force to OCR, even if the page already contains fonts
-
-
-
-################################## 
-# Detect the characteristics of the embedded image for 
-# the page number provided as parameter
-#
-# Param 1: page number
-# Param 2: PDF page width in pt
-# Param 3: PDF page height in pt
-# Param 4: temporary file path (Path of the file in which the output should be written)
-# Output:  A file containing the characteristics of the embedded image. File structure:
-#          DPI=<dpi>
-#          COLOR_SPACE=<colorspace>
-#          DEPTH=<colordepth>
-# Returns:
-#       - 0: if no error occurs
-#       - 1: in case the page already contains fonts (which should be the case for PDF generated from scanned pages) 
-#       - 2: in case the page contains more than one image
-##################################
-getImgInfo() {
-	local page widthPDF heightPDF curImgInfo nbImg curImg propCurImg widthCurImg heightCurImg colorspaceCurImg depthCurImg dpi
-
-	# page number
-	page="$1"
-	# width / height of PDF page (in pt)
-	widthPDF="$2"
-	heightPDF="$3"
-	# path of the file in which the output should be written
-	curImgInfo="$4"
-
-	
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Size ${heightPDF}x${widthPDF} (h*w in pt)"
-	
-	
-	# check if the page already contains fonts (which should not be the case for PDF based on scanned files
-	[ `pdffonts -f $page -l $page "${FILE_INPUT_PDF}" | wc -l` -gt 2 ] && echo "Page $page: Page already contains font data !!!" && return 1
-
-	
-	# extract raw image from pdf file to compute resolution
-	# unfortunately this image can have another orientation than in the pdf...
-	# so we will have to extract it again later using pdftoppm
-	pdfimages -f $page -l $page -j "$FILE_INPUT_PDF" "$curOrigImg" 1>&2	
-	# count number of extracted images 
-	nbImg=$((`ls -1 "$curOrigImg"* 2>/dev/null | wc -l`))
-	if [ $nbImg -ne "1" ]; then
-		[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Expecting exactly 1 image covering the whole page (found $nbImg). Cannot compute dpi value."
-		return 2
-	fi
-	# Get characteristics of the extracted image
-	curImg=`ls -1 "$curOrigImg"* 2>/dev/null`
-	propCurImg=`identify -format "%w %h %[colorspace] %[depth]" "$curImg"`
-	widthCurImg=`echo "$propCurImg" | cut -f1 -d" "`
-	heightCurImg=`echo "$propCurImg" | cut -f2 -d" "`
-	colorspaceCurImg=`echo "$propCurImg" | cut -f3 -d" "`
-	depthCurImg=`echo "$propCurImg" | cut -f4 -d" "`
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Size ${heightCurImg}x${widthCurImg} (in pixel)"	
-
-	# compute the resolution of the image (making the assumption that x & y resolution are equal)
-	# and round it to the nearest integer
-	dpi=`echo "scale=5;sqrt($widthCurImg*72*$heightCurImg*72/$widthPDF/$heightPDF)+0.5" | bc`
-	dpi=`echo "scale=0;$dpi/1" | bc`
-	
-	# save the image characteristics
-	echo "DPI=$dpi" > "$curImgInfo"
-	echo "COLOR_SPACE=$colorspaceCurImg" >> "$curImgInfo"
-	echo "DEPTH=$depthCurImg" >> "$curImgInfo"
-	
-	return 0
-}
-
-
-page=`echo $PAGE_INFO | cut -f1 -d" "`
-[ $VERBOSITY -ge $LOG_INFO ] && echo "Processing page $page / $NUM_PAGES"
-
-# get width / height of PDF page (in pt)
-widthPDF=`echo $PAGE_INFO | cut -f2 -d" "`
-heightPDF=`echo $PAGE_INFO | cut -f3 -d" "`
-
-# create the name of the required temporary files
-curOrigImg="$TMP_FLD/${page}.orig-img"				# original image available in the current PDF page 
-								# (the image file may have a different orientation than in the pdf file)
-curHocr="$TMP_FLD/${page}.hocr"					# hocr file to be generated by the OCR SW for the current page
-curOCRedPDF="$TMP_FLD/${page}.ocred.pdf"			# PDF file containing the image + the OCRed text for the current page
-curOCRedPDFDebug="$TMP_FLD/${page}.ocred.todebug.pdf"		# PDF file containing data required to find out if OCR worked correctly
-curImgInfo="$TMP_FLD/${page}.orig-img-info.txt"			# Detected characteristics of the embedded image
-
-
-# auto-detect the characteristics of the embedded image
-depthCurImg="8"			# default color depth
-colorspaceCurImg="sRGB"		# default color space
-dpi=$DEFAULT_DPI		# default resolution
-
-getImgInfo "$page" "$widthPDF" "$heightPDF" "$curImgInfo"
-ret_code="$?"
-
-# in case the page contains text do not OCR, unless the FORCE_OCR flag is set
-if ([ "$ret_code" -eq "1" ] && [ "$FORCE_OCR" -eq "0" ]); then
-	echo "Page $page: Exiting... (Use the -f option to force OCRing, even though fonts are available in the input file)" && exit $EXIT_BAD_INPUT_FILE
-elif ([ "$ret_code" -eq "1" ] && [ "$FORCE_OCR" -eq "1" ]); then
-	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: OCRing anyway, assuming a default resolution of $dpi dpi"
-# in case the page contains more than one image, warn the user but go on with default parameters
-elif [ "$ret_code" -eq "2" ]; then
-	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Continuing anyway, assuming a default resolution of $dpi dpi"
-else
-	# read the image characteristics from the file
-	dpi=`cat "$curImgInfo" | grep "^DPI=" | cut -f2 -d"="`
-	colorspaceCurImg=`cat "$curImgInfo" | grep "^COLOR_SPACE=" | cut -f2 -d"="`
-	depthCurImg=`cat "$curImgInfo" | grep "^DEPTH=" | cut -f2 -d"="`
-fi
-
-# perform oversampling if the resolution is not sufficient to get good OCR results
-if [ "$dpi" -lt "$OVERSAMPLING_DPI" ]; then
-	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Low image resolution detected ($dpi dpi). Performing oversampling ($OVERSAMPLING_DPI dpi) to try to get better OCR results." 
-	dpi="$OVERSAMPLING_DPI"
-elif [ "$dpi" -lt "200" ]; then
-	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Low image resolution detected ($dpi dpi). If needed, please use the \"-o\" to try to get better OCR results." 
-fi
-	
-# Identify if page image should be saved as ppm (color), pgm (gray) or pbm (b&w)
-ext="ppm" 	# by default (color image) the extension of the extracted image is ppm
-opt=""		# by default (color image) no option as to be passed to pdftoppm
-if [ "$colorspaceCurImg" = "Gray" ] && [ "$depthCurImg" = "1" ]; then		# if monochrome (b&w)
-	ext="pbm"
-	opt="-mono"
-elif [ "$colorspaceCurImg" = "Gray" ]; then					# if gray
-	ext="pgm"
-	opt="-gray"
-fi
-curImgPixmap="$TMP_FLD/$page.$ext"
-curImgPixmapDeskewed="$TMP_FLD/$page.deskewed.$ext"
-curImgPixmapClean="$TMP_FLD/$page.cleaned.$ext"
-
-# extract current page as image with correct orientation and resolution
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Extracting image as $ext file (${dpi} dpi)"
-! pdftoppm -f $page -l $page -r $dpi $opt "$FILE_INPUT_PDF" > "$curImgPixmap" \
-	&& echo "Could not extract page $page as $ext from \"$FILE_INPUT_PDF\". Exiting..." && exit $EXIT_OTHER_ERROR
-
-# if requested deskew image (without changing its size in pixel)
-widthCurImg=$(($dpi*$widthPDF/72))
-heightCurImg=$(($dpi*$heightPDF/72))
-if [ "$PREPROCESS_DESKEW" -eq "1" ]; then
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Deskewing image"
-	! convert "$curImgPixmap" -deskew 40% -gravity center -extent ${widthCurImg}x${heightCurImg} "$curImgPixmapDeskewed" \
-		&& echo "Could not deskew \"$curImgPixmap\". Exiting..." && exit $EXIT_OTHER_ERROR
-else
-	ln -s `basename "$curImgPixmap"` "$curImgPixmapDeskewed"
-fi
-
-# if requested clean image with unpaper to get better OCR results
-if [ "$PREPROCESS_CLEAN" -eq "1" ]; then
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Cleaning image with unpaper"
-	! unpaper --dpi $dpi --mask-scan-size 100 \
-		--no-deskew --no-grayfilter --no-blackfilter --no-mask-center --no-border-align \
-		"$curImgPixmapDeskewed" "$curImgPixmapClean" 1> /dev/null \
-		&& echo "Could not clean \"$curImgPixmapDeskewed\". Exiting..." && exit $EXIT_OTHER_ERROR
-else
-	ln -s `basename "$curImgPixmapDeskewed"` "$curImgPixmapClean"
-fi
-
-# perform OCR
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Performing OCR"
-! tesseract -l "$LAN" "$curImgPixmapClean" "$curHocr" hocr $TESS_CFG_FILES 1> /dev/null 2> /dev/null \
-	&& echo "Could not OCR file \"$curImgPixmapClean\". Exiting..." && exit $EXIT_OTHER_ERROR
-# Tesseract names the output files differently in some distributions.
-if [ -e "$curHocr.html" ]; then
-	mv "$curHocr.html" "$curHocr"
-elif [ -e "$curHocr.hocr" ]; then
-	mv "$curHocr.hocr" "$curHocr"
-elif [ ! -e "$curHocr" ]; then
-	echo "\"$curHocr[.html|.hocr]\" not found. Exiting..." && exit $EXIT_OTHER_ERROR
-fi
-
-# embed text and image to new pdf file
-if [ "$PREPROCESS_CLEANTOPDF" -eq "1" ]; then
-	image4finalPDF="$curImgPixmapClean"
-else
-	image4finalPDF="$curImgPixmapDeskewed"	
-fi
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF"
-! python2 $SRC/hocrTransform.py -r $dpi -i "$image4finalPDF" "$curHocr" "$curOCRedPDF" \
-	&& echo "Could not create PDF file from \"$curHocr\". Exiting..." && exit $EXIT_OTHER_ERROR
-
-# if requested generate special debug PDF page with visible OCR text
-if [ $PDF_NOIMG -eq "1" ] ; then
-	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Embedding text in PDF (debug page)"
-	! python2 $SRC/hocrTransform.py -b -r $dpi "$curHocr" "$curOCRedPDFDebug" \
-		&& echo "Could not create PDF file from \"$curHocr\". Exiting..." && exit $EXIT_OTHER_ERROR	
-fi
-
-# delete temporary files created for the current page
-# to avoid using to much disk space in case of PDF files having many pages
-if [ $KEEP_TMP -eq 0 ]; then
-	rm -f "$curOrigImg"*
-	rm -f "$curHocr"
-	rm -f "$curImgPixmap"
-	rm -f "$curImgPixmapDeskewed"
-	rm -f "$curImgPixmapClean"
-	rm -f "$curImgInfo"
-fi
-
-exit 0

tess-cfg/README.md

@@ -0,0 +1,10 @@
+Note
+====
+
+The file(s) located in this folder are tesseract configuration files.
+(Tesseract configuration files enable to tune the behaviour of tesseract)
+
+If needed, these files should be copied to the "tessdata/configs" folder of your tesseract installation.
+
+To request OCRmyPDF.sh to use a configuration file, please use the -C option
+

tess-cfg/no_ligature

@@ -0,0 +1 @@
+tessedit_char_blacklist fifl