update release notes for v2.2-stable

Python does not map the expression to its return code automatically, so
this line returns success regardless of the reportlab version installed.
(I also realized that hasattr is superfluous).

.gitignore

@@ -1,2 +1,3 @@
 tmp/
-log/
+log/
+*.pyc

OCRmyPDF.sh

@@ -3,8 +3,12 @@
 # Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
 ##############################################################################
 
+# Determine real path of this script, following symlinks if present
+! command -v python2 > /dev/null && echo "Please install python v2.x. Exiting..." && exit 1
+BASEPATH="$(dirname $(python2 -c "import os; print os.path.realpath(\"$0\")"))"
+
 # Import required scripts
-. "`dirname $0`/src/config.sh"
+. "$BASEPATH/src/config.sh"
 
 # Set variables corresponding to the input parameters
 ARGUMENTS="$@"
@@ -34,18 +38,18 @@ Usage: OCRmyPDF.sh  [-h] [-v] [-g] [-k] [-d] [-c] [-i] [-o dpi] [-f] [-l languag
 -d : Deskew each page before performing OCR
 -c : Clean each page before performing OCR
 -i : Incorporate the cleaned image in the final PDF file (by default the original image	
-     image, or the deskewed image if the -d option is set, is incorporated)
+     image, or the deskewed image if the -d option is set)
 -o : If the resolution of an image is lower than dpi value provided as argument, provide the OCR engine with 
      an oversampled image having the latter dpi value. This can improve the OCR results but can lead to a larger output PDF file.
      (default: no oversampling performed)
 -f : Force to OCR the whole document, even if some page already contain font data 
-     (which should not be the case for PDF files built from scnanned images) 
+     (which should not be the case for PDF files built from scanned images) 
 -l : Set the language of the PDF file in order to improve OCR results (default "eng")
-     Any language supported by tesseract is supported.
+     Any language supported by tesseract is supported (Tesseract uses 3-character ISO 639-2 language codes)
+     Multiple languages may be specified, separated by '+' characters.
 -C : Pass an additional configuration file to the tesseract OCR engine.
      (this option can be used more than once)
      Note 1: The configuration file must be available in the "tessdata/configs" folder of your tesseract installation
-     Note 2: The folder "./tess-cfg" contains useful tesseract configuration files
 inputfile  : PDF file to be OCRed
 outputfile : The PDF/A file that will be generated 
 --------------------------------------------------------------------------------------
@@ -111,17 +115,17 @@ done
 # Remove the optional arguments parsed above.
 shift $((OPTIND-1))
 
-# Check if the number of mandatory parameters
+# Check if the number of mandatory parameters provided is as expected
-# provided is as expected
 if [ "$#" -ne "2" ]; then
 	echo "Exactly two mandatory argument shall be provided ($# arguments provided)"
 	usage
 	exit $EXIT_BAD_ARGS
 fi
 
-! absolutePath "$1" > /dev/null \
+[ ! -f "$1" ] \
-	&& echo "The folder in which the input file should be located does not exist. Exiting..." && exit $EXIT_BAD_ARGS
+	&& echo "The input file does not exist. Exiting..." && exit $EXIT_BAD_ARGS
 FILE_INPUT_PDF="`absolutePath "$1"`"
+
 ! absolutePath "$2" > /dev/null \
 	&& echo "The folder in which the output file should be generated does not exist. Exiting..." && exit $EXIT_BAD_ARGS
 FILE_OUTPUT_PDFA="`absolutePath "$2"`"
@@ -129,7 +133,7 @@ FILE_OUTPUT_PDFA="`absolutePath "$2"`"
 
 
 # set script path as working directory
-cd "`dirname $0`"
+cd "$BASEPATH"
 
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "$TOOLNAME version: $VERSION"
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Arguments: $ARGUMENTS"
@@ -141,20 +145,36 @@ cd "`dirname $0`"
 ! command -v pdfimages > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
 ! command -v pdftoppm > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
 ! command -v pdffonts > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
-! command -v pdftk > /dev/null && echo "Please install pdftk. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
 [ $PREPROCESS_CLEAN -eq 1 ] && ! command -v unpaper > /dev/null && echo "Please install unpaper. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
 ! command -v tesseract > /dev/null && echo "Please install tesseract and tesseract-data. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
-! command -v python2 > /dev/null && echo "Please install python v2.x, and the python libraries: reportlab, lxml. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! python2 -c 'import lxml' 2>/dev/null && echo "Please install the python library lxml. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
-! command -v gs > /dev/null && echo "Please install ghostcript. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! python2 -c 'import sys, reportlab; (getattr(reportlab, "Version", "0.0") >= "3.0") or sys.exit(1)' 2>/dev/null \
+	&& echo "Please install the python library reportlab. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+
+! command -v gs > /dev/null && echo "Please install ghostscript. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
 ! command -v java > /dev/null && echo "Please install java. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
 
 
-# ensure tesseract v3.02.02 or newer is installed
+# ensure the right tesseract version is installed
 # older versions are known to produce malformed hocr output and should not be used
-tessversion=`tesseract -v 2>&1 | grep "tesseract"`
+# Even 3.02.01 fails in few cases (see issue #28). I decided to allow this version anyway because
-if [ $VERBOSITY -ge $LOG_WARN -a $((`echo ${tessversion} | sed s/[^0-9]//g`-30202)) -lt 0 ]; then
+# 3.02.02 is not yet available for some widespread linux distributions
-	echo "Warning: Please use tesseract 3.02.02 or newer. Older versions are known to produce invalid hocr output (installed version: ${tessversion})"
+reqtessversion="3.02.01"
-fi
+tessversion=`tesseract -v 2>&1 | grep "tesseract" | sed s/[^0-9.]//g`
+tesstooold=$(echo "`echo $tessversion | sed s/[.]//2`-`echo $reqtessversion | sed s/[.]//2` < 0" | bc)
+[ "$tesstooold" -eq "1" ] \
+	&& echo "Please install tesseract ${reqtessversion} or newer (currently installed version is ${tessversion})" && exit $EXIT_MISSING_DEPENDENCY
+
+# ensure the right GNU parallel version is installed
+# older version do not support -q flag (required to escape special characters)
+reqparallelversion="20121122"
+parallelversion=`parallel --minversion 0`
+! parallel --minversion "$reqparallelversion" > /dev/null \
+	&& echo "Please install GNU parallel ${reqparallelversion} or newer (currently installed version is ${parallelversion})" && exit $EXIT_MISSING_DEPENDENCY
+
+# ensure pdftoppm is provided by poppler-utils, not the older xpdf version
+! pdftoppm -v 2>&1 | grep -q 'Poppler' && echo "Please remove xpdf and install poppler-utils. Exiting..." && $EXIT_MISSING_DEPENDENCY
+
 
 
 # Display the version of the tools if log level is LOG_DEBUG
@@ -171,9 +191,6 @@ if [ $VERBOSITY -ge $LOG_DEBUG ]; then
 	pdftoppm -v
 	pdffonts -v
 	echo "--------------------------------"
-	echo "pdftk version:"
-	pdftk --version
-	echo "--------------------------------"
 	echo "unpaper version:"
 	unpaper --version
 	echo "--------------------------------"
@@ -193,24 +210,44 @@ fi
 
 
 
-# Initialize path to temporary files
+# check if the languages passed to tesseract are all supported
-today=$(date +"%Y%m%d_%H%M")
+for currentlan in `echo "$LAN" | sed 's/+/ /g'`; do
-fld=$(basename "$FILE_INPUT_PDF" | sed 's/[.][^.]*//')
+	if ! tesseract --list-langs 2>&1 | grep "^$currentlan\$" > /dev/null; then
-TMP_FLD="${TMP}/$today.filename.$fld"
+		echo "The language \"$currentlan\" is not supported by tesseract."
+		tesseract --list-langs 2>&1 | tr '\n' ' '; echo
+		echo "Exiting..."
+		exit $EXIT_BAD_ARGS
+	fi
+done
+
+
+
+# Initialize path to temporary files using mktemp
+# Goal: save tmp file in a sub-folder of the $TMPDIR environment variable (or in "/tmp" if unset)
+# Unfortunately, Linux mktemp is not compatible with FreeBSD/OSX mktemp
+# Linux version requires no arg
+# FreeBSD requires '-t prefix' to be used so that $TMPDIR is taken into account
+# But in Linux '-t template' is handled differently than in FreeBSD
+# Therefore different calls must be used for Linux and for FreeBSD
+prefix="$(date +"%Y%m%d_%H%M").filename.$(basename "$FILE_INPUT_PDF" | sed 's/[.][^.]*$//')"	# prefix made of date, time and pdf file name without extension
+TMP_FLD=`mktemp -d 2>/dev/null || mktemp -d -t "${prefix}" 2>/dev/null`				# try Linux syntax first, if it fails try FreeBSD/OSX			
+if [ $? -ne 0 ]; then
+	if [ -z "$TMPDIR" ]; then
+		echo "Could not create folder for temporary files. Please ensure you have sufficient right and \"/tmp\" exists"
+	else
+		echo "Could not create folder for temporary files. Please ensure you have sufficient right and \"$TMPDIR\" exists"
+	fi
+	exit $EXIT_FILE_ACCESS_ERROR
+fi
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Created temporary folder: \"$TMP_FLD\""
+
 FILE_TMP="${TMP_FLD}/tmp.txt"						# temporary file with a very short lifetime (may be used for several things)
 FILE_PAGES_INFO="${TMP_FLD}/pages-info.txt"				# for each page: page #; width in pt; height in pt
-FILE_OUTPUT_PDF_CAT="${TMP_FLD}/ocred.pdf"				# concatenated OCRed PDF files
 FILE_VALIDATION_LOG="${TMP_FLD}/pdf_validation.log"			# log file containing the results of the validation of the PDF/A file
 
-# Create tmp folder
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Creating temporary folder: \"$TMP_FLD\""
-rm -r -f "${TMP_FLD}"
-mkdir -p "${TMP_FLD}"
 
 
-
+# get the size of each pdf page (width / height) in pt (i.e. inch/72)
-
-# get the size of each pdf page (width / height) in pt (inch*72)
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Input file: Extracting size of each page (in pt)"
 ! identify -format "%w %h\n" "$FILE_INPUT_PDF" > "$FILE_TMP" \
 	&& echo "Could not get size of PDF pages. Exiting..." && exit $EXIT_BAD_INPUT_FILE
@@ -219,27 +256,23 @@ sed '/^$/d' "$FILE_TMP" | awk '{printf "%04d %s\n", NR, $0}' > "$FILE_PAGES_INFO
 numpages=`tail -n 1 "$FILE_PAGES_INFO" | cut -f1 -d" "`
 
 # process each page of the input pdf file
-parallel -q -k --halt-on-error 1 "$OCR_PAGE" "$FILE_INPUT_PDF" "{}" "$numpages" "$TMP_FLD" \
+parallel --gnu -q -k --halt-on-error 1 "$OCR_PAGE" "$FILE_INPUT_PDF" "{}" "$numpages" "$TMP_FLD" \
 	"$VERBOSITY" "$LAN" "$KEEP_TMP" "$PREPROCESS_DESKEW" "$PREPROCESS_CLEAN" "$PREPROCESS_CLEANTOPDF" "$OVERSAMPLING_DPI" \
 	"$PDF_NOIMG" "$TESS_CFG_FILES" "$FORCE_OCR" < "$FILE_PAGES_INFO"
 ret_code="$?"
 [ $ret_code -ne 0 ] && exit $ret_code 
 
-# concatenate all pages
+# concatenate all pages and convert the pdf file to match PDF/A format
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Concatenating all pages"
+[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Concatenating all pages to the final PDF/A file" 
-! pdftk "${TMP_FLD}/"*ocred*.pdf cat output "$FILE_OUTPUT_PDF_CAT" \
-	&& echo "Could not concatenate individual PDF pages (\"${TMP_FLD}/*-ocred.pdf\") to one file. Exiting..." && exit $EXIT_OTHER_ERROR
-
-# convert the pdf file to match PDF/A format
-[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Converting to PDF/A" 
 ! gs -dQUIET -dPDFA -dBATCH -dNOPAUSE -dUseCIEColor \
 	-sProcessColorModel=DeviceCMYK -sDEVICE=pdfwrite -sPDFACompatibilityPolicy=2 \
-	-sOutputFile="$FILE_OUTPUT_PDFA" "$FILE_OUTPUT_PDF_CAT" 1> /dev/null 2> /dev/null \
+	-sOutputFile="$FILE_OUTPUT_PDFA" "${TMP_FLD}/"*ocred*.pdf 1> /dev/null 2> /dev/null \
-	&& echo "Could not convert PDF file \"$FILE_OUTPUT_PDF_CAT\" to PDF/A. Exiting..." && exit $EXIT_OTHER_ERROR
+	&& echo "Could not concatenate all pages to the final PDF/A file. Exiting..." && exit $EXIT_OTHER_ERROR
 
 # validate generated pdf file (compliance to PDF/A)
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Checking compliance to PDF/A standard" 
-java -jar "$JHOVE" -c "$JHOVE_CFG" -m PDF-hul "$FILE_OUTPUT_PDFA" > "$FILE_VALIDATION_LOG"
+! java -jar "$JHOVE" -c "$JHOVE_CFG" -m PDF-hul "$FILE_OUTPUT_PDFA" 2> /dev/null 1> "$FILE_VALIDATION_LOG" \
+	&& echo "Unexpected error while checking compliance to PDF/A file. Exiting..." && exit $EXIT_OTHER_ERROR
 grep -i "Status|Message" "$FILE_VALIDATION_LOG" # summary of the validation
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "The full validation log is available here: \"$FILE_VALIDATION_LOG\""
 # check the validation results
@@ -265,4 +298,4 @@ END=`date +%s`
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Script took $(($END-$START)) seconds"
 
 
-[ $pdf_valid -ne 1 ] && exit $EXIT_INVALID_OUPUT_PDFA || exit 0
+[ $pdf_valid -ne 1 ] && exit $EXIT_INVALID_OUTPUT_PDFA || exit 0

README.md

@@ -11,12 +11,14 @@ Main features
 - Generates a searchable PDF/A file from a PDF file containing only images
 - Places OCRed text accurately below the image to ease copy / paste
 - Keeps the exact resolution of the original embedded images
-    - or if requested oversample the images before OCRing so as to get better results 
+    - or if requested oversamples the images before OCRing so as to get better results 
 - If requested deskews and / or clean the image before performing OCR
 - Validates the generated file against the PDF/A specification using jhove
 - Provides debug mode to enable easy verification of the OCR results
 - Processes several pages in parallel if more than one CPU core is available
 
+For details: please consult the release notes
+
 Motivation
 ----------
 
@@ -35,13 +37,14 @@ I found many, but none of them were really satisfying.
 Install
 -------
 
-Download OCRmyPDF here: https://github.com/fritz-hh/OCRmyPDF/tags
+Download OCRmyPDF here: https://github.com/fritz-hh/OCRmyPDF/releases
 
 Copy the file in onto your linux/unix machine and extract it.
 
 Run: "sh ./OCRmyPDF.sh -h" to get the script usage
 
-If not yet installed, the script will notify you about dependencies that need to be installed
+If not yet installed, the script will notify you about dependencies that need to be installed.
+The script requires specific versions of the dependencies. Older version than the ones mentioned in the release notes are likely not to be compatible to OCRmyPDF.
 
 Support
 -------
@@ -49,7 +52,7 @@ Support
 In case you detect an issue, please:
 
 - Check if your issue is already known
-- if no problem report exists on github, please create one here: https://github.com/fritz-hh/OCRmyPDF/issues
+- If no problem report exists on github, please create one here: https://github.com/fritz-hh/OCRmyPDF/issues
 - Describe your problem thoroughly
 - Append the console output of the script when running the debug mode (-g option)
 - If possible provide your input PDF file as well as the content of the temporary folder (using a file sharing service like www.file-upload.net)
@@ -57,4 +60,10 @@ In case you detect an issue, please:
 Press & Media
 -------------
 
-- c't 1-2014, page 59: Detailed presentation of OCRmyPDF v1.0 in the leading german IT magazine (c't)
+- c't 1-2014, page 59: Detailed presentation of OCRmyPDF v1.0 in the leading german IT magazine c't (http://www.heise.de/ct/inhalt/2014/1/58/) 
+- heise Open Source, 09/2014: Texterkennung mit OCRmyPDF (http://www.heise.de/-2356670)
+
+Disclaimer
+----------
+
+The software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

RELEASE_NOTES.md

@@ -5,6 +5,151 @@ Please always read this file before installing the package
 
 Download software here: https://github.com/fritz-hh/OCRmyPDF/tags
 
+v2.2-stable (2014-09-29):
+=======
+
+New features
+------------
+
+- None
+
+Changes
+-------
+
+- Update to jhove v1.11
+- Request the python library reportlab v3.0 or newer (So that we could remove a patch to the previous version of reportlab leading to issues for some users)
+
+Fixes
+-----
+
+- Fix bug on Mac OS X (resolution of simlink to OCRmyPDF.sh script) (thanks to jbarlow83)
+- Check if the input pdf file exists before to continue
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.2
+- Dependencies:
+   - parallel 20140822
+   - poppler-utils 0.24.5
+   - ImageMagick 6.8.9-4 2014-09-17
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.8
+   - ghostcript (gs): 9.06
+   - java: openjdk version "1.7.0_65"
+
+v2.1-stable (2014-09-20):
+=======
+
+New features
+------------
+
+- None
+
+Changes
+-------
+
+- None
+
+Fixes
+-----
+
+- Allow execution via simlink
+- Add support for tesseract 3.03
+- Add support for newer version of reportlab
+- Lowered minimum version of gnu parallel
+- Various typo
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.1
+- Dependencies:
+   - parallel 20130222
+   - poppler-utils 0.22.2
+   - ImageMagick 6.8.0-7 2013-03-30
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.3
+   - ghostcript (gs): 9.06
+   - java: openjdk version "1.7.0_17"
+
+v2.0-stable (2014-01-25):
+=======
+
+New features
+------------
+
+- Check if the language(s) passed using the -l option is supported by tesseract (fixes #60)
+
+Changes
+-------
+
+- Allow OCRmyPDF to be used with tesseract 3.02.01, even though OCR might fail for few PDF file (see issue #28). Rationale: For some linux distribution, no newer version than tesseract 3.02.01 is available
+
+Fixes
+-----
+
+- More robust algorithm for checking the version of the installed tesseract package
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.1
+- Dependencies:
+   - parallel 20130222
+   - poppler-utils 0.22.2
+   - ImageMagick 6.8.0-7 2013-03-30
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.3
+   - ghostcript (gs): 9.06
+   - java: openjdk version "1.7.0_17"
+
+v2.0-rc2 (2014-01-16):
+=======
+   
+New features
+------------
+
+- None
+
+Changes
+-------
+
+- Size reduction of final PDF file: (fixes #50)
+   - Support for monochrome (Black&White) images (massive size reduction in final PDF: >80%)
+   - Reduced size of grayscale images (by 13% on test PDF file)
+- Preventing fi, fl ligatures does not require anymore to pass an additional config file to tesseract using the -C option (fixes #58)
+- Location of temporary folder according to content of environment variable TMPDIR.
+- Dependency to pdftk removed
+- Check for compatible versions of dependencies: (fixes #51)
+   - parallel and tesseract
+   - python libraries reportlab and lxml
+
+Fixes
+-----
+
+- Improved portability with various shells (dash, bash, tcsh) and OS (FreeBSD, MAC OSX, Linux) (fixes #59)
+- Corrected bug in case the input PDF file contains a space character (fixes #48)
+- Prevent spurious error message in case there is no image in a PDF page
+- Prevent collision of temporary folder names (fixes #57)
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.1
+- Dependencies:
+   - parallel 20130222
+   - poppler-utils 0.22.2
+   - ImageMagick 6.8.0-7 2013-03-30
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.3
+   - ghostcript (gs): 9.06
+   - java: openjdk version "1.7.0_17"
+
 v2.0-rc1 (2014-01-07):
 ====
 
@@ -23,7 +168,7 @@ Changes
 -------
 
 - In debug mode: The debug page is now placed after the respective "normal" page
-- Reduced disk space usage if -d (deskew) or -c (cleanup) options are not selected
+- Reduced disk space usage in temporary folder if -d (deskew) or -c (cleanup) options are not selected
 - New file src/config.sh containing various configuration parameters
 - Documentation of the tesseract config file "tess-cfg/no_ligature" improved 
 - Improved consistency of the temporary file names
@@ -42,13 +187,14 @@ Tested with
 
 - Operating system: FreeBSD 9.1
 - Dependencies:
+   - parallel 20130222
    - poppler-utils 0.22.2
    - ImageMagick 6.8.0-7 2013-03-30
    - Unpaper 0.3
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"
 
 v1.1-stable (2014-01-06):
@@ -80,7 +226,7 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"
 
 v1.0-stable (2013-05-06):
@@ -117,7 +263,7 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"
 
 v1.0-rc2 (2013-04-29):
@@ -153,7 +299,7 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"
 
 v1.0-rc1 (2013-04-26):
@@ -185,5 +331,5 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"

jhove/README

@@ -3,7 +3,7 @@ Copyright 2003-2012 by JSTOR and the President and Fellows of Harvard College
 JHOVE is made available under the GNU Lesser General Public License (LGPL;
 see the file LICENSE for details)
 
-Rev. 1.9, 2012-12-17
+Rev. 1.11, 2013-09-29
 
 JHOVE (the JSTOR/Harvard Object Validation Environment, pronounced "jhove")
 is an extensible software framework for performing format identification,

jhove/RELEASENOTES

@@ -6,6 +6,85 @@ see the file LICENSE for details)
 
 Versions 1.7 and beyond of JHOVE are no longer under the control of Harvard.
 
+RELEASE NOTES FOR JHOVE 1.11
+
+GENERAL
+
+1. I've added lots of logging code. Calls at the FINE level and lower 
+   don't show up no matter what I do, so I've put them at the INFO level.
+   The level is set in JhoveBase.java.
+
+2. All .bat and _bat.tmpl files now have CR-LF line endings. That is, they
+   do in the gzip and zip archives you download. I'm not sure how
+   SourceForge will treat files that you download individually,
+   but hopefully it will have the sense to keep CR-LF when downloading
+   to a Windows system. 
+
+3. All .bat files now assume JHOVE_HOME is the directory from which they're
+   run. They no longer try to set JAVA_HOME (which was still stuck in
+   Java 1.4 and probably wasn't working for many people), instead assuming
+   that the JAVA command is available on the command line.
+
+4. All javac commands in build.xml files now specify source=1.5 for
+   compatibility with more recent compilers.
+  
+5. gdumpwin.bat is deleted. It's redundant with gdump.bat and has bugs
+   of its own.
+   
+PDF MODULE
+
+1. Fix to PDF module, submitted by willp-bl, may reduce tendency 
+   to run out of heap space on some files.
+
+RELEASE NOTES FOR JHOVE 1.10
+
+GENERAL
+
+1. The amount of logging code has been increased, mostly at the
+   DEBUG level.
+   
+2. Further work on generics in Java code.
+
+3. JhoveView now checks for Java 1.5. Was previously allowing 1.4 even
+   though it wouldn't work.
+
+HTML MODULE
+
+1. XHTML files are processed by the HTML module, which invokes the XML
+   modules. In this case, the XML module doesn't have the parameters
+   specified in the JHOVE configuration file and so won't use local
+   copies of schemas. Starting with this version, the parameters of
+   the HTML module are passed to the XML module when invoking it.
+   However, this doesn't work properly (in either module) for a DTD
+   that invokes additional DTDs by relative URLs. Such DTDs should
+   be edited to use only absolute URLs.
+   
+PDF MODULE
+
+1. Failure to get a page object number wasn't being handled cleanly,
+   resulting in a report of an invalid document without an error message
+   to explain it (SourceForge bug 49). This has been fixed.
+
+2. The PDF module unnecessarily uses huge amounts of memory to build
+   complex structure trees, when it doesn't need to keep the whole 
+   tree in memory to validate it. In the new version, it uses memory
+   more economically. This should result in the successful processing
+   of some PDF files that ran out of memory or took hours to process before.
+   
+3. If an annotation isn't a dictionary object, report that explicitly.
+   This happens with some otherwise good files; I can't find any warrant
+   for it in the PDF spec.
+
+4. Some efficiency improvements to PDF parser. Increased buffer size from 4K 
+   to 64K. Made Parser.collapseObjectVector more efficient. Parser now
+   returns pseudo-objects for array and dictionary end instead of throwing
+   an exception.
+
+5. Minor cleanup of error reporting.
+
+6. If an object uses a compression scheme which JHOVE can't deal with, JHOVE
+   will try to give a specific error message.
+
 RELEASE NOTES FOR JHOVE 1.9
 
 GENERAL

jhove/conf/jhove-byteoffset=true.conf

@@ -3,7 +3,7 @@
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
  xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.4/jhoveConfig.xsd">
+                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
  <jhoveHome>/users/stephen/projects/jhove</jhoveHome>
  <defaultEncoding>utf-8</defaultEncoding>
  <tempDirectory>/var/tmp</tempDirectory>

jhove/conf/jhove-withTextMD.conf

@@ -3,7 +3,7 @@
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
  xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.4/jhoveConfig.xsd">
+                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
  <jhoveHome>/users/stephen/projects/jhove</jhoveHome>
  <defaultEncoding>utf-8</defaultEncoding>
  <tempDirectory>/var/tmp</tempDirectory>

jhove/conf/jhove.conf

@@ -3,7 +3,7 @@
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
  xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.4/jhoveConfig.xsd">
+                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
  <jhoveHome>./jhove/</jhoveHome>
  <defaultEncoding>utf-8</defaultEncoding>
  <tempDirectory>/var/tmp</tempDirectory>

jhove/jhove_bat.tmpl

@@ -35,9 +35,6 @@ REM EXTRA_JARS Extra jar files to add to CLASSPATH
 REM SET JHOVE_HOME="C:\Program Files\jhove"
 SET JHOVE_HOME="[your directory path]\jhove"
 
-SET JAVA_HOME="C:\Program Files\java\j2re1.4.1_02"
-SET JAVA=%JAVA_HOME%\bin\java
-
 SET EXTRA_JARS=
 
 REM NOTE: Nothing below this line should be edited
@@ -60,4 +57,4 @@ IF "%1"=="" GOTO LOOP
 :LOOP
 
 REM Set the CLASSPATH and invoke the Java loader
-%JAVA% -classpath %CP% Jhove %ARGS%
+JAVA -classpath %CP% Jhove %ARGS%

src/config.sh

@@ -1,11 +1,22 @@
+#####################################################################################
+# The following parameters might be changed by the user
+#####################################################################################
+
+DEFAULT_DPI=300				# dpi value used as fall back if the page dpi cannot be determined
+
+#####################################################################################
+# Do NOT change the following parameters
+#####################################################################################
+
 TOOLNAME="OCRmyPDF"
-VERSION="v2.0-rc1"
+VERSION="v2.2-stable"
 
 # possible exit codes
 EXIT_BAD_ARGS="1"
 EXIT_BAD_INPUT_FILE="2"
 EXIT_MISSING_DEPENDENCY="3"
-EXIT_INVALID_OUPUT_PDFA="4"
+EXIT_INVALID_OUTPUT_PDFA="4"
+EXIT_FILE_ACCESS_ERROR="5"
 EXIT_OTHER_ERROR="15"
 
 # possible log levels
@@ -16,10 +27,6 @@ LOG_DEBUG="3"				# debug level logging
 
 # various paths
 SRC="./src"				# location of the source folder (except source of external tools like jhove)
-TMP="./tmp"				# location of the temporary files (one sub-folder will be created per PDF file to be processed)
 OCR_PAGE="$SRC/ocrPage.sh"		# path to the script aimed at OCRing one page
 JHOVE="./jhove/bin/JhoveApp.jar"	# java SW for validating the final PDF/A
 JHOVE_CFG="./jhove/conf/jhove.conf"	# location of the jhove config file
-
-# other 
-DEFAULT_DPI=300				# dpi value used as fall back if the page dpi cannot be determined

src/hocrTransform.py

@@ -1,4 +1,5 @@
-#!/usr/bin/python
+#!/usr/local/bin/python2
+# coding: utf-8
 ##############################################################################
 # Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
 #
@@ -12,6 +13,7 @@ from PIL import Image
 import re, sys
 import argparse
 
+
 class hocrTransform():
 	"""
 	A class for converting documents from the hOCR format.
@@ -24,20 +26,21 @@ class hocrTransform():
 
 		self.hocr = ElementTree.ElementTree()
 		self.hocr.parse(hocrFileName)
- 
+
 		# if the hOCR file has a namespace, ElementTree requires its use to find elements
 		matches = re.match('({.*})html', self.hocr.getroot().tag)
 		self.xmlns = ''
 		if matches:
 			self.xmlns = matches.group(1)
-			
+
 		# get dimension in pt (not pixel!!!!) of the OCRed image
+		self.width, self.height = None, None
 		for div in self.hocr.findall(".//%sdiv[@class='ocr_page']"%(self.xmlns)):
 			coords = self.element_coordinates(div)
 			self.width = self.px2pt(coords[2]-coords[0])
 			self.height = self.px2pt(coords[3]-coords[1])
 			break # there shouldn't be more than one, and if there is, we don't want it
-			
+
 		# no width and heigh definition in the ocr_image element of the hocr file
 		if self.width is None:
 			print("No page dimension found in the hocr file")
@@ -54,7 +57,7 @@ class hocrTransform():
 			return self._get_element_text(body).encode('utf-8') # XML gives unicode
 		else:
 			return ''
-  
+
 	def _get_element_text(self, element):
 		"""
 		Return the textual content of the element and its children
@@ -67,7 +70,7 @@ class hocrTransform():
 		if element.tail is not None:
 			text = text + element.tail
 		return text
-    
+
 	def element_coordinates(self, element):
 		"""
 		Returns a tuple containing the coordinates of the bounding box around
@@ -80,13 +83,24 @@ class hocrTransform():
 				coords = matches.group(1).split()
 				out = (int(coords[0]),int(coords[1]),int(coords[2]),int(coords[3]))
 		return out
-    
+
 	def px2pt(self, pxl):
 		"""
 		Returns the length in pt given length in pxl
 		"""
 		return float(pxl)/self.dpi*inch
-			
+
+	def replace_unsupported_chars(self, str):
+		"""
+		Given an input string, returns the corresponding string that:
+		- is available in the helvetica facetype
+		- does not contain any ligature (to allow easy search in the PDF file)
+		"""		
+		# The 'u' before the character to replace indicates that it is a unicode character
+		str=str.replace(u"fl","fl")
+		str=str.replace(u"fi","fi")
+		return str
+		
 	def to_pdf(self, outFileName, imageFileName, showBoundingboxes, fontname="Helvetica"):
 		"""
 		Creates a PDF file with an image superimposed on top of the text.
@@ -97,13 +111,13 @@ class hocrTransform():
 		"""
 		# create the PDF file
 		pdf = Canvas(outFileName, pagesize=(self.width, self.height), pageCompression=1) # page size in points (1/72 in.)
-		
+
 		# draw bounding box for each paragraph
 		pdf.setStrokeColorRGB(0,1,1)	# light blue for bounding box of paragraph
 		pdf.setFillColorRGB(0,1,1)	# light blue for bounding box of paragraph
 		pdf.setLineWidth(0)		# no line for bounding box
 		for elem in self.hocr.findall(".//%sp[@class='%s']" % (self.xmlns, "ocr_par")):
-		
+
 			elemtxt=self._get_element_text(elem).rstrip()
 			if len(elemtxt) == 0:
 				continue
@@ -113,12 +127,12 @@ class hocrTransform():
 			y1=self.px2pt(coords[1])
 			x2=self.px2pt(coords[2])
 			y2=self.px2pt(coords[3])
-			
+
 			# draw the bbox border
 			if showBoundingboxes == True:
-				pdf.rect(x1, self.height-y2, x2-x1, y2-y1, fill=1)		
+				pdf.rect(x1, self.height-y2, x2-x1, y2-y1, fill=1)
-		
+
-		
+
 		# check if element with class 'ocrx_word' are available
 		# otherwise use 'ocr_line' as fallback
 		elemclass="ocr_line"
@@ -133,6 +147,9 @@ class hocrTransform():
 		for elem in self.hocr.findall(".//%sspan[@class='%s']" % (self.xmlns, elemclass)):
 
 			elemtxt=self._get_element_text(elem).rstrip()
+			
+			elemtxt=self.replace_unsupported_chars(elemtxt)
+			
 			if len(elemtxt) == 0:
 				continue
 
@@ -141,7 +158,7 @@ class hocrTransform():
 			y1=self.px2pt(coords[1])
 			x2=self.px2pt(coords[2])
 			y2=self.px2pt(coords[3])
-			
+
 			# draw the bbox border
 			if showBoundingboxes == True:
 				pdf.rect(x1, self.height-y2, x2-x1, y2-y1, fill=0)
@@ -152,7 +169,7 @@ class hocrTransform():
 
 			# set cursor to bottom left corner of bbox (adjust for dpi)
 			text.setTextOrigin(x1, self.height-y2)
-       
+
 			# scale the width of the text to fill the width of the bbox
 			text.setHorizScale(100*(x2-x1)/pdf.stringWidth(elemtxt, fontname, fontsize))
 
@@ -162,13 +179,14 @@ class hocrTransform():
 
 		# put the image on the page, scaled to fill the page
 		if imageFileName != None:
-			im = Image.open(imageFileName)		
+			im = Image.open(imageFileName)
 			pdf.drawInlineImage(im, 0, 0, width=self.width, height=self.height)
-			
+
 		# finish up the page and save it
 		pdf.showPage()
 		pdf.save()
-  
+
+
 if __name__ == "__main__":
 	parser = argparse.ArgumentParser(description='Convert hocr file to PDF')
 	parser.add_argument('-b', '--boundingboxes', action="store_true", default=False, help='Show bounding boxes borders')
@@ -181,5 +199,5 @@ if __name__ == "__main__":
 	hocr = hocrTransform(args.hocrfile, args.resolution)
 	hocr.to_pdf(args.outputfile, args.image, args.boundingboxes)
 
-	
+
-	
+

src/ocrPage.sh

@@ -37,13 +37,14 @@ FORCE_OCR="${14}"			# Force to OCR, even if the page already contains fonts
 # Output:  A file containing the characteristics of the embedded image. File structure:
 #          DPI=<dpi>
 #          COLOR_SPACE=<colorspace>
+#          DEPTH=<colordepth>
 # Returns:
 #       - 0: if no error occurs
 #       - 1: in case the page already contains fonts (which should be the case for PDF generated from scanned pages) 
 #       - 2: in case the page contains more than one image
 ##################################
 getImgInfo() {
-	local page widthPDF heightPDF curImgInfo nbImg curImg propCurImg widthCurImg heightCurImg colorspaceCurImg dpi
+	local page widthPDF heightPDF curImgInfo nbImg curImg propCurImg widthCurImg heightCurImg colorspaceCurImg depthCurImg dpi
 
 	# page number
 	page="$1"
@@ -58,25 +59,26 @@ getImgInfo() {
 	
 	
 	# check if the page already contains fonts (which should not be the case for PDF based on scanned files
-	[ `pdffonts -f $page -l $page ${FILE_INPUT_PDF} | wc -l` -gt 2 ] && echo "Page $page: Page already contains font data !!!" && return 1
+	[ `pdffonts -f $page -l $page "${FILE_INPUT_PDF}" | wc -l` -gt 2 ] && echo "Page $page: Page already contains font data !!!" && return 1
 
 	
 	# extract raw image from pdf file to compute resolution
 	# unfortunately this image can have another orientation than in the pdf...
 	# so we will have to extract it again later using pdftoppm
 	pdfimages -f $page -l $page -j "$FILE_INPUT_PDF" "$curOrigImg" 1>&2	
-	# count number of extracted images
+	# count number of extracted images 
-	nbImg=`ls -1 "$curOrigImg"* | wc -l`
+	nbImg=$((`ls -1 "$curOrigImg"* 2>/dev/null | wc -l`))
 	if [ $nbImg -ne "1" ]; then
 		[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Expecting exactly 1 image covering the whole page (found $nbImg). Cannot compute dpi value."
 		return 2
 	fi
 	# Get characteristics of the extracted image
-	curImg=`ls -1 "$curOrigImg"*`
+	curImg=`ls -1 "$curOrigImg"* 2>/dev/null`
-	propCurImg=`identify -format "%w %h %[colorspace]" "$curImg"`
+	propCurImg=`identify -format "%w %h %[colorspace] %[depth]" "$curImg"`
 	widthCurImg=`echo "$propCurImg" | cut -f1 -d" "`
 	heightCurImg=`echo "$propCurImg" | cut -f2 -d" "`
 	colorspaceCurImg=`echo "$propCurImg" | cut -f3 -d" "`
+	depthCurImg=`echo "$propCurImg" | cut -f4 -d" "`
 	[ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Size ${heightCurImg}x${widthCurImg} (in pixel)"	
 
 	# compute the resolution of the image (making the assumption that x & y resolution are equal)
@@ -87,6 +89,7 @@ getImgInfo() {
 	# save the image characteristics
 	echo "DPI=$dpi" > "$curImgInfo"
 	echo "COLOR_SPACE=$colorspaceCurImg" >> "$curImgInfo"
+	echo "DEPTH=$depthCurImg" >> "$curImgInfo"
 	
 	return 0
 }
@@ -109,24 +112,26 @@ curImgInfo="$TMP_FLD/${page}.orig-img-info.txt"			# Detected characteristics of
 
 
 # auto-detect the characteristics of the embedded image
+depthCurImg="8"			# default color depth
+colorspaceCurImg="sRGB"		# default color space
+dpi=$DEFAULT_DPI		# default resolution
+
 getImgInfo "$page" "$widthPDF" "$heightPDF" "$curImgInfo"
 ret_code="$?"
+
 # in case the page contains text do not OCR, unless the FORCE_OCR flag is set
-if [ "$ret_code" -eq "1" -a "$FORCE_OCR" -eq "0" ]; then
+if ([ "$ret_code" -eq "1" ] && [ "$FORCE_OCR" -eq "0" ]); then
 	echo "Page $page: Exiting... (Use the -f option to force OCRing, even though fonts are available in the input file)" && exit $EXIT_BAD_INPUT_FILE
-elif [ "$ret_code" -eq "1" -a "$FORCE_OCR" -eq "1" ]; then
+elif ([ "$ret_code" -eq "1" ] && [ "$FORCE_OCR" -eq "1" ]); then
-	colorspaceCurImg="sRGB"
-	dpi=$DEFAULT_DPI
 	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: OCRing anyway, assuming a default resolution of $dpi dpi"
 # in case the page contains more than one image, warn the user but go on with default parameters
 elif [ "$ret_code" -eq "2" ]; then
-	colorspaceCurImg="sRGB"
-	dpi=$DEFAULT_DPI
 	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Continuing anyway, assuming a default resolution of $dpi dpi"
 else
 	# read the image characteristics from the file
 	dpi=`cat "$curImgInfo" | grep "^DPI=" | cut -f2 -d"="`
 	colorspaceCurImg=`cat "$curImgInfo" | grep "^COLOR_SPACE=" | cut -f2 -d"="`
+	depthCurImg=`cat "$curImgInfo" | grep "^DEPTH=" | cut -f2 -d"="`
 fi
 
 # perform oversampling if the resolution is not sufficient to get good OCR results
@@ -137,10 +142,13 @@ elif [ "$dpi" -lt "200" ]; then
 	[ $VERBOSITY -ge $LOG_WARN ] && echo "Page $page: Low image resolution detected ($dpi dpi). If needed, please use the \"-o\" to try to get better OCR results." 
 fi
 	
-# Identify if page image should be saved as ppm (color) or pgm (gray)
+# Identify if page image should be saved as ppm (color), pgm (gray) or pbm (b&w)
-ext="ppm"
+ext="ppm" 	# by default (color image) the extension of the extracted image is ppm
-opt=""
+opt=""		# by default (color image) no option as to be passed to pdftoppm
-if [ "$colorspaceCurImg" = "Gray" ]; then
+if [ "$colorspaceCurImg" = "Gray" ] && [ "$depthCurImg" = "1" ]; then		# if monochrome (b&w)
+	ext="pbm"
+	opt="-mono"
+elif [ "$colorspaceCurImg" = "Gray" ]; then					# if gray
 	ext="pgm"
 	opt="-gray"
 fi
@@ -179,7 +187,14 @@ fi
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Performing OCR"
 ! tesseract -l "$LAN" "$curImgPixmapClean" "$curHocr" hocr $TESS_CFG_FILES 1> /dev/null 2> /dev/null \
 	&& echo "Could not OCR file \"$curImgPixmapClean\". Exiting..." && exit $EXIT_OTHER_ERROR
-mv "$curHocr.html" "$curHocr"
+# Tesseract names the output files differently in some distributions.
+if [ -e "$curHocr.html" ]; then
+	mv "$curHocr.html" "$curHocr"
+elif [ -e "$curHocr.hocr" ]; then
+	mv "$curHocr.hocr" "$curHocr"
+elif [ ! -e "$curHocr" ]; then
+	echo "\"$curHocr[.html|.hocr]\" not found. Exiting..." && exit $EXIT_OTHER_ERROR
+fi
 
 # embed text and image to new pdf file
 if [ "$PREPROCESS_CLEANTOPDF" -eq "1" ]; then
@@ -201,7 +216,7 @@ fi
 # delete temporary files created for the current page
 # to avoid using to much disk space in case of PDF files having many pages
 if [ $KEEP_TMP -eq 0 ]; then
-	rm -f "$curOrigImg"*.*
+	rm -f "$curOrigImg"*
 	rm -f "$curHocr"
 	rm -f "$curImgPixmap"
 	rm -f "$curImgPixmapDeskewed"
@@ -209,4 +224,4 @@ if [ $KEEP_TMP -eq 0 ]; then
 	rm -f "$curImgInfo"
 fi
 
-exit 0
+exit 0

tess-cfg/README.md

@@ -1,12 +0,0 @@
-##############################################################################
-# Readme
-#
-# Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
-##############################################################################
-
-The file(s) located in this folder are tesseract configuration files.
-(Tesseract configuration files enable to tune the behaviour of tesseract)
-
-If needed, these files should be copied to the "tessdata/configs" folder of your tesseract installation.
-
-To request OCRmyPDF.sh to use a configuration file, please use the -C option

tess-cfg/no_ligature

@@ -1,9 +0,0 @@
-##############################################################################
-# tesseract config file provided for OCRmyPDF
-#
-# prevents tesseract to detect ligatures, as ligatures are not displayed correctly in the final PDF file
-# but are replaced by a black square
-#
-# Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
-##############################################################################
-tessedit_char_blacklist fifl