update release notes for v2.2-stable

Python does not map the expression to its return code automatically, so
this line returns success regardless of the reportlab version installed.
(I also realized that hasattr is superfluous).

.gitignore

@@ -1,2 +1,3 @@
 tmp/
-log/
+log/
+*.pyc

OCRmyPDF.sh

@@ -3,8 +3,12 @@
 # Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
 ##############################################################################
 
+# Determine real path of this script, following symlinks if present
+! command -v python2 > /dev/null && echo "Please install python v2.x. Exiting..." && exit 1
+BASEPATH="$(dirname $(python2 -c "import os; print os.path.realpath(\"$0\")"))"
+
 # Import required scripts
-. "`dirname $0`/src/config.sh"
+. "$BASEPATH/src/config.sh"
 
 # Set variables corresponding to the input parameters
 ARGUMENTS="$@"
@@ -34,12 +38,12 @@ Usage: OCRmyPDF.sh  [-h] [-v] [-g] [-k] [-d] [-c] [-i] [-o dpi] [-f] [-l languag
 -d : Deskew each page before performing OCR
 -c : Clean each page before performing OCR
 -i : Incorporate the cleaned image in the final PDF file (by default the original image	
-     image, or the deskewed image if the -d option is set, is incorporated)
+     image, or the deskewed image if the -d option is set)
 -o : If the resolution of an image is lower than dpi value provided as argument, provide the OCR engine with 
      an oversampled image having the latter dpi value. This can improve the OCR results but can lead to a larger output PDF file.
      (default: no oversampling performed)
 -f : Force to OCR the whole document, even if some page already contain font data 
-     (which should not be the case for PDF files built from scnanned images) 
+     (which should not be the case for PDF files built from scanned images) 
 -l : Set the language of the PDF file in order to improve OCR results (default "eng")
      Any language supported by tesseract is supported (Tesseract uses 3-character ISO 639-2 language codes)
      Multiple languages may be specified, separated by '+' characters.
@@ -118,9 +122,10 @@ if [ "$#" -ne "2" ]; then
 	exit $EXIT_BAD_ARGS
 fi
 
-! absolutePath "$1" > /dev/null \
-	&& echo "The folder in which the input file should be located does not exist. Exiting..." && exit $EXIT_BAD_ARGS
+[ ! -f "$1" ] \
+	&& echo "The input file does not exist. Exiting..." && exit $EXIT_BAD_ARGS
 FILE_INPUT_PDF="`absolutePath "$1"`"
+
 ! absolutePath "$2" > /dev/null \
 	&& echo "The folder in which the output file should be generated does not exist. Exiting..." && exit $EXIT_BAD_ARGS
 FILE_OUTPUT_PDFA="`absolutePath "$2"`"
@@ -128,7 +133,7 @@ FILE_OUTPUT_PDFA="`absolutePath "$2"`"
 
 
 # set script path as working directory
-cd "`dirname $0`"
+cd "$BASEPATH"
 
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "$TOOLNAME version: $VERSION"
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Arguments: $ARGUMENTS"
@@ -142,12 +147,12 @@ cd "`dirname $0`"
 ! command -v pdffonts > /dev/null && echo "Please install poppler-utils. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
 [ $PREPROCESS_CLEAN -eq 1 ] && ! command -v unpaper > /dev/null && echo "Please install unpaper. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
 ! command -v tesseract > /dev/null && echo "Please install tesseract and tesseract-data. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
-! command -v python2 > /dev/null && echo "Please install python v2.x. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
 ! python2 -c 'import lxml' 2>/dev/null && echo "Please install the python library lxml. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
-! python2 -c 'import reportlab' 2>/dev/null && echo "Please install the python library reportlab. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
-! command -v gs > /dev/null && echo "Please install ghostcript. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
-! command -v java > /dev/null && echo "Please install java. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! python2 -c 'import sys, reportlab; (getattr(reportlab, "Version", "0.0") >= "3.0") or sys.exit(1)' 2>/dev/null \
+	&& echo "Please install the python library reportlab. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
 
+! command -v gs > /dev/null && echo "Please install ghostscript. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
+! command -v java > /dev/null && echo "Please install java. Exiting..." && exit $EXIT_MISSING_DEPENDENCY
 
 
 # ensure the right tesseract version is installed
@@ -162,7 +167,7 @@ tesstooold=$(echo "`echo $tessversion | sed s/[.]//2`-`echo $reqtessversion | se
 
 # ensure the right GNU parallel version is installed
 # older version do not support -q flag (required to escape special characters)
-reqparallelversion="20130222"
+reqparallelversion="20121122"
 parallelversion=`parallel --minversion 0`
 ! parallel --minversion "$reqparallelversion" > /dev/null \
 	&& echo "Please install GNU parallel ${reqparallelversion} or newer (currently installed version is ${parallelversion})" && exit $EXIT_MISSING_DEPENDENCY
@@ -266,7 +271,8 @@ ret_code="$?"
 
 # validate generated pdf file (compliance to PDF/A)
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Output file: Checking compliance to PDF/A standard" 
-java -jar "$JHOVE" -c "$JHOVE_CFG" -m PDF-hul "$FILE_OUTPUT_PDFA" > "$FILE_VALIDATION_LOG"
+! java -jar "$JHOVE" -c "$JHOVE_CFG" -m PDF-hul "$FILE_OUTPUT_PDFA" 2> /dev/null 1> "$FILE_VALIDATION_LOG" \
+	&& echo "Unexpected error while checking compliance to PDF/A file. Exiting..." && exit $EXIT_OTHER_ERROR
 grep -i "Status|Message" "$FILE_VALIDATION_LOG" # summary of the validation
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "The full validation log is available here: \"$FILE_VALIDATION_LOG\""
 # check the validation results
@@ -292,4 +298,4 @@ END=`date +%s`
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Script took $(($END-$START)) seconds"
 
 
-[ $pdf_valid -ne 1 ] && exit $EXIT_INVALID_OUPUT_PDFA || exit 0
+[ $pdf_valid -ne 1 ] && exit $EXIT_INVALID_OUTPUT_PDFA || exit 0

README.md

@@ -60,7 +60,8 @@ In case you detect an issue, please:
 Press & Media
 -------------
 
-- c't 1-2014, page 59: Detailed presentation of OCRmyPDF v1.0 in the leading german IT magazine c't (http://www.heise.de/ct/inhalt/2014/1/58/)
+- c't 1-2014, page 59: Detailed presentation of OCRmyPDF v1.0 in the leading german IT magazine c't (http://www.heise.de/ct/inhalt/2014/1/58/) 
+- heise Open Source, 09/2014: Texterkennung mit OCRmyPDF (http://www.heise.de/-2356670)
 
 Disclaimer
 ----------

RELEASE_NOTES.md

@@ -5,6 +5,76 @@ Please always read this file before installing the package
 
 Download software here: https://github.com/fritz-hh/OCRmyPDF/tags
 
+v2.2-stable (2014-09-29):
+=======
+
+New features
+------------
+
+- None
+
+Changes
+-------
+
+- Update to jhove v1.11
+- Request the python library reportlab v3.0 or newer (So that we could remove a patch to the previous version of reportlab leading to issues for some users)
+
+Fixes
+-----
+
+- Fix bug on Mac OS X (resolution of simlink to OCRmyPDF.sh script) (thanks to jbarlow83)
+- Check if the input pdf file exists before to continue
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.2
+- Dependencies:
+   - parallel 20140822
+   - poppler-utils 0.24.5
+   - ImageMagick 6.8.9-4 2014-09-17
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.8
+   - ghostcript (gs): 9.06
+   - java: openjdk version "1.7.0_65"
+
+v2.1-stable (2014-09-20):
+=======
+
+New features
+------------
+
+- None
+
+Changes
+-------
+
+- None
+
+Fixes
+-----
+
+- Allow execution via simlink
+- Add support for tesseract 3.03
+- Add support for newer version of reportlab
+- Lowered minimum version of gnu parallel
+- Various typo
+
+Tested with
+-----------
+
+- Operating system: FreeBSD 9.1
+- Dependencies:
+   - parallel 20130222
+   - poppler-utils 0.22.2
+   - ImageMagick 6.8.0-7 2013-03-30
+   - Unpaper 0.3
+   - tesseract 3.02.02
+   - Python 2.7.3
+   - ghostcript (gs): 9.06
+   - java: openjdk version "1.7.0_17"
+
 v2.0-stable (2014-01-25):
 =======
 
@@ -34,7 +104,7 @@ Tested with
    - Unpaper 0.3
    - tesseract 3.02.02
    - Python 2.7.3
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"
 
 v2.0-rc2 (2014-01-16):
@@ -77,7 +147,7 @@ Tested with
    - Unpaper 0.3
    - tesseract 3.02.02
    - Python 2.7.3
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"
 
 v2.0-rc1 (2014-01-07):
@@ -124,7 +194,7 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"
 
 v1.1-stable (2014-01-06):
@@ -156,7 +226,7 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"
 
 v1.0-stable (2013-05-06):
@@ -193,7 +263,7 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"
 
 v1.0-rc2 (2013-04-29):
@@ -229,7 +299,7 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"
 
 v1.0-rc1 (2013-04-26):
@@ -261,5 +331,5 @@ Tested with
    - tesseract 3.02.02
    - Python 2.7.3
    - pdftk 1.45
-   - ghoscript (gs): 9.06
+   - ghostcript (gs): 9.06
    - java: openjdk version "1.7.0_17"

jhove/README

@@ -3,7 +3,7 @@ Copyright 2003-2012 by JSTOR and the President and Fellows of Harvard College
 JHOVE is made available under the GNU Lesser General Public License (LGPL;
 see the file LICENSE for details)
 
-Rev. 1.9, 2012-12-17
+Rev. 1.11, 2013-09-29
 
 JHOVE (the JSTOR/Harvard Object Validation Environment, pronounced "jhove")
 is an extensible software framework for performing format identification,

jhove/RELEASENOTES

@@ -6,6 +6,85 @@ see the file LICENSE for details)
 
 Versions 1.7 and beyond of JHOVE are no longer under the control of Harvard.
 
+RELEASE NOTES FOR JHOVE 1.11
+
+GENERAL
+
+1. I've added lots of logging code. Calls at the FINE level and lower 
+   don't show up no matter what I do, so I've put them at the INFO level.
+   The level is set in JhoveBase.java.
+
+2. All .bat and _bat.tmpl files now have CR-LF line endings. That is, they
+   do in the gzip and zip archives you download. I'm not sure how
+   SourceForge will treat files that you download individually,
+   but hopefully it will have the sense to keep CR-LF when downloading
+   to a Windows system. 
+
+3. All .bat files now assume JHOVE_HOME is the directory from which they're
+   run. They no longer try to set JAVA_HOME (which was still stuck in
+   Java 1.4 and probably wasn't working for many people), instead assuming
+   that the JAVA command is available on the command line.
+
+4. All javac commands in build.xml files now specify source=1.5 for
+   compatibility with more recent compilers.
+  
+5. gdumpwin.bat is deleted. It's redundant with gdump.bat and has bugs
+   of its own.
+   
+PDF MODULE
+
+1. Fix to PDF module, submitted by willp-bl, may reduce tendency 
+   to run out of heap space on some files.
+
+RELEASE NOTES FOR JHOVE 1.10
+
+GENERAL
+
+1. The amount of logging code has been increased, mostly at the
+   DEBUG level.
+   
+2. Further work on generics in Java code.
+
+3. JhoveView now checks for Java 1.5. Was previously allowing 1.4 even
+   though it wouldn't work.
+
+HTML MODULE
+
+1. XHTML files are processed by the HTML module, which invokes the XML
+   modules. In this case, the XML module doesn't have the parameters
+   specified in the JHOVE configuration file and so won't use local
+   copies of schemas. Starting with this version, the parameters of
+   the HTML module are passed to the XML module when invoking it.
+   However, this doesn't work properly (in either module) for a DTD
+   that invokes additional DTDs by relative URLs. Such DTDs should
+   be edited to use only absolute URLs.
+   
+PDF MODULE
+
+1. Failure to get a page object number wasn't being handled cleanly,
+   resulting in a report of an invalid document without an error message
+   to explain it (SourceForge bug 49). This has been fixed.
+
+2. The PDF module unnecessarily uses huge amounts of memory to build
+   complex structure trees, when it doesn't need to keep the whole 
+   tree in memory to validate it. In the new version, it uses memory
+   more economically. This should result in the successful processing
+   of some PDF files that ran out of memory or took hours to process before.
+   
+3. If an annotation isn't a dictionary object, report that explicitly.
+   This happens with some otherwise good files; I can't find any warrant
+   for it in the PDF spec.
+
+4. Some efficiency improvements to PDF parser. Increased buffer size from 4K 
+   to 64K. Made Parser.collapseObjectVector more efficient. Parser now
+   returns pseudo-objects for array and dictionary end instead of throwing
+   an exception.
+
+5. Minor cleanup of error reporting.
+
+6. If an object uses a compression scheme which JHOVE can't deal with, JHOVE
+   will try to give a specific error message.
+
 RELEASE NOTES FOR JHOVE 1.9
 
 GENERAL

jhove/conf/jhove-byteoffset=true.conf

@@ -3,7 +3,7 @@
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
  xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.4/jhoveConfig.xsd">
+                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
  <jhoveHome>/users/stephen/projects/jhove</jhoveHome>
  <defaultEncoding>utf-8</defaultEncoding>
  <tempDirectory>/var/tmp</tempDirectory>

jhove/conf/jhove-withTextMD.conf

@@ -3,7 +3,7 @@
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
  xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.4/jhoveConfig.xsd">
+                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
  <jhoveHome>/users/stephen/projects/jhove</jhoveHome>
  <defaultEncoding>utf-8</defaultEncoding>
  <tempDirectory>/var/tmp</tempDirectory>

jhove/conf/jhove.conf

@@ -3,7 +3,7 @@
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
  xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.4/jhoveConfig.xsd">
+                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
  <jhoveHome>./jhove/</jhoveHome>
  <defaultEncoding>utf-8</defaultEncoding>
  <tempDirectory>/var/tmp</tempDirectory>

jhove/jhove_bat.tmpl

@@ -35,9 +35,6 @@ REM EXTRA_JARS Extra jar files to add to CLASSPATH
 REM SET JHOVE_HOME="C:\Program Files\jhove"
 SET JHOVE_HOME="[your directory path]\jhove"
 
-SET JAVA_HOME="C:\Program Files\java\j2re1.4.1_02"
-SET JAVA=%JAVA_HOME%\bin\java
-
 SET EXTRA_JARS=
 
 REM NOTE: Nothing below this line should be edited
@@ -60,4 +57,4 @@ IF "%1"=="" GOTO LOOP
 :LOOP
 
 REM Set the CLASSPATH and invoke the Java loader
-%JAVA% -classpath %CP% Jhove %ARGS%
+JAVA -classpath %CP% Jhove %ARGS%

src/config.sh

@@ -9,13 +9,13 @@ DEFAULT_DPI=300				# dpi value used as fall back if the page dpi cannot be deter
 #####################################################################################
 
 TOOLNAME="OCRmyPDF"
-VERSION="v2.0-stable"
+VERSION="v2.2-stable"
 
 # possible exit codes
 EXIT_BAD_ARGS="1"
 EXIT_BAD_INPUT_FILE="2"
 EXIT_MISSING_DEPENDENCY="3"
-EXIT_INVALID_OUPUT_PDFA="4"
+EXIT_INVALID_OUTPUT_PDFA="4"
 EXIT_FILE_ACCESS_ERROR="5"
 EXIT_OTHER_ERROR="15"
 

src/hocrTransform.py

@@ -7,7 +7,6 @@
 # Initial version by Jonathan Brinley, jonathanbrinley@gmail.com
 ##############################################################################
 from reportlab.pdfgen.canvas import Canvas
-from reportlab.pdfgen.pdfimages import PDFImage
 from reportlab.lib.units import inch
 from lxml import etree as ElementTree
 from PIL import Image
@@ -15,87 +14,6 @@ import re, sys
 import argparse
 
 
-def monkeypatch_method(cls):
-	'''
-	Override a class method at runtime.
-
-	Rationale:
-	https://mail.python.org/pipermail/python-dev/2008-January/076194.html
-	'''
-	def decorator(func):
-		setattr(cls, func.__name__, func)
-		return func
-	return decorator
-
-
-@monkeypatch_method(PDFImage)
-def PIL_imagedata(self):
-	'''
-	Add ability to output greyscale and 1-bit PIL images without conversion to RGB.
-
-	The upstream Python 2.7 version of reportlab converts 1-bit PIL images to RGB
-	instead of saving them in a lower BPP format.  They have since added the following
-	fix to their Python 3.3 branch, but it has not been back-ported.
-
-	https://bitbucket.org/rptlab/reportlab/commits/177ddcbe4df6f9b461dac62612df9b8da3966a5d
-	'''
-	image = self.image
-	if image.format == 'JPEG':
-		fp = image.fp
-		fp.seek(0)
-		return self._jpg_imagedata(fp)
-
-	from reportlab.lib.utils import import_zlib
-	from reportlab import rl_config
-	from reportlab.pdfbase.pdfutils import _AsciiBase85Encode, _chunker
-
-	self.source = 'PIL'
-	zlib = import_zlib()
-	if not zlib:
-		return
-
-	bpc = 8
-	# Use the colorSpace in the image
-	if image.mode == 'CMYK':
-		myimage = image
-		colorSpace = 'DeviceCMYK'
-		bpp = 4
-	elif image.mode == '1':
-		myimage = image
-		colorSpace = 'DeviceGray'
-		bpp = 1
-		bpc = 1
-	elif image.mode == 'L':
-		myimage = image
-		colorSpace = 'DeviceGray'
-		bpp = 1
-	else:
-		myimage = image.convert('RGB')
-		colorSpace = 'RGB'
-		bpp = 3
-	imgwidth, imgheight = myimage.size
-
-	# this describes what is in the image itself
-	# *NB* according to the spec you can only use the short form in inline images
-
-	imagedata = ['BI /W %d /H %d /BPC %d /CS /%s /F [%s/Fl] ID' %
-				 (imgwidth, imgheight, bpc, colorSpace, rl_config.useA85 and '/A85 ' or '')]
-
-	# use a flate filter and, optionally, Ascii Base 85 to compress
-	raw = myimage.tostring()
-	rowstride = (imgwidth * bpc * bpp + 7) / 8
-	assert len(raw) == rowstride * imgheight, "Wrong amount of data for image"
-	data = zlib.compress(raw)  # this bit is very fast...
-
-	if rl_config.useA85:
-		# ...sadly this may not be
-		data = _AsciiBase85Encode(data)
-	# append in blocks of 60 characters
-	_chunker(data, imagedata)
-	imagedata.append('EI')
-	return (imagedata, imgwidth, imgheight)
-
-
 class hocrTransform():
 	"""
 	A class for converting documents from the hOCR format.

src/ocrPage.sh

@@ -187,7 +187,14 @@ fi
 [ $VERBOSITY -ge $LOG_DEBUG ] && echo "Page $page: Performing OCR"
 ! tesseract -l "$LAN" "$curImgPixmapClean" "$curHocr" hocr $TESS_CFG_FILES 1> /dev/null 2> /dev/null \
 	&& echo "Could not OCR file \"$curImgPixmapClean\". Exiting..." && exit $EXIT_OTHER_ERROR
-mv "$curHocr.html" "$curHocr"
+# Tesseract names the output files differently in some distributions.
+if [ -e "$curHocr.html" ]; then
+	mv "$curHocr.html" "$curHocr"
+elif [ -e "$curHocr.hocr" ]; then
+	mv "$curHocr.hocr" "$curHocr"
+elif [ ! -e "$curHocr" ]; then
+	echo "\"$curHocr[.html|.hocr]\" not found. Exiting..." && exit $EXIT_OTHER_ERROR
+fi
 
 # embed text and image to new pdf file
 if [ "$PREPROCESS_CLEANTOPDF" -eq "1" ]; then