diff --git a/docs/batch.rst b/docs/batch.rst index 9e4fe749..e9c96fa7 100644 --- a/docs/batch.rst +++ b/docs/batch.rst @@ -210,6 +210,9 @@ be launched as follows: -v :/input \ -v :/output \ -e OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1 \ + -e OCR_ON_SUCCESS_DELETE=1 \ + -e OCR_DESKEW=1 \ + -e PYTHONUNBUFFERED=1 \ -it --entrypoint python3 \ jbarlow83/ocrmypdf \ watcher.py @@ -224,6 +227,9 @@ convert it to a OCRed PDF in ``/output/``. The parameters to this image are: "``-v :/input``", "Files placed in this location will be OCRed" "``-v :/output``", "This is where OCRed files will be stored" "``-e OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1``", "This will place files in the output in {output}/{year}/{month}/{filename}" + "``-e OCR_ON_SUCCESS_DELETE=1``", "This will delete the input file if the exit code is 0 (OK)" + "``-e OCR_DESKEW=1``", "This will enable deskew for crooked PDFs" + "``-e PYTHONBUFFERED=1``", "This will force STDOUT to be unbuffered and allow you to see messages in docker logs" This service relies on polling to check for changes to the filesystem. It may not be suitable for some environments, such as filesystems shared on a diff --git a/misc/watcher.py b/misc/watcher.py index 99253001..e97d4625 100644 --- a/misc/watcher.py +++ b/misc/watcher.py @@ -13,51 +13,114 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import logging import os import time from datetime import datetime from pathlib import Path +import pikepdf from watchdog.events import PatternMatchingEventHandler from watchdog.observers import Observer import ocrmypdf +# pylint: disable=logging-format-interpolation + INPUT_DIRECTORY = os.getenv('OCR_INPUT_DIRECTORY', '/input') OUTPUT_DIRECTORY = os.getenv('OCR_OUTPUT_DIRECTORY', '/output') OUTPUT_DIRECTORY_YEAR_MONTH = bool(os.getenv('OCR_OUTPUT_DIRECTORY_YEAR_MONTH', False)) +ON_SUCCESS_DELETE = bool(os.getenv('OCR_ON_SUCCESS_DELETE', False)) +DESKEW = bool(os.getenv('OCR_DESKEW', False)) +POLL_NEW_FILE_SECONDS = os.getenv('OCR_POLL_NEW_FILE_SECONDS', 1) +LOGLEVEL = os.environ.get('OCR_LOGLEVEL', 'INFO').upper() PATTERNS = ['*.pdf'] +log = logging.getLogger('ocrmypdf-watcher') -def execute_ocrmypdf(file_path): - filename = Path(file_path).name + +def get_output_dir(root, basename): if OUTPUT_DIRECTORY_YEAR_MONTH: today = datetime.today() - output_directory_year_month = Path( - f'{OUTPUT_DIRECTORY}/{today.year}/{today.month}' + output_directory_year_month = ( + Path(root) / str(today.year) / f'{today.month:02d}' ) if not output_directory_year_month.exists(): output_directory_year_month.mkdir(parents=True, exist_ok=True) - output_path = Path(output_directory_year_month) / filename + output_path = Path(output_directory_year_month) / basename else: - output_path = Path(OUTPUT_DIRECTORY) / filename - print(f'New file: {file_path}.\nAttempting to OCRmyPDF to: {output_path}') - ocrmypdf.ocr(file_path, output_path) + output_path = Path(OUTPUT_DIRECTORY) / basename + return output_path + + +def wait_for_file_ready(file_path): + # This loop waits to make sure that the file is completely loaded on + # disk before attempting to read. Docker sometimes will publish the + # watchdog event before the file is actually fully on disk, causing + # pikepdf to fail. + + retries = 5 + while retries: + try: + pdf = pikepdf.open(file_path) + except (FileNotFoundError, pikepdf.PdfError) as e: + log.info(f"File {file_path} is not ready yet") + log.debug("Exception was", exc_info=e) + time.sleep(POLL_NEW_FILE_SECONDS) + retries -= 1 + else: + pdf.close() + return True + + return False + + +def execute_ocrmypdf(file_path): + file_path = Path(file_path) + output_path = get_output_dir(OUTPUT_DIRECTORY, file_path.name) + + log.info("-" * 20) + log.info(f'New file: {file_path}. Waiting until fully loaded...') + if not wait_for_file_ready(file_path): + log.info(f"Gave up waiting for {file_path} to become ready") + return + log.info(f'Attempting to OCRmyPDF to: {output_path}') + exit_code = ocrmypdf.ocr( + input_file=file_path, output_file=output_path, deskew=DESKEW + ) + if exit_code == 0 and ON_SUCCESS_DELETE: + log.info(f'OCR is done. Deleting: {file_path}') + file_path.unlink() + else: + log.info('OCR is done') class HandleObserverEvent(PatternMatchingEventHandler): def on_any_event(self, event): - if event.event_type in ['created', 'modified']: + if event.event_type in ['created']: execute_ocrmypdf(event.src_path) -if __name__ == "__main__": - print( +def main(): + ocrmypdf.configure_logging( + verbosity=ocrmypdf.Verbosity.default, manage_root_logger=True + ) + log.info( f"Starting OCRmyPDF watcher with config:\n" f"Input Directory: {INPUT_DIRECTORY}\n" f"Output Directory: {OUTPUT_DIRECTORY}\n" f"Output Directory Year & Month: {OUTPUT_DIRECTORY_YEAR_MONTH}" ) + log.debug( + f"INPUT_DIRECTORY: {INPUT_DIRECTORY}\n" + f"OUTPUT_DIRECTORY: {OUTPUT_DIRECTORY}\n" + f"OUTPUT_DIRECTORY_YEAR_MONTH: {OUTPUT_DIRECTORY_YEAR_MONTH}\n" + f"ON_SUCCESS_DELETE: {ON_SUCCESS_DELETE}\n" + f"DESKEW: {DESKEW}\n" + f"POLL_NEW_FILE_SECONDS: {POLL_NEW_FILE_SECONDS}\n" + f"LOGLEVEL: {LOGLEVEL}\n" + ) + handler = HandleObserverEvent(patterns=PATTERNS) observer = Observer() observer.schedule(handler, INPUT_DIRECTORY, recursive=True) @@ -68,3 +131,7 @@ if __name__ == "__main__": except KeyboardInterrupt: observer.stop() observer.join() + + +if __name__ == "__main__": + main()