From b7f38e976b5a85d9936d3b1c40c46a1663eb0de0 Mon Sep 17 00:00:00 2001 From: Ian Alexander <1693187+ianalexander@users.noreply.github.com> Date: Sun, 19 Jan 2020 19:11:54 -0800 Subject: [PATCH 1/6] Watched folder bug fixes, new flags, and docs updates. --- docs/batch.rst | 6 ++++++ misc/watcher.py | 27 +++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/docs/batch.rst b/docs/batch.rst index 9e4fe749..e9c96fa7 100644 --- a/docs/batch.rst +++ b/docs/batch.rst @@ -210,6 +210,9 @@ be launched as follows: -v :/input \ -v :/output \ -e OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1 \ + -e OCR_ON_SUCCESS_DELETE=1 \ + -e OCR_DESKEW=1 \ + -e PYTHONUNBUFFERED=1 \ -it --entrypoint python3 \ jbarlow83/ocrmypdf \ watcher.py @@ -224,6 +227,9 @@ convert it to a OCRed PDF in ``/output/``. The parameters to this image are: "``-v :/input``", "Files placed in this location will be OCRed" "``-v :/output``", "This is where OCRed files will be stored" "``-e OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1``", "This will place files in the output in {output}/{year}/{month}/{filename}" + "``-e OCR_ON_SUCCESS_DELETE=1``", "This will delete the input file if the exit code is 0 (OK)" + "``-e OCR_DESKEW=1``", "This will enable deskew for crooked PDFs" + "``-e PYTHONBUFFERED=1``", "This will force STDOUT to be unbuffered and allow you to see messages in docker logs" This service relies on polling to check for changes to the filesystem. It may not be suitable for some environments, such as filesystems shared on a diff --git a/misc/watcher.py b/misc/watcher.py index 99253001..86059cd4 100644 --- a/misc/watcher.py +++ b/misc/watcher.py @@ -25,12 +25,15 @@ import ocrmypdf INPUT_DIRECTORY = os.getenv('OCR_INPUT_DIRECTORY', '/input') OUTPUT_DIRECTORY = os.getenv('OCR_OUTPUT_DIRECTORY', '/output') +ON_SUCCESS_DELETE = bool(os.getenv('OCR_ON_SUCCESS_DELETE', False)) +DESKEW = bool(os.getenv('OCR_DESKEW', False)) OUTPUT_DIRECTORY_YEAR_MONTH = bool(os.getenv('OCR_OUTPUT_DIRECTORY_YEAR_MONTH', False)) PATTERNS = ['*.pdf'] def execute_ocrmypdf(file_path): - filename = Path(file_path).name + new_file = Path(file_path) + filename = new_file.name if OUTPUT_DIRECTORY_YEAR_MONTH: today = datetime.today() output_directory_year_month = Path( @@ -41,13 +44,29 @@ def execute_ocrmypdf(file_path): output_path = Path(output_directory_year_month) / filename else: output_path = Path(OUTPUT_DIRECTORY) / filename - print(f'New file: {file_path}.\nAttempting to OCRmyPDF to: {output_path}') - ocrmypdf.ocr(file_path, output_path) + print(f'New file: {file_path}. Waiting until fully loaded...') + # This loop waits to make sure that the file is completely loaded on + # disk before attempting to read. Docker sometimes will publish the + # watchdog event before the file is actually fully on disk, causing + # pikepdf to fail. + current_size = None + while current_size != new_file.stat().st_size: + current_size = new_file.stat().st_size + time.sleep(1) + print(f'Attempting to OCRmyPDF to: {output_path}') + exit_code = ocrmypdf.ocr( + input_file=file_path, output_file=output_path, deskew=DESKEW + ) + if exit_code == 0 and ON_SUCCESS_DELETE: + print(f'Done. Deleting: {file_path}') + new_file.unlink() + else: + print('Done') class HandleObserverEvent(PatternMatchingEventHandler): def on_any_event(self, event): - if event.event_type in ['created', 'modified']: + if event.event_type in ['created']: execute_ocrmypdf(event.src_path) From 3eab161771f63aac04891f4e7a0f4d47dae96afa Mon Sep 17 00:00:00 2001 From: Ian Alexander <1693187+ianalexander@users.noreply.github.com> Date: Mon, 20 Jan 2020 10:45:28 -0800 Subject: [PATCH 2/6] Update logging and env var extensibility --- misc/watcher.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/misc/watcher.py b/misc/watcher.py index 86059cd4..114365de 100644 --- a/misc/watcher.py +++ b/misc/watcher.py @@ -15,6 +15,7 @@ import os import time +import logging from datetime import datetime from pathlib import Path @@ -25,11 +26,15 @@ import ocrmypdf INPUT_DIRECTORY = os.getenv('OCR_INPUT_DIRECTORY', '/input') OUTPUT_DIRECTORY = os.getenv('OCR_OUTPUT_DIRECTORY', '/output') +OUTPUT_DIRECTORY_YEAR_MONTH = bool(os.getenv('OCR_OUTPUT_DIRECTORY_YEAR_MONTH', False)) ON_SUCCESS_DELETE = bool(os.getenv('OCR_ON_SUCCESS_DELETE', False)) DESKEW = bool(os.getenv('OCR_DESKEW', False)) -OUTPUT_DIRECTORY_YEAR_MONTH = bool(os.getenv('OCR_OUTPUT_DIRECTORY_YEAR_MONTH', False)) +POLL_NEW_FILE_SECONDS = os.getenv('OCR_POLL_NEW_FILE_SECONDS', 1) +LOGLEVEL = os.environ.get('OCR_LOGLEVEL', 'INFO').upper() PATTERNS = ['*.pdf'] +logging.basicConfig(level=LOGLEVEL) +logger = logging.getLogger('ocrmypdf-watcher') def execute_ocrmypdf(file_path): new_file = Path(file_path) @@ -44,7 +49,7 @@ def execute_ocrmypdf(file_path): output_path = Path(output_directory_year_month) / filename else: output_path = Path(OUTPUT_DIRECTORY) / filename - print(f'New file: {file_path}. Waiting until fully loaded...') + logger.info(f'New file: {file_path}. Waiting until fully loaded...') # This loop waits to make sure that the file is completely loaded on # disk before attempting to read. Docker sometimes will publish the # watchdog event before the file is actually fully on disk, causing @@ -52,16 +57,17 @@ def execute_ocrmypdf(file_path): current_size = None while current_size != new_file.stat().st_size: current_size = new_file.stat().st_size - time.sleep(1) - print(f'Attempting to OCRmyPDF to: {output_path}') + logger.debug(f'new_file current_size: {current_size}') + time.sleep(POLL_NEW_FILE_SECONDS) + logger.info(f'Attempting to OCRmyPDF to: {output_path}') exit_code = ocrmypdf.ocr( input_file=file_path, output_file=output_path, deskew=DESKEW ) if exit_code == 0 and ON_SUCCESS_DELETE: - print(f'Done. Deleting: {file_path}') + logger.info(f'Done. Deleting: {file_path}') new_file.unlink() else: - print('Done') + logger.info('Done') class HandleObserverEvent(PatternMatchingEventHandler): @@ -71,12 +77,21 @@ class HandleObserverEvent(PatternMatchingEventHandler): if __name__ == "__main__": - print( + logger.info( f"Starting OCRmyPDF watcher with config:\n" f"Input Directory: {INPUT_DIRECTORY}\n" f"Output Directory: {OUTPUT_DIRECTORY}\n" f"Output Directory Year & Month: {OUTPUT_DIRECTORY_YEAR_MONTH}" ) + logger.debug( + f"INPUT_DIRECTORY: {INPUT_DIRECTORY}\n" + f"OUTPUT_DIRECTORY: {OUTPUT_DIRECTORY}\n" + f"OUTPUT_DIRECTORY_YEAR_MONTH: {OUTPUT_DIRECTORY_YEAR_MONTH}\n" + f"ON_SUCCESS_DELETE: {ON_SUCCESS_DELETE}\n" + f"DESKEW: {DESKEW}\n" + f"POLL_NEW_FILE_SECONDS: {POLL_NEW_FILE_SECONDS}\n" + f"LOGLEVEL: {LOGLEVEL}\n" + ) handler = HandleObserverEvent(patterns=PATTERNS) observer = Observer() observer.schedule(handler, INPUT_DIRECTORY, recursive=True) From 4952af16047bef36b38c115c2b5f563b7c626b9d Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 28 Jan 2020 12:56:19 -0800 Subject: [PATCH 3/6] watcher: some refactoring --- misc/watcher.py | 55 +++++++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/misc/watcher.py b/misc/watcher.py index 114365de..d06108e4 100644 --- a/misc/watcher.py +++ b/misc/watcher.py @@ -13,9 +13,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import logging import os import time -import logging from datetime import datetime from pathlib import Path @@ -33,41 +33,52 @@ POLL_NEW_FILE_SECONDS = os.getenv('OCR_POLL_NEW_FILE_SECONDS', 1) LOGLEVEL = os.environ.get('OCR_LOGLEVEL', 'INFO').upper() PATTERNS = ['*.pdf'] -logging.basicConfig(level=LOGLEVEL) -logger = logging.getLogger('ocrmypdf-watcher') +log = logging.getLogger('ocrmypdf-watcher') -def execute_ocrmypdf(file_path): - new_file = Path(file_path) - filename = new_file.name + +def get_output_dir(root, basename): if OUTPUT_DIRECTORY_YEAR_MONTH: today = datetime.today() - output_directory_year_month = Path( - f'{OUTPUT_DIRECTORY}/{today.year}/{today.month}' + output_directory_year_month = ( + Path(root) / str(today.year) / f'{today.month:02d}' ) if not output_directory_year_month.exists(): output_directory_year_month.mkdir(parents=True, exist_ok=True) - output_path = Path(output_directory_year_month) / filename + output_path = Path(output_directory_year_month) / basename else: - output_path = Path(OUTPUT_DIRECTORY) / filename - logger.info(f'New file: {file_path}. Waiting until fully loaded...') + output_path = Path(OUTPUT_DIRECTORY) / basename + return output_path + + +def wait_for_file_ready(file_path): # This loop waits to make sure that the file is completely loaded on # disk before attempting to read. Docker sometimes will publish the # watchdog event before the file is actually fully on disk, causing # pikepdf to fail. + current_size = None - while current_size != new_file.stat().st_size: - current_size = new_file.stat().st_size - logger.debug(f'new_file current_size: {current_size}') + while current_size != file_path.stat().st_size: + current_size = file_path.stat().st_size + log.debug(f'file_path current_size: {current_size}') time.sleep(POLL_NEW_FILE_SECONDS) - logger.info(f'Attempting to OCRmyPDF to: {output_path}') + + +def execute_ocrmypdf(file_path): + file_path = Path(file_path) + output_path = get_output_dir(OUTPUT_DIRECTORY, file_path.name) + + log.info("-" * 20) + log.info(f'New file: {file_path}. Waiting until fully loaded...') + log.info(f'Attempting to OCRmyPDF to: {output_path}') + wait_for_file_ready(file_path) exit_code = ocrmypdf.ocr( input_file=file_path, output_file=output_path, deskew=DESKEW ) if exit_code == 0 and ON_SUCCESS_DELETE: - logger.info(f'Done. Deleting: {file_path}') - new_file.unlink() + log.info(f'OCR is done. Deleting: {file_path}') + file_path.unlink() else: - logger.info('Done') + log.info('OCR is done') class HandleObserverEvent(PatternMatchingEventHandler): @@ -77,13 +88,16 @@ class HandleObserverEvent(PatternMatchingEventHandler): if __name__ == "__main__": - logger.info( + ocrmypdf.configure_logging( + verbosity=ocrmypdf.Verbosity.default, manage_root_logger=True + ) + log.info( f"Starting OCRmyPDF watcher with config:\n" f"Input Directory: {INPUT_DIRECTORY}\n" f"Output Directory: {OUTPUT_DIRECTORY}\n" f"Output Directory Year & Month: {OUTPUT_DIRECTORY_YEAR_MONTH}" ) - logger.debug( + log.debug( f"INPUT_DIRECTORY: {INPUT_DIRECTORY}\n" f"OUTPUT_DIRECTORY: {OUTPUT_DIRECTORY}\n" f"OUTPUT_DIRECTORY_YEAR_MONTH: {OUTPUT_DIRECTORY_YEAR_MONTH}\n" @@ -92,6 +106,7 @@ if __name__ == "__main__": f"POLL_NEW_FILE_SECONDS: {POLL_NEW_FILE_SECONDS}\n" f"LOGLEVEL: {LOGLEVEL}\n" ) + handler = HandleObserverEvent(patterns=PATTERNS) observer = Observer() observer.schedule(handler, INPUT_DIRECTORY, recursive=True) From 82f393dd096a5895c23cb2b43b9de2b23607d84d Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 30 Jan 2020 12:40:19 -0800 Subject: [PATCH 4/6] Order of events --- misc/watcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/watcher.py b/misc/watcher.py index d06108e4..427a23bc 100644 --- a/misc/watcher.py +++ b/misc/watcher.py @@ -69,8 +69,8 @@ def execute_ocrmypdf(file_path): log.info("-" * 20) log.info(f'New file: {file_path}. Waiting until fully loaded...') - log.info(f'Attempting to OCRmyPDF to: {output_path}') wait_for_file_ready(file_path) + log.info(f'Attempting to OCRmyPDF to: {output_path}') exit_code = ocrmypdf.ocr( input_file=file_path, output_file=output_path, deskew=DESKEW ) From b8a780d684bb85260fbe9dc70ead21e8e153ad78 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 30 Jan 2020 12:40:48 -0800 Subject: [PATCH 5/6] Wait for file based on pikepdf --- misc/watcher.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/misc/watcher.py b/misc/watcher.py index 427a23bc..2c2dc5be 100644 --- a/misc/watcher.py +++ b/misc/watcher.py @@ -56,11 +56,20 @@ def wait_for_file_ready(file_path): # watchdog event before the file is actually fully on disk, causing # pikepdf to fail. - current_size = None - while current_size != file_path.stat().st_size: - current_size = file_path.stat().st_size - log.debug(f'file_path current_size: {current_size}') - time.sleep(POLL_NEW_FILE_SECONDS) + retries = 5 + while retries: + try: + pdf = pikepdf.open(file_path) + except (FileNotFoundError, pikepdf.PdfError) as e: + log.info(f"File {file_path} is not ready yet") + log.debug("Exception was", exc_info=e) + time.sleep(POLL_NEW_FILE_SECONDS) + retries -= 1 + else: + pdf.close() + return True + + return False def execute_ocrmypdf(file_path): @@ -69,7 +78,9 @@ def execute_ocrmypdf(file_path): log.info("-" * 20) log.info(f'New file: {file_path}. Waiting until fully loaded...') - wait_for_file_ready(file_path) + if not wait_for_file_ready(file_path): + log.info(f"Gave up waiting for {file_path} to become ready") + return log.info(f'Attempting to OCRmyPDF to: {output_path}') exit_code = ocrmypdf.ocr( input_file=file_path, output_file=output_path, deskew=DESKEW From bdb7f92131acd7f16ce5649e564a89924f32f1a6 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Mon, 10 Feb 2020 01:10:12 -0800 Subject: [PATCH 6/6] ifmain -> main() --- misc/watcher.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/misc/watcher.py b/misc/watcher.py index 2c2dc5be..e97d4625 100644 --- a/misc/watcher.py +++ b/misc/watcher.py @@ -19,11 +19,14 @@ import time from datetime import datetime from pathlib import Path +import pikepdf from watchdog.events import PatternMatchingEventHandler from watchdog.observers import Observer import ocrmypdf +# pylint: disable=logging-format-interpolation + INPUT_DIRECTORY = os.getenv('OCR_INPUT_DIRECTORY', '/input') OUTPUT_DIRECTORY = os.getenv('OCR_OUTPUT_DIRECTORY', '/output') OUTPUT_DIRECTORY_YEAR_MONTH = bool(os.getenv('OCR_OUTPUT_DIRECTORY_YEAR_MONTH', False)) @@ -98,7 +101,7 @@ class HandleObserverEvent(PatternMatchingEventHandler): execute_ocrmypdf(event.src_path) -if __name__ == "__main__": +def main(): ocrmypdf.configure_logging( verbosity=ocrmypdf.Verbosity.default, manage_root_logger=True ) @@ -128,3 +131,7 @@ if __name__ == "__main__": except KeyboardInterrupt: observer.stop() observer.join() + + +if __name__ == "__main__": + main()