From f453e94f146b149f0d11f2889824407b9959e98e Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 13 Apr 2021 13:16:35 -0700 Subject: [PATCH] awslambda: better documentation --- src/ocrmypdf/extra_plugins/awslambda.py | 26 ++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/ocrmypdf/extra_plugins/awslambda.py b/src/ocrmypdf/extra_plugins/awslambda.py index 0f98f470..bf7569f3 100644 --- a/src/ocrmypdf/extra_plugins/awslambda.py +++ b/src/ocrmypdf/extra_plugins/awslambda.py @@ -5,7 +5,20 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. -"""Alternate executor to support OCRmyPDF in AWS Lambda""" +"""Alternate executor to support OCRmyPDF in AWS Lambda. + +AWS Lambda does not support the standard multiprocessing module because it +environment. + +This alternate executor avoids that. However, it has drawbacks. Most notably, +it divvies up work among worker processes at the beginning, to avoid coordinating +effort in ways that require shared queues and semaphores. In this implementation, +there is no shared queue, so no possible lock contention between workers. The +main process has a shared pipe with each worker. + +If some tasks are larger than others, some workers will may fall far behind +while others have deep queues. The last worker may end up with fewer tasks. +""" import logging @@ -16,7 +29,7 @@ from enum import Enum, auto from itertools import islice, repeat, takewhile, zip_longest from multiprocessing import Pipe, Process from multiprocessing.connection import Connection, wait -from typing import Callable, Iterable, Optional +from typing import Callable, Iterable, Iterator from unittest.mock import Mock from ocrmypdf import Executor, hookimpl @@ -31,7 +44,14 @@ class MessageType(Enum): complete = auto() -def split_every(n: int, iterable: Iterable): +def split_every(n: int, iterable: Iterable) -> Iterator: + """Split iterable into groups of n. + + >>> list(split_every(4, range(10))) + [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] + + https://stackoverflow.com/a/22919323 + """ iterator = iter(iterable) return takewhile(bool, (list(islice(iterator, n)) for _ in repeat(None)))