Skip to content

Instantly share code, notes, and snippets.

@Cyberes

Cyberes/How to re-implement remove-background in ocrmypdf.md

Last active February 3, 2023 22:58
Show Gist options
  • Save Cyberes/8316c84bff1fe45f12ef2ae8d583e5b2 to your computer and use it in GitHub Desktop.
Save Cyberes/8316c84bff1fe45f12ef2ae8d583e5b2 to your computer and use it in GitHub Desktop.
Download ZIP
Re-implement --remove-background in OCRmyPDF
Raw
How to re-implement remove-background in ocrmypdf.md

I needed --remove-background so I re-implemented this feature myself.

https://github.com/ocrmypdf/OCRmyPDF

Build Dependencies

Leptonica

This feature requires Leptonica, an image processing and analysis library. You have to build it from source, which is why it was removed from OCRmyPDF.

Try installing the package first:

sudo apt update && sudo apt install libleptonica-dev

Or, build from source:

wget https://github.com/DanBloomberg/leptonica/releases/download/1.83.1/leptonica-1.83.1.tar.gz
tar -xvzf leptonica-1.83.1.tar.gz
cd leptonica-1.83.1/
./configure
make -j $(nproc)
sudo make install

Build OCRmyPDF

Clone the repository

git clone -b master https://github.com/ocrmypdf/OCRmyPDF.git
git reset --hard 2685f91 # I'm using this specific commit to ensure reproducibility
cd OCRmyPDF

Apply the Git Patch

curl -s https://gist.githubusercontent.com/Cyberes/8316c84bff1fe45f12ef2ae8d583e5b2/raw/c6f62fa07808478bfaced6c5edaa89d66a8f229d/restore-remove-background.patch | git apply --whitespace=fix

Build and Install

sudo apt update && sudo apt install libjbig2dec0 libjbig2dec0-dev
pip install --upgrade pip
pip install --upgrade setuptools wheel
pip uninstall -y ocrmypdf
pip install .
Raw
restore-remove-background.patch
From 2e55e53d058bc4efba438a65476487d4cbfa1817 Mon Sep 17 00:00:00 2001
From: Cyberes <64224601+Cyberes@users.noreply.github.com>
Date: Fri, 3 Feb 2023 15:56:02 -0700
Subject: [PATCH] restore --remove-background
---
.pre-commit-config.yaml | 1 +
README.md | 2 +-
misc/completion/ocrmypdf.bash | 1 +
pyproject.toml | 9 +-
src/ocrmypdf/__init__.py | 2 +-
src/ocrmypdf/_pipeline.py | 9 +-
src/ocrmypdf/leptonica.py | 1019 +++++++++++++++++++++++++
src/ocrmypdf/lib/__init__.py | 8 +
src/ocrmypdf/lib/_leptonica.py | 11 +
src/ocrmypdf/lib/compile_leptonica.py | 516 +++++++++++++
tests/test_image_input.py | 2 +-
tests/test_lept.py | 96 +++
tests/test_main.py | 2 +-
tests/test_preprocessing.py | 2 +-
14 files changed, 1670 insertions(+), 10 deletions(-)
create mode 100644 src/ocrmypdf/leptonica.py
create mode 100644 src/ocrmypdf/lib/__init__.py
create mode 100644 src/ocrmypdf/lib/_leptonica.py
create mode 100644 src/ocrmypdf/lib/compile_leptonica.py
create mode 100644 tests/test_lept.py
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index b6a207c8..00c02fb3 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -20,6 +20,7 @@ repos:
hooks:
- id: black
language_version: python
+ exclude: ^src/ocrmypdf/lib/_leptonica.py
- repo: https://github.com/asottile/setup-cfg-fmt
rev: v1.20.2
hooks:
diff --git a/README.md b/README.md
index a2eb4c5c..a30ace45 100644
--- a/README.md
+++ b/README.md
@@ -112,7 +112,7 @@ Please report issues on our [GitHub issues](https://github.com/ocrmypdf/OCRmyPDF
## Requirements
-In addition to the required Python version (3.7+), OCRmyPDF requires external program installations of Ghostscript and Tesseract OCR. OCRmyPDF is pure Python, and runs on pretty much everything: Linux, macOS, Windows and FreeBSD.
+In addition to the required Python version (3.7+), OCRmyPDF requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. OCRmyPDF is pure Python, but uses CFFI to portably generate library bindings. OCRmyPDF is pure Python, and runs on pretty much everything: Linux, macOS, Windows and FreeBSD.
## Press & Media
diff --git a/misc/completion/ocrmypdf.bash b/misc/completion/ocrmypdf.bash
index fa857054..96c9081b 100644
--- a/misc/completion/ocrmypdf.bash
+++ b/misc/completion/ocrmypdf.bash
@@ -52,6 +52,7 @@ __ocrmypdf_arguments()
--user-words (specify location of user words file)
--user-patterns (specify location of user patterns file)
--no-progress-bar (disable the progress bar)
+--remove-background (attempt to remove background from gray or color pages, setting it to white)
"
COMPREPLY=( $( compgen -W "$arguments" -- "$cur") )
diff --git a/pyproject.toml b/pyproject.toml
index dda01034..73d4794e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,8 @@
requires = [
"setuptools >= 61",
"setuptools_scm[toml] >= 7.0.5",
- "wheel"
+ "wheel",
+ "cffi",
]
build-backend = "setuptools.build_meta"
@@ -17,6 +18,7 @@ license = {text = "MPL-2.0"}
requires-python = ">=3.8"
dependencies = [
"Pillow>=8.2.0",
+ "cffi == 1.14.5",
"coloredlogs>=14.0",
"deprecation>=2.1.0",
"img2pdf>=0.3.0", # pure Python
@@ -114,6 +116,7 @@ exclude = '''
| docs
| misc
| \.egg-info
+ | src/ocrmypdf/lib/_leptonica.py
)/
'''
@@ -145,6 +148,8 @@ profile = "black"
known_first_party = "ocrmypdf"
known_third_party = [
"PIL",
+ "_cffi_backend",
+ "cffi",
"flask",
"img2pdf",
"ocrmypdf",
@@ -177,6 +182,8 @@ module = [
'tqdm',
'coloredlogs',
'img2pdf',
+ 'cffi',
+ '_cffi_backend',
'pdfminer.*',
'reportlab.*',
'fitz',
diff --git a/src/ocrmypdf/__init__.py b/src/ocrmypdf/__init__.py
index 25c3039d..c64582dc 100644
--- a/src/ocrmypdf/__init__.py
+++ b/src/ocrmypdf/__init__.py
@@ -7,7 +7,7 @@ from __future__ import annotations
from pluggy import HookimplMarker as _HookimplMarker
-from ocrmypdf import helpers, hocrtransform, pdfa, pdfinfo
+from ocrmypdf import helpers, hocrtransform, leptonica, pdfa, pdfinfo
from ocrmypdf._concurrent import Executor
from ocrmypdf._jobcontext import PageContext, PdfContext
from ocrmypdf._version import PROGRAM_NAME, __version__
diff --git a/src/ocrmypdf/_pipeline.py b/src/ocrmypdf/_pipeline.py
index bdfc4d84..553f2808 100644
--- a/src/ocrmypdf/_pipeline.py
+++ b/src/ocrmypdf/_pipeline.py
@@ -21,6 +21,7 @@ import pikepdf
from pikepdf.models.metadata import encode_pdf_date
from PIL import Image, ImageColor, ImageDraw
+from ocrmypdf import leptonica
from ocrmypdf._concurrent import Executor
from ocrmypdf._exec import unpaper
from ocrmypdf._jobcontext import PageContext, PdfContext
@@ -474,10 +475,10 @@ def rasterize(
def preprocess_remove_background(input_file: Path, page_context: PageContext) -> Path:
if any(image.bpc > 1 for image in page_context.pageinfo.images):
- raise NotImplementedError("--remove-background is temporarily not implemented")
- # output_file = page_context.get_path('pp_rm_bg.png')
- # leptonica.remove_background(input_file, output_file)
- # return output_file
+ # raise NotImplementedError("--remove-background is temporarily not implemented")
+ output_file = page_context.get_path('pp_rm_bg.png')
+ leptonica.remove_background(input_file, output_file)
+ return output_file
log.info("background removal skipped on mono page")
return input_file
diff --git a/src/ocrmypdf/leptonica.py b/src/ocrmypdf/leptonica.py
new file mode 100644
index 00000000..e4814f1a
--- /dev/null
+++ b/src/ocrmypdf/leptonica.py
@@ -0,0 +1,1019 @@
+#!/usr/bin/env python3
+#
+# © 2013-16: jbarlow83 from Github (https://github.com/jbarlow83)
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#
+# Python FFI wrapper for Leptonica library
+
+import argparse
+import logging
+import os
+import platform
+import sys
+import threading
+from collections import deque
+from collections.abc import Sequence
+from contextlib import suppress
+from ctypes.util import find_library
+from functools import lru_cache
+from io import BytesIO, UnsupportedOperation
+from os import fspath
+from tempfile import TemporaryFile
+from typing import ContextManager, Type
+from warnings import warn
+
+from ocrmypdf.exceptions import MissingDependencyError
+from ocrmypdf.lib._leptonica import ffi
+
+# pylint: disable=protected-access
+
+logger = logging.getLogger(__name__)
+
+if os.name == 'nt':
+ from ocrmypdf.subprocess._windows import shim_env_path
+
+ libname = 'liblept-5'
+ os.environ['PATH'] = shim_env_path()
+else:
+ libname = 'lept'
+_libpath = find_library(libname)
+if not _libpath:
+ raise MissingDependencyError(
+ """
+ ---------------------------------------------------------------------
+ This error normally occurs when ocrmypdf can't find the Leptonica
+ library, which is usually installed with Tesseract OCR. It could be that
+ Tesseract is not installed properly, we can't find the installation
+ on your system PATH environment variable.
+
+ The library we are looking for is usually called:
+ liblept-5.dll (Windows)
+ liblept*.dylib (macOS)
+ liblept*.so (Linux/BSD)
+
+ Please review our installation procedures to find a solution:
+ https://ocrmypdf.readthedocs.io/en/latest/installation.html
+ ---------------------------------------------------------------------
+ """
+ )
+if os.name == 'nt':
+ # On Windows, recent versions of libpng require zlib. We have to make sure
+ # the zlib version being loaded is the same one that libpng was built with.
+ # This tries to import zlib from Tesseract's installation folder, falling back
+ # to find_library() if liblept is being loaded from somewhere else.
+ # Loading zlib from other places could cause a version mismatch
+ _zlib_path = os.path.join(os.path.dirname(_libpath), 'zlib1.dll')
+ if not os.path.exists(_zlib_path):
+ _zlib_path = find_library('zlib') or ''
+ try:
+ zlib = ffi.dlopen(_zlib_path)
+ except ffi.error as e:
+ raise MissingDependencyError(
+ """
+ Could not load the zlib library. It could be that Tesseract is not installed properly,
+ we can't find the installation on your system PATH environment variable.
+ """
+ ) from e
+try:
+ lept = ffi.dlopen(_libpath)
+ lept.setMsgSeverity(lept.L_SEVERITY_WARNING)
+except ffi.error as e:
+ raise MissingDependencyError(
+ f"Leptonica library found at {_libpath}, but we could not access it"
+ ) from e
+
+
+class _LeptonicaErrorTrap_Redirect(ContextManager):
+ """
+ Context manager to trap errors reported by Leptonica < 1.79 or on Apple Silicon.
+
+ Leptonica's error return codes don't provide much information about what
+ went wrong. Leptonica does, however, write more detailed errors to stderr
+ (provided this is not disabled at compile time). The Leptonica source
+ code is very consistent in its use of macros to generate errors.
+
+ This context manager redirects stderr to a temporary file which is then
+ read and parsed for error messages. As a side benefit, debug messages
+ from Leptonica are also suppressed.
+
+ """
+
+ leptonica_lock = threading.Lock()
+
+ def __init__(self):
+ self.tmpfile = None
+ self.copy_of_stderr = -1
+ self.no_stderr = False
+
+ def __enter__(self):
+ self.tmpfile = TemporaryFile()
+
+ # Save the old stderr, and redirect stderr to temporary file
+ self.leptonica_lock.acquire()
+ try:
+ # It would make sense to do sys.stderr.flush() here, but that can deadlock
+ # due to https://bugs.python.org/issue6721. So don't flush. Pretend
+ # there's nothing important in sys.stderr. If the user cared they would
+ # be using Leptonica 1.79 or later anyway to avoid this mess.
+ self.copy_of_stderr = os.dup(sys.stderr.fileno())
+ os.dup2(self.tmpfile.fileno(), sys.stderr.fileno(), inheritable=False)
+ except AttributeError:
+ # We are in some unusual context where our Python process does not
+ # have a sys.stderr. Leptonica still expects to write to file
+ # descriptor 2, so we are going to ensure it is redirected.
+ self.copy_of_stderr = None
+ self.no_stderr = True
+ os.dup2(self.tmpfile.fileno(), 2, inheritable=False)
+ except UnsupportedOperation:
+ self.copy_of_stderr = None
+ except Exception:
+ self.leptonica_lock.release()
+ raise
+ return
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ # Restore old stderr
+ with suppress(AttributeError):
+ sys.stderr.flush()
+
+ if self.copy_of_stderr is not None:
+ os.dup2(self.copy_of_stderr, sys.stderr.fileno())
+ os.close(self.copy_of_stderr)
+ if self.no_stderr:
+ os.close(2)
+
+ # Get data from tmpfile
+ self.tmpfile.seek(0) # Cursor will be at end, so move back to beginning
+ leptonica_output = self.tmpfile.read().decode(errors='replace')
+ self.tmpfile.close()
+ self.leptonica_lock.release()
+
+ # If there are Python errors, record them
+ if exc_type:
+ logger.warning(leptonica_output)
+
+ # If there are Leptonica errors, wrap them in Python excpetions
+ if 'Error' in leptonica_output:
+ if 'image file not found' in leptonica_output:
+ raise FileNotFoundError()
+ if 'pixWrite: stream not opened' in leptonica_output:
+ raise LeptonicaIOError()
+ if 'index not valid' in leptonica_output:
+ raise IndexError()
+ raise LeptonicaError(leptonica_output)
+
+ return False
+
+
+tls = threading.local()
+tls.trap = None
+
+
+class _LeptonicaErrorTrap_Queue(ContextManager):
+ def __init__(self):
+ self.queue = deque()
+
+ def __enter__(self):
+ self.queue.clear()
+ tls.trap = self.queue
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ tls.trap = None
+ output = ''.join(self.queue)
+ self.queue.clear()
+
+ # If there are Python errors, record them
+ if exc_type:
+ logger.warning(output)
+
+ if 'Error' in output:
+ if 'image file not found' in output:
+ raise FileNotFoundError()
+ elif 'pixWrite: stream not opened' in output:
+ raise LeptonicaIOError()
+ elif 'index not valid' in output:
+ raise IndexError()
+ elif 'pixGetInvBackgroundMap: w and h must be >= 5' in output:
+ logger.warning(
+ "Leptonica attempted to remove background from a low resolution - "
+ "you may want to review in a PDF viewer"
+ )
+ else:
+ raise LeptonicaError(output)
+ return False
+
+
+try:
+
+ @ffi.callback("void(char *)")
+ def _stderr_handler(cstr):
+ msg = ffi.string(cstr).decode(errors='replace')
+ if msg.startswith("Error"):
+ logger.error(msg)
+ elif msg.startswith("Warning"):
+ logger.warning(msg)
+ else:
+ logger.debug(msg)
+ if tls.trap is not None:
+ tls.trap.append(msg)
+ return
+
+ lept.leptSetStderrHandler(_stderr_handler)
+except (ffi.error, MemoryError):
+ # Pre-1.79 Leptonica does not have leptSetStderrHandler
+ # And some platforms, notably Apple ARM 64, do not allow the write+execute
+ # memory needed to set up the callback function.
+ _LeptonicaErrorTrap: Type[ContextManager] = _LeptonicaErrorTrap_Redirect
+else:
+ # 1.79 have this new symbol
+ _LeptonicaErrorTrap = _LeptonicaErrorTrap_Queue
+
+
+class LeptonicaError(Exception):
+ pass
+
+
+class LeptonicaIOError(LeptonicaError):
+ pass
+
+
+class LeptonicaObject:
+ """General wrapper for Leptonica objects
+
+ When Leptonica returns an object, we bundled it in a wrapper class, which
+ manages its memory. The wrapper class assumes that it will be calling some
+ sort of lept.thingDestroy() function when the instance is deleted. Most
+ Leptonica objects are reference counted, and destroy decrements the
+ refcount.
+
+ Most of the time, when Leptonica returns something, we wrap and it the job
+ is done. When wrapping objects that came from a Leptonica container, like
+ a PIXA returning PIX, the subclass must clone the object before passing it
+ here, to maintain the reference count.
+
+ CFFI ensures that the destroy function is called at garbage collection time
+ so we do not need to mess with __del__.
+ """
+
+ cdata_destroy = lambda cdata: None
+ LEPTONICA_TYPENAME = ''
+
+ def __init__(self, cdata):
+ if not cdata:
+ raise ValueError('Tried to wrap a NULL ' + self.LEPTONICA_TYPENAME)
+ self._cdata = ffi.gc(cdata, self._destroy)
+
+ @classmethod
+ def _destroy(cls, cdata):
+ """Destroy some cdata"""
+ # Leptonica API uses double-pointers for its destroy APIs to prevent
+ # dangling pointers. This means we need to put our single pointer,
+ # cdata, in a temporary CDATA**.
+ pp = ffi.new(f'{cls.LEPTONICA_TYPENAME} **', cdata)
+ cls.cdata_destroy(pp)
+
+
+class Pix(LeptonicaObject):
+ """
+ Wrapper around leptonica's PIX object.
+
+ Leptonica uses referencing counting on PIX objects. Also, many Leptonica
+ functions return the original object with an increased reference count
+ if the operation had no effect (for example, image skew was found to be 0).
+ This has complications for memory management in Python. Whenever Leptonica
+ returns a PIX object (new or old), we wrap it in this class, which
+ registers it with the FFI garbage collector. pixDestroy() decrements the
+ reference count and only destroys when the last reference is removed.
+
+ Leptonica's reference counting is not threadsafe. This class can be used
+ in a threadsafe manner if a Python threading.Lock protects the data.
+
+ This class treats Pix objects as immutable. All methods return new
+ modified objects. This allows convenient chaining:
+
+ >>> Pix.open('filename.jpg').scale((0.5, 0.5)).deskew().show()
+ """
+
+ LEPTONICA_TYPENAME = "PIX"
+ cdata_destroy = lept.pixDestroy
+
+ def __repr__(self):
+ if self._cdata:
+ s = "<leptonica.Pix image size={0}x{1} depth={2}{4} at 0x{3:x}>"
+ return s.format(
+ self._cdata.w,
+ self._cdata.h,
+ self._cdata.d,
+ int(ffi.cast('intptr_t', self._cdata)),
+ '(colormapped)' if self._cdata.colormap else '',
+ )
+ else:
+ return "<leptonica.Pix image NULL>"
+
+ def _repr_png_(self):
+ """iPython display hook
+
+ returns png version of image
+ """
+
+ data = ffi.new('l_uint8 **')
+ size = ffi.new('size_t *')
+
+ err = lept.pixWriteMemPng(data, size, self._cdata, 0)
+ if err != 0:
+ raise LeptonicaIOError("pixWriteMemPng")
+
+ char_data = ffi.cast('char *', data[0])
+ return ffi.buffer(char_data, size[0])[:]
+
+ def __getstate__(self):
+ data = ffi.new('l_uint32 **')
+ size = ffi.new('size_t *')
+
+ err = lept.pixSerializeToMemory(self._cdata, data, size)
+ if err != 0:
+ raise LeptonicaIOError("pixSerializeToMemory")
+
+ char_data = ffi.cast('char *', data[0])
+
+ # Copy from C bytes to python bytes()
+ data_bytes = ffi.buffer(char_data, size[0])[:]
+
+ # Can now free C bytes
+ lept.lept_free(char_data)
+ return dict(data=data_bytes)
+
+ def __setstate__(self, state):
+ cdata_bytes = ffi.new('char[]', state['data'])
+ cdata_uint32 = ffi.cast('l_uint32 *', cdata_bytes)
+
+ pix = lept.pixDeserializeFromMemory(cdata_uint32, len(state['data']))
+ Pix.__init__(self, pix)
+
+ def __eq__(self, other):
+ if not isinstance(other, Pix):
+ return NotImplemented
+ same = ffi.new('l_int32 *', 0)
+ with _LeptonicaErrorTrap():
+ err = lept.pixEqual(self._cdata, other._cdata, same)
+ if err:
+ raise TypeError()
+ return bool(same[0])
+
+ @property
+ def width(self):
+ return self._cdata.w
+
+ @property
+ def height(self):
+ return self._cdata.h
+
+ @property
+ def depth(self):
+ return self._cdata.d
+
+ @property
+ def size(self):
+ return (self._cdata.w, self._cdata.h)
+
+ @property
+ def info(self):
+ return {'dpi': (self._cdata.xres, self._cdata.yres)}
+
+ @property
+ def mode(self):
+ "Return mode like PIL.Image"
+ if self.depth == 1:
+ return '1'
+ elif self.depth >= 16:
+ return 'RGB'
+ elif not self._cdata.colormap:
+ return 'L'
+ else:
+ return 'P'
+
+ @classmethod
+ def read(cls, path):
+ warn('Use Pix.open() instead', DeprecationWarning)
+ return cls.open(path)
+
+ @classmethod
+ def open(cls, path):
+ """Load an image file into a PIX object.
+
+ Leptonica can load TIFF, PNM (PBM, PGM, PPM), PNG, and JPEG. If
+ loading fails then the object will wrap a C null pointer.
+ """
+ with open(path, 'rb') as py_file:
+ data = py_file.read()
+ buffer = ffi.from_buffer(data)
+ with _LeptonicaErrorTrap():
+ return cls(lept.pixReadMem(buffer, len(buffer)))
+
+ def write_implied_format(self, path, jpeg_quality=0, jpeg_progressive=0):
+ """Write pix to the filename, with the extension indicating format.
+
+ jpeg_quality -- quality (iff JPEG; 1 - 100, 0 for default)
+ jpeg_progressive -- (iff JPEG; 0 for baseline seq., 1 for progressive)
+ """
+ lept_format = lept.getImpliedFileFormat(os.fsencode(path))
+ with open(path, 'wb') as py_file:
+ data = ffi.new('l_uint8 **pdata')
+ size = ffi.new('size_t *psize')
+ with _LeptonicaErrorTrap():
+ if lept_format == lept.L_JPEG_ENCODE:
+ lept.pixWriteMemJpeg(
+ data, size, self._cdata, jpeg_quality, jpeg_progressive
+ )
+ else:
+ lept.pixWriteMem(data, size, self._cdata, lept_format)
+ buffer = ffi.buffer(data[0], size[0])
+ py_file.write(buffer)
+
+ @classmethod
+ def frompil(cls, pillow_image):
+ """Create a copy of a PIL.Image from this Pix"""
+ bio = BytesIO()
+ pillow_image.save(bio, format='png', compress_level=1)
+ py_buffer = bio.getbuffer()
+ if platform.python_implementation() == 'PyPy':
+ # PyPy complains that it cannot do from_buffer(memoryview)
+ py_buffer = bytes(py_buffer)
+ c_buffer = ffi.from_buffer(py_buffer)
+ with _LeptonicaErrorTrap():
+ pix = Pix(lept.pixReadMem(c_buffer, len(c_buffer)))
+ return pix
+
+ def topil(self):
+ """Returns a PIL.Image version of this Pix"""
+ from PIL import Image # pylint: disable=import-outside-toplevel
+
+ # Leptonica manages data in words, so it implicitly does an endian
+ # swap. Tell Pillow about this when it reads the data.
+ pix = self
+ if sys.byteorder == 'little':
+ if self.mode == 'RGB':
+ raw_mode = 'XBGR'
+ elif self.mode == 'RGBA':
+ raw_mode = 'ABGR'
+ elif self.mode == '1':
+ raw_mode = '1;I'
+ pix = Pix(lept.pixEndianByteSwapNew(pix._cdata))
+ else:
+ raw_mode = self.mode
+ pix = Pix(lept.pixEndianByteSwapNew(pix._cdata))
+ else:
+ raw_mode = self.mode # no endian swap needed
+
+ size = (pix._cdata.w, pix._cdata.h)
+ bytecount = pix._cdata.wpl * 4 * pix._cdata.h
+ buf = ffi.buffer(pix._cdata.data, bytecount)
+ stride = pix._cdata.wpl * 4
+
+ im = Image.frombytes(self.mode, size, buf, 'raw', raw_mode, stride)
+
+ return im
+
+ def show(self):
+ return self.topil().show()
+
+ def deskew(self, reduction_factor=0):
+ """Returns the deskewed pix object.
+
+ A clone of the original is returned when the algorithm cannot find a
+ skew angle with sufficient confidence.
+
+ reduction_factor -- amount to downsample (0 for default) when searching
+ for skew angle
+ """
+ with _LeptonicaErrorTrap():
+ return Pix(lept.pixDeskew(self._cdata, reduction_factor))
+
+ def scale(self, scale_xy):
+ "Returns the pix object rescaled according to the proportions given."
+ with _LeptonicaErrorTrap():
+ return Pix(lept.pixScale(self._cdata, scale_xy[0], scale_xy[1]))
+
+ def rotate180(self):
+ with _LeptonicaErrorTrap():
+ return Pix(lept.pixRotate180(ffi.NULL, self._cdata))
+
+ def rotate_orth(self, quads):
+ "Orthographic rotation, quads: 0-3, number of clockwise rotations"
+ with _LeptonicaErrorTrap():
+ return Pix(lept.pixRotateOrth(self._cdata, quads))
+
+ def find_skew(self):
+ """Returns a tuple (deskew angle in degrees, confidence value).
+
+ Returns (None, None) if no angle is available.
+ """
+ with _LeptonicaErrorTrap():
+ angle = ffi.new('float *', 0.0)
+ confidence = ffi.new('float *', 0.0)
+ result = lept.pixFindSkew(self._cdata, angle, confidence)
+ if result == 0:
+ return (angle[0], confidence[0])
+ else:
+ return (None, None)
+
+ def convert_rgb_to_luminance(self):
+ with _LeptonicaErrorTrap():
+ gray_pix = lept.pixConvertRGBToLuminance(self._cdata)
+ if gray_pix:
+ return Pix(gray_pix)
+ return None
+
+ def remove_colormap(self, removal_type):
+ """Remove a palette (colormap); if no colormap, returns a copy of this
+ image
+
+ removal_type - any of lept.REMOVE_CMAP_*
+
+ """
+ with _LeptonicaErrorTrap():
+ return Pix(
+ lept.pixRemoveColormapGeneral(self._cdata, removal_type, lept.L_COPY)
+ )
+
+ def otsu_adaptive_threshold(
+ self, tile_size=(300, 300), kernel_size=(4, 4), scorefract=0.1
+ ):
+ with _LeptonicaErrorTrap():
+ sx, sy = tile_size
+ smoothx, smoothy = kernel_size
+ p_pix = ffi.new('PIX **')
+
+ pix = Pix(lept.pixConvertTo8(self._cdata, 0))
+ result = lept.pixOtsuAdaptiveThreshold(
+ pix._cdata, sx, sy, smoothx, smoothy, scorefract, ffi.NULL, p_pix
+ )
+ if result == 0:
+ return Pix(p_pix[0])
+ else:
+ return None
+
+ def otsu_threshold_on_background_norm(
+ self,
+ mask=None,
+ tile_size=(10, 15),
+ thresh=100,
+ mincount=50,
+ bgval=255,
+ kernel_size=(2, 2),
+ scorefract=0.1,
+ ):
+ with _LeptonicaErrorTrap():
+ sx, sy = tile_size
+ smoothx, smoothy = kernel_size
+ mask = ffi.NULL
+ if isinstance(mask, Pix):
+ mask = mask._cdata
+
+ pix = Pix(lept.pixConvertTo8(self._cdata, 0))
+ thresh_pix = lept.pixOtsuThreshOnBackgroundNorm(
+ pix._cdata,
+ mask,
+ sx,
+ sy,
+ thresh,
+ mincount,
+ bgval,
+ smoothx,
+ smoothy,
+ scorefract,
+ ffi.NULL,
+ )
+ return Pix(thresh_pix)
+
+ def masked_threshold_on_background_norm(
+ self,
+ mask=None,
+ tile_size=(10, 15),
+ thresh=100,
+ mincount=50,
+ kernel_size=(2, 2),
+ scorefract=0.1,
+ ):
+ with _LeptonicaErrorTrap():
+ sx, sy = tile_size
+ smoothx, smoothy = kernel_size
+ mask = ffi.NULL
+ if isinstance(mask, Pix):
+ mask = mask._cdata
+
+ pix = Pix(lept.pixConvertTo8(self._cdata, 0))
+ thresh_pix = lept.pixMaskedThreshOnBackgroundNorm(
+ pix._cdata,
+ mask,
+ sx,
+ sy,
+ thresh,
+ mincount,
+ smoothx,
+ smoothy,
+ scorefract,
+ ffi.NULL,
+ )
+ return Pix(thresh_pix)
+
+ def crop_to_foreground(self, threshold=128, mindist=70, erasedist=30, showmorph=0):
+ if get_leptonica_version() < 'leptonica-1.76':
+ # Leptonica 1.76 changed the API for pixFindPageForeground; we don't
+ # support the old version
+ raise LeptonicaError("Not available in this version of Leptonica")
+ with _LeptonicaErrorTrap():
+ cropbox = Box(
+ lept.pixFindPageForeground(
+ self._cdata, threshold, mindist, erasedist, showmorph, ffi.NULL
+ )
+ )
+
+ cropped_pix = lept.pixClipRectangle(self._cdata, cropbox._cdata, ffi.NULL)
+
+ return Pix(cropped_pix)
+
+ def clean_background_to_white(
+ self, mask=None, grayscale=None, gamma=1.0, black=0, white=255
+ ):
+ with _LeptonicaErrorTrap():
+ return Pix(
+ lept.pixCleanBackgroundToWhite(
+ self._cdata,
+ mask or ffi.NULL,
+ grayscale or ffi.NULL,
+ gamma,
+ black,
+ white,
+ )
+ )
+
+ def gamma_trc(self, gamma=1.0, minval=0, maxval=255):
+ with _LeptonicaErrorTrap():
+ return Pix(lept.pixGammaTRC(ffi.NULL, self._cdata, gamma, minval, maxval))
+
+ def background_norm(
+ self,
+ mask=None,
+ grayscale=None,
+ tile_size=(10, 15),
+ fg_threshold=60,
+ min_count=40,
+ bg_val=200,
+ smooth_kernel=(2, 1),
+ ):
+ if self.width < tile_size[0] or self.height < tile_size[1]:
+ logger.info("Skipped pixMaskedThreshOnBackgroundNorm on small image")
+ return self
+ # Background norm doesn't work on color mapped Pix, so remove colormap
+ target_pix = self.remove_colormap(lept.REMOVE_CMAP_BASED_ON_SRC)
+ with _LeptonicaErrorTrap():
+ return Pix(
+ lept.pixBackgroundNorm(
+ target_pix._cdata,
+ mask or ffi.NULL,
+ grayscale or ffi.NULL,
+ tile_size[0],
+ tile_size[1],
+ fg_threshold,
+ min_count,
+ bg_val,
+ smooth_kernel[0],
+ smooth_kernel[1],
+ )
+ )
+
+ @staticmethod
+ @lru_cache(maxsize=1)
+ def make_pixel_sum_tab8():
+ return lept.makePixelSumTab8()
+
+ @staticmethod
+ def correlation_binary(pix1, pix2):
+ if get_leptonica_version() < 'leptonica-1.72':
+ # Older versions of Leptonica (pre-1.72) have a buggy
+ # implementation of pixCorrelationBinary that overflows on larger
+ # images. Ubuntu 14.04/trusty has 1.70. Ubuntu PPA
+ # ppa:alex-p/tesseract-ocr has leptonlib 1.75.
+ raise LeptonicaError("Leptonica version is too old")
+
+ correlation = ffi.new('float *', 0.0)
+ result = lept.pixCorrelationBinary(pix1._cdata, pix2._cdata, correlation)
+ if result != 0:
+ raise LeptonicaError("Correlation failed")
+ return correlation[0]
+
+ def generate_pdf_ci_data(self, type_, quality):
+ "Convert to PDF data, with transcoding"
+ p_compdata = ffi.new('L_COMP_DATA **')
+ result = lept.pixGenerateCIData(self._cdata, type_, quality, 0, p_compdata)
+ if result != 0:
+ raise LeptonicaError("Generate PDF data failed")
+ return CompressedData(p_compdata[0])
+
+ def invert(self):
+ return Pix(lept.pixInvert(ffi.NULL, self._cdata))
+
+ def locate_barcodes(self):
+ try:
+ with _LeptonicaErrorTrap():
+ pix = Pix(lept.pixConvertTo8(self._cdata, 0))
+ pixa_candidates = PixArray(lept.pixExtractBarcodes(pix._cdata, 0))
+ if not pixa_candidates:
+ return
+ sarray = StringArray(
+ lept.pixReadBarcodes(
+ pixa_candidates._cdata,
+ lept.L_BF_ANY,
+ lept.L_USE_WIDTHS,
+ ffi.NULL,
+ 0,
+ )
+ )
+ except (LeptonicaError, ValueError, IndexError):
+ return
+ finally:
+ leptonica_junk = ('junkpixt.png', 'junkpixt')
+ for junk in leptonica_junk:
+ with suppress(FileNotFoundError):
+ os.unlink(junk) # leptonica may produce this
+
+ for n, s in enumerate(sarray):
+ decoded = s.decode()
+ if decoded.strip() == '':
+ continue
+ box = pixa_candidates.get_box(n)
+ left, top = box.x, box.y
+ right, bottom = box.x + box.w, box.y + box.h
+ yield (decoded, (left, top, right, bottom))
+
+ def despeckle(self, size):
+ if size == 2:
+ speckle2 = """
+ oooo
+ oC o
+ o o
+ oooo
+ """
+ sel1 = Sel.from_selstr(speckle2, 'speckle2')
+ sel2 = Sel.create_brick(2, 2, 0, 0, lept.SEL_HIT)
+ elif size == 3:
+ speckle3 = """
+ ooooo
+ oC o
+ o o
+ o o
+ ooooo
+ """
+ sel1 = Sel.from_selstr(speckle3, 'speckle3')
+ sel2 = Sel.create_brick(3, 3, 0, 0, lept.SEL_HIT)
+ else:
+ raise ValueError(size)
+
+ pixhmt = Pix(lept.pixHMT(ffi.NULL, self._cdata, sel1._cdata))
+ pixdilated = Pix(lept.pixDilate(ffi.NULL, pixhmt._cdata, sel2._cdata))
+
+ pixsub = Pix(lept.pixSubtract(ffi.NULL, self._cdata, pixdilated._cdata))
+ return pixsub
+
+
+class CompressedData(LeptonicaObject):
+ """Wrapper for L_COMP_DATA - abstract compressed image data"""
+
+ LEPTONICA_TYPENAME = 'L_COMP_DATA'
+ cdata_destroy = lept.l_CIDataDestroy
+
+ @classmethod
+ def open(cls, path, jpeg_quality=75):
+ "Open compressed data, without transcoding"
+ filename = fspath(path)
+
+ p_compdata = ffi.new('L_COMP_DATA **')
+ result = lept.l_generateCIDataForPdf(
+ os.fsencode(filename), ffi.NULL, jpeg_quality, p_compdata
+ )
+ if result != 0:
+ raise LeptonicaError("CompressedData.open")
+ return CompressedData(p_compdata[0])
+
+ def __len__(self):
+ return self._cdata.nbytescomp
+
+ def read(self):
+ buf = ffi.buffer(self._cdata.datacomp, self._cdata.nbytescomp)
+ return bytes(buf)
+
+ def __getattr__(self, name):
+ if hasattr(self._cdata, name):
+ return getattr(self._cdata, name)
+ raise AttributeError(name)
+
+ def get_palette_pdf_string(self):
+ "Returns palette pre-formatted for use in PDF"
+ buflen = len('< ') + len(' rrggbb') * self._cdata.ncolors + len('>')
+ buf = ffi.buffer(self._cdata.cmapdatahex, buflen)
+ return bytes(buf)
+
+
+class PixArray(LeptonicaObject, Sequence):
+ """Wrapper around PIXA (array of PIX)"""
+
+ LEPTONICA_TYPENAME = 'PIXA'
+ cdata_destroy = lept.pixaDestroy
+
+ def __len__(self):
+ return self._cdata[0].n
+
+ def __getitem__(self, n):
+ with _LeptonicaErrorTrap():
+ return Pix(lept.pixaGetPix(self._cdata, n, lept.L_CLONE))
+
+ def get_box(self, n):
+ with _LeptonicaErrorTrap():
+ return Box(lept.pixaGetBox(self._cdata, n, lept.L_CLONE))
+
+
+class Box(LeptonicaObject):
+ """Wrapper around Leptonica's BOX objects (a pixel rectangle)
+
+ Uses x, y, w, h coordinates.
+ """
+
+ LEPTONICA_TYPENAME = 'BOX'
+ cdata_destroy = lept.boxDestroy
+
+ def __repr__(self):
+ if self._cdata:
+ return '<leptonica.Box x={} y={} w={} h={}>'.format(
+ self.x, self.y, self.w, self.h
+ )
+ return '<leptonica.Box NULL>'
+
+ @property
+ def x(self):
+ return self._cdata.x
+
+ @property
+ def y(self):
+ return self._cdata.y
+
+ @property
+ def w(self):
+ return self._cdata.w
+
+ @property
+ def h(self):
+ return self._cdata.h
+
+
+class BoxArray(LeptonicaObject, Sequence):
+ """Wrapper around Leptonica's BOXA (Array of BOX) objects."""
+
+ LEPTONICA_TYPENAME = 'BOXA'
+ cdata_destroy = lept.boxaDestroy
+
+ def __repr__(self):
+ if not self._cdata:
+ return '<BoxArray>'
+ boxes = (repr(box) for box in self)
+ return '<BoxArray [' + ', '.join(boxes) + ']>'
+
+ def __len__(self):
+ return self._cdata.n
+
+ def __getitem__(self, n):
+ if not isinstance(n, int):
+ raise TypeError('list indices must be integers')
+ if 0 <= n < len(self):
+ return Box(lept.boxaGetBox(self._cdata, n, lept.L_CLONE))
+ raise IndexError(n)
+
+
+class StringArray(LeptonicaObject, Sequence):
+ """Leptonica SARRAY/string array"""
+
+ LEPTONICA_TYPENAME = 'SARRAY'
+ cdata_destroy = lept.sarrayDestroy
+
+ def __len__(self):
+ return self._cdata.n
+
+ def __getitem__(self, n):
+ if 0 <= n < len(self):
+ return ffi.string(self._cdata.array[n])
+ raise IndexError(n)
+
+
+class Sel(LeptonicaObject):
+ """Leptonica 'sel'/selection element for hit-miss transform"""
+
+ LEPTONICA_TYPENAME = 'SEL'
+ cdata_destroy = lept.selDestroy
+
+ @classmethod
+ def from_selstr(cls, selstr, name):
+ # TODO this will strip a horizontal line of don't care's
+ lines = [line.strip() for line in selstr.split('\n') if line.strip()]
+ h = len(lines)
+ w = len(lines[0])
+ lengths = {len(line) for line in lines}
+ if len(lengths) != 1:
+ raise ValueError("All lines in selstr must be same length")
+
+ repacked = ''.join(line.strip() for line in lines)
+ buf_selstr = ffi.from_buffer(repacked.encode('ascii'))
+ buf_name = ffi.from_buffer(name.encode('ascii'))
+ sel = lept.selCreateFromString(buf_selstr, h, w, buf_name)
+ return cls(sel)
+
+ @classmethod
+ def create_brick(cls, h, w, cy, cx, type_):
+ sel = lept.selCreateBrick(h, w, cy, cx, type_)
+ return cls(sel)
+
+ def __repr__(self):
+ selstr = ffi.gc(lept.selPrintToString(self._cdata), lept.lept_free)
+ return '<Sel \n' + ffi.string(selstr).decode('ascii') + '\n>'
+
+
+@lru_cache(maxsize=1)
+def get_leptonica_version():
+ """Get Leptonica version string.
+
+ Caveat: Leptonica expects the caller to free this memory. We don't,
+ since that would involve binding to libc to access libc.free(),
+ a pointless effort to reclaim 100 bytes of memory.
+
+ Reminder that this returns "leptonica-1.xx" or "leptonica-1.yy.0".
+ """
+ return ffi.string(lept.getLeptonicaVersion()).decode()
+
+
+def deskew(infile, outfile, dpi):
+ try:
+ pix_source = Pix.open(infile)
+ except LeptonicaIOError:
+ raise LeptonicaIOError("Failed to open file: %s" % infile)
+
+ if dpi < 150:
+ reduction_factor = 1 # Don't downsample too much if DPI is already low
+ else:
+ reduction_factor = 0 # Use default
+ pix_deskewed = pix_source.deskew(reduction_factor)
+
+ try:
+ pix_deskewed.write_implied_format(outfile)
+ except LeptonicaIOError:
+ raise LeptonicaIOError("Failed to open destination file: %s" % outfile)
+
+
+def remove_background(
+ infile,
+ outfile,
+ tile_size=(40, 60),
+ gamma=1.0,
+ black_threshold=70,
+ white_threshold=190,
+):
+ try:
+ pix = Pix.open(infile)
+ except LeptonicaIOError:
+ raise LeptonicaIOError("Failed to open file: %s" % infile)
+
+ pix = pix.background_norm(tile_size=tile_size).gamma_trc(
+ gamma, black_threshold, white_threshold
+ )
+
+ try:
+ pix.write_implied_format(outfile)
+ except LeptonicaIOError:
+ raise LeptonicaIOError("Failed to open destination file: %s" % outfile)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description="Python wrapper to access Leptonica")
+
+ subparsers = parser.add_subparsers(
+ title='commands', description='supported operations'
+ )
+
+ parser_deskew = subparsers.add_parser('deskew')
+ parser_deskew.add_argument(
+ '-r',
+ '--dpi',
+ dest='dpi',
+ action='store',
+ type=int,
+ default=300,
+ help='input resolution',
+ )
+ parser_deskew.add_argument('infile', help='image to deskew')
+ parser_deskew.add_argument('outfile', help='deskewed output image')
+ parser_deskew.set_defaults(func=deskew)
+
+ args = parser.parse_args()
+ args.func(args)
diff --git a/src/ocrmypdf/lib/__init__.py b/src/ocrmypdf/lib/__init__.py
new file mode 100644
index 00000000..45460814
--- /dev/null
+++ b/src/ocrmypdf/lib/__init__.py
@@ -0,0 +1,8 @@
+# © 2017 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+"""Bindings to external libraries"""
diff --git a/src/ocrmypdf/lib/_leptonica.py b/src/ocrmypdf/lib/_leptonica.py
new file mode 100644
index 00000000..996d5f8a
--- /dev/null
+++ b/src/ocrmypdf/lib/_leptonica.py
@@ -0,0 +1,11 @@
+# auto-generated file
+import _cffi_backend
+
+ffi = _cffi_backend.FFI('ocrmypdf.lib._leptonica',
+ _version = 0x2601,
+ _types = b'\x00\x00\x01\x0D\x00\x01\x5C\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x5D\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x61\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x63\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x62\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x04\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x09\x03\x00\x00\x18\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x01\x5E\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x01\x11\x00\x00\x01\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x18\x11\x00\x00\x05\x03\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x01\x67\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x2A\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x2A\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x01\x6A\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x01\x7C\x03\x00\x00\x1C\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x01\x7E\x03\x00\x00\x1C\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x65\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x65\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x65\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xA4\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x4D\x0D\x00\x00\x92\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x92\x11\x00\x00\x00\x0F\x00\x00\x4D\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x69\x0D\x00\x00\x4D\x11\x00\x00\x00\x0F\x00\x01\x69\x0D\x00\x00\x00\x0F\x00\x00\x2A\x0D\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x1C\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x1C\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x04\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x3A\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x2A\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xD6\x11\x00\x00\xD6\x11\x00\x00\xD6\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\xD6\x11\x00\x00\xD6\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x2A\x11\x00\x00\x2A\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x2A\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x5F\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xD6\x11\x00\x00\xD6\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x18\x11\x00\x00\x18\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x01\x7D\x03\x00\x00\x96\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x92\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x92\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\xFF\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x92\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x7B\x03\x00\x01\x17\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x2C\x11\x00\x01\x17\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x2C\x11\x00\x01\x17\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x81\x0D\x00\x00\x25\x11\x00\x00\x00\x0F\x00\x01\x81\x0D\x00\x00\x04\x03\x00\x00\x00\x0F\x00\x01\x81\x0D\x00\x00\xFF\x11\x00\x00\x00\x0F\x00\x01\x81\x0D\x00\x00\x18\x11\x00\x00\x00\x0F\x00\x01\x81\x0D\x00\x00\x11\x03\x00\x00\x00\x0F\x00\x01\x81\x0D\x00\x00\xA4\x11\x00\x00\x00\x0F\x00\x01\x81\x0D\x00\x00\x4D\x03\x00\x00\x00\x0F\x00\x01\x81\x0D\x00\x00\x92\x11\x00\x00\x00\x0F\x00\x01\x81\x0D\x00\x01\x81\x03\x00\x00\x00\x0F\x00\x01\x81\x0D\x00\x01\x53\x03\x00\x00\x00\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x0A\x09\x00\x01\x60\x03\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x06\x09\x00\x00\x07\x09\x00\x00\x04\x09\x00\x01\x66\x03\x00\x00\x08\x09\x00\x00\x09\x09\x00\x01\x69\x03\x00\x01\x6A\x03\x00\x00\x02\x01\x00\x00\x0E\x01\x00\x00\x00\x0B\x00\x00\x01\x0B\x00\x00\x02\x0B\x00\x00\x03\x0B\x00\x00\x04\x0B\x00\x00\x05\x0B\x00\x00\x06\x0B\x00\x00\x2A\x03\x00\x00\x0B\x01\x00\x00\x05\x01\x00\x00\x03\x01\x00\x01\x64\x03\x00\x01\x79\x03\x00\x01\x7A\x03\x00\x00\x05\x09\x00\x01\x7C\x03\x00\x00\x04\x01\x00\x01\x7E\x03\x00\x00\x08\x01\x00\x00\x0C\x01\x00\x00\x06\x01\x00\x00\x00\x01',
+ _globals = (b'\xFF\xFF\xFF\x0BL_BF_ANY',1,b'\xFF\xFF\xFF\x0BL_BF_CODABAR',9,b'\xFF\xFF\xFF\x0BL_BF_CODE128',2,b'\xFF\xFF\xFF\x0BL_BF_CODE2OF5',5,b'\xFF\xFF\xFF\x0BL_BF_CODE39',7,b'\xFF\xFF\xFF\x0BL_BF_CODE93',8,b'\xFF\xFF\xFF\x0BL_BF_CODEI2OF5',6,b'\xFF\xFF\xFF\x0BL_BF_EAN13',4,b'\xFF\xFF\xFF\x0BL_BF_EAN8',3,b'\xFF\xFF\xFF\x0BL_BF_UNKNOWN',0,b'\xFF\xFF\xFF\x0BL_BF_UPCA',10,b'\xFF\xFF\xFF\x0BL_CLONE',2,b'\xFF\xFF\xFF\x0BL_COPY',1,b'\xFF\xFF\xFF\x0BL_COPY_CLONE',3,b'\xFF\xFF\xFF\x0BL_DEFAULT_ENCODE',0,b'\xFF\xFF\xFF\x0BL_FLATE_ENCODE',3,b'\xFF\xFF\xFF\x0BL_G4_ENCODE',2,b'\xFF\xFF\xFF\x0BL_INSERT',0,b'\xFF\xFF\xFF\x0BL_JP2K_ENCODE',4,b'\xFF\xFF\xFF\x0BL_JPEG_ENCODE',1,b'\xFF\xFF\xFF\x0BL_NOCOPY',0,b'\xFF\xFF\xFF\x0BL_SEVERITY_ALL',1,b'\xFF\xFF\xFF\x0BL_SEVERITY_DEBUG',2,b'\xFF\xFF\xFF\x0BL_SEVERITY_ERROR',5,b'\xFF\xFF\xFF\x0BL_SEVERITY_EXTERNAL',0,b'\xFF\xFF\xFF\x0BL_SEVERITY_INFO',3,b'\xFF\xFF\xFF\x0BL_SEVERITY_NONE',6,b'\xFF\xFF\xFF\x0BL_SEVERITY_WARNING',4,b'\xFF\xFF\xFF\x0BL_USE_WIDTHS',1,b'\xFF\xFF\xFF\x0BL_USE_WINDOWS',2,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_BASED_ON_SRC',4,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_BINARY',0,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_FULL_COLOR',2,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_GRAYSCALE',1,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_WITH_ALPHA',3,b'\xFF\xFF\xFF\x0BSEL_DONT_CARE',0,b'\xFF\xFF\xFF\x0BSEL_HIT',1,b'\xFF\xFF\xFF\x0BSEL_MISS',2,b'\x00\x00\x00\x23boxClone',0,b'\x00\x01\x3E\x23boxDestroy',0,b'\x00\x01\x41\x23boxaDestroy',0,b'\x00\x00\x03\x23boxaGetBox',0,b'\x00\x01\x19\x23getImpliedFileFormat',0,b'\x00\x00\xBE\x23getLeptonicaVersion',0,b'\x00\x01\x44\x23l_CIDataDestroy',0,b'\x00\x01\x1C\x23l_generateCIDataForPdf',0,b'\x00\x01\x59\x23leptSetStderrHandler',0,b'\x00\x01\x56\x23lept_free',0,b'\x00\x00\xC0\x23makePixelSumTab8',0,b'\x00\x00\x31\x23pixAnd',0,b'\x00\x00\x3E\x23pixBackgroundNorm',0,b'\x00\x00\x36\x23pixCleanBackgroundToWhite',0,b'\x00\x00\x22\x23pixClipRectangle',0,b'\x00\x01\x01\x23pixColorFraction',0,b'\x00\x00\x85\x23pixColorMagnitude',0,b'\x00\x00\x1F\x23pixConvertRGBToLuminance',0,b'\x00\x00\x7C\x23pixConvertTo8',0,b'\x00\x00\xD3\x23pixCorrelationBinary',0,b'\x00\x00\xED\x23pixCountPixels',0,b'\x00\x00\x98\x23pixDeserializeFromMemory',0,b'\x00\x00\x7C\x23pixDeskew',0,b'\x00\x01\x47\x23pixDestroy',0,b'\x00\x00\x4A\x23pixDilate',0,b'\x00\x00\x1F\x23pixEndianByteSwapNew',0,b'\x00\x00\xD8\x23pixEqual',0,b'\x00\x00\x4A\x23pixErode',0,b'\x00\x00\x9C\x23pixExtractBarcodes',0,b'\x00\x00\x08\x23pixFindPageForeground',0,b'\x00\x00\xE8\x23pixFindSkew',0,b'\x00\x00\x4F\x23pixGammaTRC',0,b'\x00\x00\x27\x23pixGenHalftoneMask',0,b'\x00\x00\xFA\x23pixGenerateCIData',0,b'\x00\x00\xDD\x23pixGetAverageMaskedRGB',0,b'\x00\x00\x56\x23pixGlobalNormRGB',0,b'\x00\x00\x4A\x23pixHMT',0,b'\x00\x00\x2D\x23pixInvert',0,b'\x00\x00\x15\x23pixLocateBarcodes',0,b'\x00\x00\x80\x23pixMaskOverColorPixels',0,b'\x00\x00\x5E\x23pixMaskedThreshOnBackgroundNorm',0,b'\x00\x00\xF2\x23pixNumSignificantGrayColors',0,b'\x00\x01\x0A\x23pixOtsuAdaptiveThreshold',0,b'\x00\x00\x6A\x23pixOtsuThreshOnBackgroundNorm',0,b'\x00\x00\xA0\x23pixProcessBarcodes',0,b'\x00\x00\x91\x23pixRead',0,b'\x00\x00\xA7\x23pixReadBarcodes',0,b'\x00\x00\x94\x23pixReadMem',0,b'\x00\x00\x1B\x23pixReadStream',0,b'\x00\x00\x7C\x23pixRemoveColormap',0,b'\x00\x00\x80\x23pixRemoveColormapGeneral',0,b'\x00\x00\xCD\x23pixRenderBoxa',0,b'\x00\x00\x2D\x23pixRotate180',0,b'\x00\x00\x7C\x23pixRotateOrth',0,b'\x00\x00\x77\x23pixScale',0,b'\x00\x01\x14\x23pixSerializeToMemory',0,b'\x00\x00\x31\x23pixSubtract',0,b'\x00\x01\x22\x23pixWriteImpliedFormat',0,b'\x00\x01\x31\x23pixWriteMem',0,b'\x00\x01\x37\x23pixWriteMemJpeg',0,b'\x00\x01\x2B\x23pixWriteMemPng',0,b'\x00\x00\xC2\x23pixWriteStream',0,b'\x00\x00\xC7\x23pixWriteStreamJpeg',0,b'\x00\x01\x4A\x23pixaDestroy',0,b'\x00\x00\x10\x23pixaGetBox',0,b'\x00\x00\x8C\x23pixaGetPix',0,b'\x00\x01\x4D\x23sarrayDestroy',0,b'\x00\x00\xB4\x23selCreateBrick',0,b'\x00\x00\xAE\x23selCreateFromString',0,b'\x00\x01\x50\x23selDestroy',0,b'\x00\x00\xBB\x23selPrintToString',0,b'\x00\x01\x28\x23setMsgSeverity',0),
+ _struct_unions = ((b'\x00\x00\x01\x5C\x00\x00\x00\x02Box',b'\x00\x00\x05\x11x',b'\x00\x00\x05\x11y',b'\x00\x00\x05\x11w',b'\x00\x00\x05\x11h',b'\x00\x01\x7E\x11refcount'),(b'\x00\x00\x01\x5D\x00\x00\x00\x02Boxa',b'\x00\x00\x05\x11n',b'\x00\x00\x05\x11nalloc',b'\x00\x01\x7E\x11refcount',b'\x00\x00\x25\x11box'),(b'\x00\x00\x01\x60\x00\x00\x00\x02L_Compressed_Data',b'\x00\x00\x05\x11type',b'\x00\x01\x7B\x11datacomp',b'\x00\x00\x96\x11nbytescomp',b'\x00\x01\x69\x11data85',b'\x00\x00\x96\x11nbytes85',b'\x00\x01\x69\x11cmapdata85',b'\x00\x01\x69\x11cmapdatahex',b'\x00\x00\x05\x11ncolors',b'\x00\x00\x05\x11w',b'\x00\x00\x05\x11h',b'\x00\x00\x05\x11bps',b'\x00\x00\x05\x11spp',b'\x00\x00\x05\x11minisblack',b'\x00\x00\x05\x11predictor',b'\x00\x00\x96\x11nbytes',b'\x00\x00\x05\x11res'),(b'\x00\x00\x01\x61\x00\x00\x00\x02Pix',b'\x00\x01\x7E\x11w',b'\x00\x01\x7E\x11h',b'\x00\x01\x7E\x11d',b'\x00\x01\x7E\x11spp',b'\x00\x01\x7E\x11wpl',b'\x00\x01\x7E\x11refcount',b'\x00\x00\x05\x11xres',b'\x00\x00\x05\x11yres',b'\x00\x00\x05\x11informat',b'\x00\x00\x05\x11special',b'\x00\x01\x69\x11text',b'\x00\x01\x77\x11colormap',b'\x00\x01\x7D\x11data'),(b'\x00\x00\x01\x64\x00\x00\x00\x02PixColormap',b'\x00\x01\x57\x11array',b'\x00\x00\x05\x11depth',b'\x00\x00\x05\x11nalloc',b'\x00\x00\x05\x11n'),(b'\x00\x00\x01\x7A\x00\x00\x00\x10PixComp',),(b'\x00\x00\x01\x62\x00\x00\x00\x02Pixa',b'\x00\x00\x05\x11n',b'\x00\x00\x05\x11nalloc',b'\x00\x01\x7E\x11refcount',b'\x00\x00\x18\x11pix',b'\x00\x00\x04\x11boxa'),(b'\x00\x00\x01\x63\x00\x00\x00\x02PixaComp',b'\x00\x00\x05\x11n',b'\x00\x00\x05\x11nalloc',b'\x00\x00\x05\x11offset',b'\x00\x01\x78\x11pixc',b'\x00\x00\x04\x11boxa'),(b'\x00\x00\x01\x66\x00\x00\x00\x02Sarray',b'\x00\x00\x05\x11nalloc',b'\x00\x00\x05\x11n',b'\x00\x00\x05\x11refcount',b'\x00\x01\x68\x11array'),(b'\x00\x00\x01\x67\x00\x00\x00\x02Sel',b'\x00\x00\x05\x11sy',b'\x00\x00\x05\x11sx',b'\x00\x00\x05\x11cy',b'\x00\x00\x05\x11cx',b'\x00\x01\x73\x11data',b'\x00\x01\x69\x11name'),(b'\x00\x00\x01\x5E\x00\x00\x00\x10_IO_FILE',)),
+ _enums = (b'\x00\x00\x01\x6C\x00\x00\x00\x16$1\x00L_DEFAULT_ENCODE,L_JPEG_ENCODE,L_G4_ENCODE,L_FLATE_ENCODE,L_JP2K_ENCODE',b'\x00\x00\x01\x6D\x00\x00\x00\x16$2\x00REMOVE_CMAP_TO_BINARY,REMOVE_CMAP_TO_GRAYSCALE,REMOVE_CMAP_TO_FULL_COLOR,REMOVE_CMAP_WITH_ALPHA,REMOVE_CMAP_BASED_ON_SRC',b'\x00\x00\x01\x6E\x00\x00\x00\x16$3\x00L_NOCOPY,L_INSERT,L_COPY,L_CLONE,L_COPY_CLONE',b'\x00\x00\x01\x6F\x00\x00\x00\x16$4\x00L_USE_WIDTHS,L_USE_WINDOWS',b'\x00\x00\x01\x70\x00\x00\x00\x16$5\x00L_BF_UNKNOWN,L_BF_ANY,L_BF_CODE128,L_BF_EAN8,L_BF_EAN13,L_BF_CODE2OF5,L_BF_CODEI2OF5,L_BF_CODE39,L_BF_CODE93,L_BF_CODABAR,L_BF_UPCA',b'\x00\x00\x01\x71\x00\x00\x00\x16$6\x00L_SEVERITY_EXTERNAL,L_SEVERITY_ALL,L_SEVERITY_DEBUG,L_SEVERITY_INFO,L_SEVERITY_WARNING,L_SEVERITY_ERROR,L_SEVERITY_NONE',b'\x00\x00\x01\x72\x00\x00\x00\x16$7\x00SEL_DONT_CARE,SEL_HIT,SEL_MISS'),
+ _typenames = (b'\x00\x00\x01\x5CBOX',b'\x00\x00\x01\x5DBOXA',b'\x00\x00\x01\x5EFILE',b'\x00\x00\x01\x60L_COMP_DATA',b'\x00\x00\x01\x61PIX',b'\x00\x00\x01\x62PIXA',b'\x00\x00\x01\x63PIXAC',b'\x00\x00\x01\x64PIXCMAP',b'\x00\x00\x01\x66SARRAY',b'\x00\x00\x01\x67SEL',b'\x00\x00\x00\x3Al_float32',b'\x00\x00\x01\x6Bl_float64',b'\x00\x00\x01\x75l_int16',b'\x00\x00\x00\x05l_int32',b'\x00\x00\x01\x74l_int64',b'\x00\x00\x01\x76l_int8',b'\x00\x00\x00\x05l_ok',b'\x00\x00\x01\x80l_uint16',b'\x00\x00\x01\x7El_uint32',b'\x00\x00\x01\x7Fl_uint64',b'\x00\x00\x01\x7Cl_uint8'),
+)
diff --git a/src/ocrmypdf/lib/compile_leptonica.py b/src/ocrmypdf/lib/compile_leptonica.py
new file mode 100644
index 00000000..0e3afb56
--- /dev/null
+++ b/src/ocrmypdf/lib/compile_leptonica.py
@@ -0,0 +1,516 @@
+#!/usr/bin/env python3
+# © 2017 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from pathlib import Path
+
+from cffi import FFI
+
+ffibuilder = FFI()
+ffibuilder.cdef(
+ """
+typedef signed char l_int8;
+typedef unsigned char l_uint8;
+typedef short l_int16;
+typedef unsigned short l_uint16;
+typedef int l_int32;
+typedef unsigned int l_uint32;
+typedef float l_float32;
+typedef double l_float64;
+typedef long long l_int64;
+typedef unsigned long long l_uint64;
+
+typedef int l_ok; /*!< return type 0 if OK, 1 on error */
+
+struct Pix
+{
+ l_uint32 w; /* width in pixels */
+ l_uint32 h; /* height in pixels */
+ l_uint32 d; /* depth in bits (bpp) */
+ l_uint32 spp; /* number of samples per pixel */
+ l_uint32 wpl; /* 32-bit words/line */
+ l_uint32 refcount; /* reference count (1 if no clones) */
+ l_int32 xres; /* image res (ppi) in x direction */
+ /* (use 0 if unknown) */
+ l_int32 yres; /* image res (ppi) in y direction */
+ /* (use 0 if unknown) */
+ l_int32 informat; /* input file format, IFF_* */
+ l_int32 special; /* special instructions for I/O, etc */
+ char *text; /* text string associated with pix */
+ struct PixColormap *colormap; /* colormap (may be null) */
+ l_uint32 *data; /* the image data */
+};
+typedef struct Pix PIX;
+
+struct PixColormap
+{
+ void *array; /* colormap table (array of RGBA_QUAD) */
+ l_int32 depth; /* of pix (1, 2, 4 or 8 bpp) */
+ l_int32 nalloc; /* number of color entries allocated */
+ l_int32 n; /* number of color entries used */
+};
+typedef struct PixColormap PIXCMAP;
+
+/*! Array of pix */
+struct Pixa
+{
+ l_int32 n; /*!< number of Pix in ptr array */
+ l_int32 nalloc; /*!< number of Pix ptrs allocated */
+ l_uint32 refcount; /*!< reference count (1 if no clones) */
+ struct Pix **pix; /*!< the array of ptrs to pix */
+ struct Boxa *boxa; /*!< array of boxes */
+};
+typedef struct Pixa PIXA;
+
+/*! Array of compressed pix */
+struct PixaComp
+{
+ l_int32 n; /*!< number of PixComp in ptr array */
+ l_int32 nalloc; /*!< number of PixComp ptrs allocated */
+ l_int32 offset; /*!< indexing offset into ptr array */
+ struct PixComp **pixc; /*!< the array of ptrs to PixComp */
+ struct Boxa *boxa; /*!< array of boxes */
+};
+typedef struct PixaComp PIXAC;
+
+struct Box
+{
+ l_int32 x;
+ l_int32 y;
+ l_int32 w;
+ l_int32 h;
+ l_uint32 refcount; /* reference count (1 if no clones) */
+
+};
+typedef struct Box BOX;
+
+/*! Array of Box */
+struct Boxa
+{
+ l_int32 n; /*!< number of box in ptr array */
+ l_int32 nalloc; /*!< number of box ptrs allocated */
+ l_uint32 refcount; /*!< reference count (1 if no clones) */
+ struct Box **box; /*!< box ptr array */
+};
+typedef struct Boxa BOXA;
+
+/*! String array: an array of C strings */
+struct Sarray
+{
+ l_int32 nalloc; /*!< size of allocated ptr array */
+ l_int32 n; /*!< number of strings allocated */
+ l_int32 refcount; /*!< reference count (1 if no clones) */
+ char **array; /*!< string array */
+};
+typedef struct Sarray SARRAY;
+
+/*! Pdf formatted encoding types */
+enum {
+ L_DEFAULT_ENCODE = 0, /*!< use default encoding based on image */
+ L_JPEG_ENCODE = 1, /*!< use dct encoding: 8 and 32 bpp, no cmap */
+ L_G4_ENCODE = 2, /*!< use ccitt g4 fax encoding: 1 bpp */
+ L_FLATE_ENCODE = 3, /*!< use flate encoding: any depth, cmap ok */
+ L_JP2K_ENCODE = 4 /*!< use jp2k encoding: 8 and 32 bpp, no cmap */
+};
+
+/*! Compressed image data */
+struct L_Compressed_Data
+{
+ l_int32 type; /*!< encoding type: L_JPEG_ENCODE, etc */
+ l_uint8 *datacomp; /*!< gzipped raster data */
+ size_t nbytescomp; /*!< number of compressed bytes */
+ char *data85; /*!< ascii85-encoded gzipped raster data */
+ size_t nbytes85; /*!< number of ascii85 encoded bytes */
+ char *cmapdata85; /*!< ascii85-encoded uncompressed cmap */
+ char *cmapdatahex; /*!< hex pdf array for the cmap */
+ l_int32 ncolors; /*!< number of colors in cmap */
+ l_int32 w; /*!< image width */
+ l_int32 h; /*!< image height */
+ l_int32 bps; /*!< bits/sample; typ. 1, 2, 4 or 8 */
+ l_int32 spp; /*!< samples/pixel; typ. 1 or 3 */
+ l_int32 minisblack; /*!< tiff g4 photometry */
+ l_int32 predictor; /*!< flate data has PNG predictors */
+ size_t nbytes; /*!< number of uncompressed raster bytes */
+ l_int32 res; /*!< resolution (ppi) */
+};
+typedef struct L_Compressed_Data L_COMP_DATA;
+
+/*! Selection */
+struct Sel
+{
+ l_int32 sy; /*!< sel height */
+ l_int32 sx; /*!< sel width */
+ l_int32 cy; /*!< y location of sel origin */
+ l_int32 cx; /*!< x location of sel origin */
+ l_int32 **data; /*!< {0,1,2}; data[i][j] in [row][col] order */
+ char *name; /*!< used to find sel by name */
+};
+typedef struct Sel SEL;
+
+enum {
+ REMOVE_CMAP_TO_BINARY = 0, /*!< remove colormap for conv to 1 bpp */
+ REMOVE_CMAP_TO_GRAYSCALE = 1, /*!< remove colormap for conv to 8 bpp */
+ REMOVE_CMAP_TO_FULL_COLOR = 2, /*!< remove colormap for conv to 32 bpp */
+ REMOVE_CMAP_WITH_ALPHA = 3, /*!< remove colormap and alpha */
+ REMOVE_CMAP_BASED_ON_SRC = 4 /*!< remove depending on src format */
+};
+
+/*! Access and storage flags */
+enum {
+ L_NOCOPY = 0, /*!< do not copy the object; do not delete the ptr */
+ L_INSERT = L_NOCOPY, /*!< stuff it in; do not copy or clone */
+ L_COPY = 1, /*!< make/use a copy of the object */
+ L_CLONE = 2, /*!< make/use clone (ref count) of the object */
+ L_COPY_CLONE = 3 /*!< make a new array object (e.g., pixa) and fill */
+ /*!< the array with clones (e.g., pix) */
+};
+
+/*! Flags for method of extracting barcode widths */
+enum {
+ L_USE_WIDTHS = 1, /*!< use histogram of barcode widths */
+ L_USE_WINDOWS = 2 /*!< find best window for decoding transitions */
+};
+
+/*! Flags for barcode formats */
+enum {
+ L_BF_UNKNOWN = 0, /*!< unknown format */
+ L_BF_ANY = 1, /*!< try decoding with all known formats */
+ L_BF_CODE128 = 2, /*!< decode with Code128 format */
+ L_BF_EAN8 = 3, /*!< decode with EAN8 format */
+ L_BF_EAN13 = 4, /*!< decode with EAN13 format */
+ L_BF_CODE2OF5 = 5, /*!< decode with Code 2 of 5 format */
+ L_BF_CODEI2OF5 = 6, /*!< decode with Interleaved 2 of 5 format */
+ L_BF_CODE39 = 7, /*!< decode with Code39 format */
+ L_BF_CODE93 = 8, /*!< decode with Code93 format */
+ L_BF_CODABAR = 9, /*!< decode with Code93 format */
+ L_BF_UPCA = 10 /*!< decode with UPC A format */
+};
+
+enum {
+ L_SEVERITY_EXTERNAL = 0, /* Get the severity from the environment */
+ L_SEVERITY_ALL = 1, /* Lowest severity: print all messages */
+ L_SEVERITY_DEBUG = 2, /* Print debugging and higher messages */
+ L_SEVERITY_INFO = 3, /* Print informational and higher messages */
+ L_SEVERITY_WARNING = 4, /* Print warning and higher messages */
+ L_SEVERITY_ERROR = 5, /* Print error and higher messages */
+ L_SEVERITY_NONE = 6 /* Highest severity: print no messages */
+};
+
+enum {
+ SEL_DONT_CARE = 0,
+ SEL_HIT = 1,
+ SEL_MISS = 2
+};
+
+"""
+)
+
+ffibuilder.cdef(
+ """
+PIX * pixRead ( const char *filename );
+PIX * pixReadMem ( const l_uint8 *data, size_t size );
+PIX * pixReadStream ( FILE *fp, l_int32 hint );
+PIX * pixScale ( PIX *pixs, l_float32 scalex, l_float32 scaley );
+l_int32 pixFindSkew ( PIX *pixs, l_float32 *pangle, l_float32 *pconf );
+l_int32 pixWriteImpliedFormat ( const char *filename, PIX *pix, l_int32 quality, l_int32 progressive );
+l_int32 getImpliedFileFormat ( const char *filename );
+l_ok pixWriteStream ( FILE *fp, PIX *pix, l_int32 format );
+l_ok pixWriteStreamJpeg ( FILE *fp, PIX *pixs, l_int32 quality, l_int32 progressive );
+l_ok pixWriteMem ( l_uint8 **pdata, size_t *psize, PIX *pix, l_int32 format );
+l_ok pixWriteMemJpeg ( l_uint8 **pdata, size_t *psize, PIX *pix, l_int32 quality, l_int32 progressive );
+l_int32
+pixWriteMemPng(l_uint8 **pdata,
+ size_t *psize,
+ PIX *pix,
+ l_float32 gamma);
+
+void pixDestroy ( PIX **ppix );
+
+l_ok
+pixEqual(PIX *pix1,
+ PIX *pix2,
+ l_int32 *psame);
+
+PIX *
+pixEndianByteSwapNew(PIX *pixs);
+
+PIX * pixDeskew ( PIX *pixs, l_int32 redsearch );
+char * getLeptonicaVersion ( );
+l_int32 pixCorrelationBinary(PIX *pix1, PIX *pix2, l_float32 *pval);
+PIX *pixRotate180(PIX *pixd, PIX *pixs);
+PIX *
+pixRotateOrth(PIX *pixs,
+ l_int32 quads);
+
+l_int32 pixCountPixels ( PIX *pix, l_int32 *pcount, l_int32 *tab8 );
+PIX * pixAnd ( PIX *pixd, PIX *pixs1, PIX *pixs2 );
+l_int32 * makePixelSumTab8 ( void );
+
+PIX * pixDeserializeFromMemory ( const l_uint32 *data, size_t nbytes );
+l_int32 pixSerializeToMemory ( PIX *pixs, l_uint32 **pdata, size_t *pnbytes );
+
+PIX * pixConvertRGBToLuminance(PIX *pixs);
+
+PIX * pixConvertTo8(PIX *pixs, l_int32 cmapflag);
+
+PIX * pixRemoveColormap(PIX *pixs, l_int32 type);
+
+l_int32
+pixOtsuAdaptiveThreshold(PIX *pixs,
+ l_int32 sx,
+ l_int32 sy,
+ l_int32 smoothx,
+ l_int32 smoothy,
+ l_float32 scorefract,
+ PIX **ppixth,
+ PIX **ppixd);
+
+PIX *
+pixOtsuThreshOnBackgroundNorm(PIX *pixs,
+ PIX *pixim,
+ l_int32 sx,
+ l_int32 sy,
+ l_int32 thresh,
+ l_int32 mincount,
+ l_int32 bgval,
+ l_int32 smoothx,
+ l_int32 smoothy,
+ l_float32 scorefract,
+ l_int32 *pthresh);
+
+PIX *
+pixMaskedThreshOnBackgroundNorm(PIX *pixs,
+ PIX *pixim,
+ l_int32 sx,
+ l_int32 sy,
+ l_int32 thresh,
+ l_int32 mincount,
+ l_int32 smoothx,
+ l_int32 smoothy,
+ l_float32 scorefract,
+ l_int32 *pthresh);
+
+PIX *
+pixCleanBackgroundToWhite(PIX *pixs,
+ PIX *pixim,
+ PIX *pixg,
+ l_float32 gamma,
+ l_int32 blackval,
+ l_int32 whiteval);
+
+BOX *
+pixFindPageForeground ( PIX *pixs,
+ l_int32 threshold,
+ l_int32 mindist,
+ l_int32 erasedist,
+ l_int32 showmorph,
+ PIXAC *pixac );
+
+PIX *
+pixClipRectangle(PIX *pixs,
+ BOX *box,
+ BOX **pboxc);
+
+PIX *
+pixBackgroundNorm(PIX *pixs,
+ PIX *pixim,
+ PIX *pixg,
+ l_int32 sx,
+ l_int32 sy,
+ l_int32 thresh,
+ l_int32 mincount,
+ l_int32 bgval,
+ l_int32 smoothx,
+ l_int32 smoothy);
+
+PIX *
+pixGammaTRC(PIX *pixd,
+ PIX *pixs,
+ l_float32 gamma,
+ l_int32 minval,
+ l_int32 maxval);
+
+
+l_int32
+pixNumSignificantGrayColors(PIX *pixs,
+ l_int32 darkthresh,
+ l_int32 lightthresh,
+ l_float32 minfract,
+ l_int32 factor,
+ l_int32 *pncolors);
+
+l_int32
+pixColorFraction(PIX *pixs,
+ l_int32 darkthresh,
+ l_int32 lightthresh,
+ l_int32 diffthresh,
+ l_int32 factor,
+ l_float32 *ppixfract,
+ l_float32 *pcolorfract);
+
+PIX *
+pixColorMagnitude(PIX *pixs,
+ l_int32 rwhite,
+ l_int32 gwhite,
+ l_int32 bwhite,
+ l_int32 type);
+
+PIX *
+pixMaskOverColorPixels(PIX *pixs,
+ l_int32 threshdiff,
+ l_int32 mindist);
+
+l_int32
+pixGetAverageMaskedRGB(PIX *pixs,
+ PIX *pixm,
+ l_int32 x,
+ l_int32 y,
+ l_int32 factor,
+ l_int32 type,
+ l_float32 *prval,
+ l_float32 *pgval,
+ l_float32 *pbval);
+
+PIX *
+pixGlobalNormRGB(PIX * pixd,
+ PIX * pixs,
+ l_int32 rval,
+ l_int32 gval,
+ l_int32 bval,
+ l_int32 mapval);
+
+PIX *
+pixInvert(PIX * pixd,
+ PIX * pixs);
+
+PIX *
+pixRemoveColormapGeneral(PIX *pixs,
+ l_int32 type,
+ l_int32 ifnocmap);
+
+l_int32
+pixGenerateCIData(PIX *pixs,
+ l_int32 type,
+ l_int32 quality,
+ l_int32 ascii85,
+ L_COMP_DATA **pcid);
+
+SARRAY *
+pixProcessBarcodes(PIX *pixs,
+ l_int32 format,
+ l_int32 method,
+ SARRAY **psaw,
+ l_int32 debugflag);
+
+PIX *
+pixaGetPix(PIXA *pixa,
+ l_int32 index,
+ l_int32 accesstype);
+
+BOX*
+pixaGetBox (PIXA * pixa,
+ l_int32 index,
+ l_int32 accesstype );
+
+PIXA *
+pixExtractBarcodes(PIX *pixs,
+ l_int32 debugflag);
+
+BOXA *
+pixLocateBarcodes ( PIX *pixs,
+ l_int32 thresh,
+ PIX **ppixb,
+ PIX **ppixm );
+
+SARRAY *
+pixReadBarcodes(PIXA *pixa,
+ l_int32 format,
+ l_int32 method,
+ SARRAY **psaw,
+ l_int32 debugflag);
+
+PIX *
+pixGenHalftoneMask(PIX *pixs,
+ PIX **ppixtext,
+ l_int32 *phtfound,
+ PIXA *pixadb);
+
+l_int32
+l_generateCIDataForPdf(const char *fname,
+ PIX *pix,
+ l_int32 quality,
+ L_COMP_DATA **pcid);
+
+
+BOX *
+boxClone ( BOX *box );
+
+BOX *
+boxaGetBox ( BOXA *boxa, l_int32 index, l_int32 accessflag );
+
+SEL *
+selCreateFromString ( const char *text, l_int32 h, l_int32 w, const char *name );
+
+SEL *
+selCreateBrick ( l_int32 h, l_int32 w, l_int32 cy, l_int32 cx, l_int32 type );
+
+char *
+selPrintToString(SEL *sel);
+
+PIX *
+pixDilate ( PIX *pixd, PIX *pixs, SEL *sel );
+
+PIX *
+pixErode ( PIX *pixd, PIX *pixs, SEL *sel );
+
+PIX *
+pixHMT ( PIX *pixd, PIX *pixs, SEL *sel );
+
+PIX *
+pixSubtract ( PIX *pixd, PIX *pixs1, PIX *pixs2 );
+
+void
+boxDestroy(BOX **pbox);
+
+void
+boxaDestroy ( BOXA **pboxa );
+
+void
+pixaDestroy(PIXA **ppixa);
+
+l_ok
+pixRenderBoxa ( PIX *pix, BOXA *boxa, l_int32 width, l_int32 op );
+
+void
+l_CIDataDestroy(L_COMP_DATA **pcid);
+
+void
+sarrayDestroy(SARRAY **psa);
+
+void
+lept_free(void *ptr);
+
+void selDestroy ( SEL **psel );
+
+l_int32
+setMsgSeverity(l_int32 newsev);
+
+void
+leptSetStderrHandler(void (*handler)(const char *));
+"""
+)
+
+
+ffibuilder.set_source("ocrmypdf.lib._leptonica", None)
+
+if __name__ == '__main__':
+ ffibuilder.compile(verbose=True)
+ if Path('ocrmypdf/lib/_leptonica.py').exists() and Path('src/ocrmypdf').exists():
+ output = Path('ocrmypdf/lib/_leptonica.py')
+ output.rename('src/ocrmypdf/lib/_leptonica.py')
+ Path('ocrmypdf/lib').rmdir()
+ Path('ocrmypdf').rmdir()
diff --git a/tests/test_image_input.py b/tests/test_image_input.py
index ef3702fe..15b00d7e 100644
--- a/tests/test_image_input.py
+++ b/tests/test_image_input.py
@@ -72,7 +72,7 @@ def test_img2pdf_fails(resources, no_outpdf):
mock.assert_called()
-@pytest.mark.xfail(reason="remove background disabled")
+# @pytest.mark.xfail(reason="remove background disabled")
def test_jpeg_in_jpeg_out(resources, outpdf):
check_ocrmypdf(
resources / 'congress.jpg',
diff --git a/tests/test_lept.py b/tests/test_lept.py
new file mode 100644
index 00000000..996aa0fe
--- /dev/null
+++ b/tests/test_lept.py
@@ -0,0 +1,96 @@
+# © 2018 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from os import fspath
+from pickle import dumps, loads
+
+import pytest
+from PIL import Image, ImageChops
+
+from ocrmypdf import leptonica as lp
+
+
+def test_colormap_backgroundnorm(resources):
+ # Issue #262 - unclear how to reproduce exactly, so just ensure leptonica
+ # can handle that case
+ pix = lp.Pix.open(resources / 'baiona_colormapped.png')
+ pix.background_norm()
+
+
+@pytest.fixture
+def crom_pix(resources):
+ pix = lp.Pix.open(resources / 'crom.png')
+ im = Image.open(resources / 'crom.png')
+ yield pix, im
+ im.close()
+
+
+def test_pix_basic(crom_pix):
+ pix, im = crom_pix
+
+ assert pix.width == im.width
+ assert pix.height == im.height
+ assert pix.mode == im.mode
+
+
+def test_pil_conversion(crom_pix):
+ pix, im = crom_pix
+
+ # Check for pixel perfect
+ assert ImageChops.difference(pix.topil(), im).getbbox() is None
+
+
+def test_pix_otsu(crom_pix):
+ pix, _ = crom_pix
+ im1bpp = pix.otsu_adaptive_threshold()
+ assert im1bpp.mode == '1'
+
+
+@pytest.mark.skipif(
+ lp.get_leptonica_version() < 'leptonica-1.76',
+ reason="needs new leptonica for API change",
+)
+def test_crop(resources):
+ pix = lp.Pix.open(resources / 'linn.png')
+ foreground = pix.crop_to_foreground()
+ assert foreground.width < pix.width
+
+
+def test_clean_bg(resources):
+ pix = lp.Pix.open(resources / 'congress.jpg')
+ imbg = pix.clean_background_to_white()
+
+
+def test_pickle(crom_pix):
+ pix, _ = crom_pix
+ pickled = dumps(pix)
+ pix2 = loads(pickled)
+ assert pix.mode == pix2.mode
+
+
+def test_leptonica_compile(tmp_path):
+ from ocrmypdf.lib.compile_leptonica import ffibuilder
+
+ # Compile the library but build it somewhere that won't interfere with
+ # existing compiled library. Also compile in API mode so that we test
+ # the interfaces, even though we use it ABI mode.
+ ffibuilder.compile(tmpdir=fspath(tmp_path), target=fspath(tmp_path / 'lepttest.*'))
+
+
+def test_file_not_found():
+ with pytest.raises(FileNotFoundError):
+ lp.Pix.open("does_not_exist1")
+
+
+@pytest.mark.skipif(
+ lp.get_leptonica_version() < 'leptonica-1.79.0',
+ reason="test not reliable on all platforms for old leptonica",
+)
+def test_error_trap():
+ with pytest.raises(lp.LeptonicaError, match=r"Error in pixReadMem"):
+ with lp._LeptonicaErrorTrap():
+ lp.Pix(lp.lept.pixReadMem(lp.ffi.NULL, 0))
diff --git a/tests/test_main.py b/tests/test_main.py
index 682f1a63..f772cb04 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -589,7 +589,7 @@ def test_pagesize_consistency(renderer, resources, outpdf):
renderer,
'--clean' if have_unpaper() else None,
'--deskew',
- # '--remove-background',
+ '--remove-background',
'--clean-final' if have_unpaper() else None,
'-k',
'--pages',
diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py
index b9975f41..090ea994 100644
--- a/tests/test_preprocessing.py
+++ b/tests/test_preprocessing.py
@@ -43,7 +43,7 @@ def test_deskew_blank_page(resources, outpdf):
check_ocrmypdf(resources / 'blank.pdf', outpdf, '--deskew')
-@pytest.mark.xfail(reason="remove background disabled")
+# @pytest.mark.xfail(reason="remove background disabled")
def test_remove_background(resources, outdir):
# Ensure the input image does not contain pure white/black
with Image.open(resources / 'congress.jpg') as im:
--
2.25.1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment