Improve auto-crop to work with noisy images

My new scanner returns images with some background noise even in the
unoccupied areas of the scan. Use a different method for auto-crop
that allows specifying a threshold value rather than cutting off only
areas with maximum intensity.

For now, we add the auto-crop configuration as a toolbar inside the
scanner settings and pass through the values manually, without
persisting them in the data store.

2 files changed, 193 insertions, 12 deletions

diff --git a/postproc.py b/postproc.py
new file mode 100644
index 0000000..c6bdf27
--- /dev/null
+++ b/postproc.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+#
+# Author: Sascha Silbe <sascha-pgp@silbe.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3
+# as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""Image post-processing functions
+
+Various functions for post-processing a scanned image.
+"""
+import logging
+import subprocess
+import tempfile
+
+import cv2
+import numpy
+from PIL import Image, ImageChops, ImageColor, ImageOps
+from scipy.ndimage import morphology, label
+
+
+def autocrop(image, threshold):
+    input_tmp = tempfile.NamedTemporaryFile(suffix='.png')
+    output_tmp = tempfile.NamedTemporaryFile(suffix='.png')
+    image.save(input_tmp.name)
+    bbox_cmd = ['convert', input_tmp.name, '-morphology', 'Smooth', 'Octagon',
+                '-fuzz', '%d%%' % (threshold, ), '-trim', '-format',
+                '%[fx:w+20]x%[fx:h+20]+%[fx:page.x-10]+%[fx:page.y-10]',
+                'info:']
+    bbox_pipe = subprocess.Popen(bbox_cmd, stdout=subprocess.PIPE)
+    extents = bbox_pipe.communicate()[0].strip()
+    if bbox_pipe.returncode:
+        raise subprocess.CalledProcessError(bbox_pipe.returncode, bbox_cmd)
+
+    crop_cmd = ['convert', input_tmp.name, '-crop', extents, output_tmp.name]
+    subprocess.check_call(crop_cmd)
+    cropped_image = Image.open(output_tmp.name)
+    cropped_image.load()
+    return cropped_image
+
+
+def autocrop_threshold(image, threshold):
+    last_size = None
+    while image.size != last_size:
+        last_size = image.size
+        for colour in [0, 255]:
+            grey_image = ImageOps.grayscale(image)
+            grey_image.save('grey_image_%s.png' % (colour, ))
+            if colour:
+                bw_image = grey_image.point(lambda pixel: (pixel > threshold) and 255)
+            else:
+                bw_image = grey_image.point(lambda pixel: (pixel >= (255 - threshold)) and 255)
+            bw_image = bw_image.convert('L')
+            bw_image.save('bw_image_%s.png' % (colour, ))
+            mono_image = Image.new('1', image.size, colour)
+            mono_image.save('mono_image_%s.png' % (colour, ))
+            bbox = ImageChops.difference(bw_image, mono_image).getbbox()
+            logging.debug('colour=%r, size=%r, bbox=%r', colour, image.size, bbox)
+            image = image.crop(bbox)
+    return image
+
+
+def autocrop_morphological(image, open_threshold=None, close_threshold=None,
+                           nth_largest=2):
+    open_threshold = open_threshold or max(min(*image.size) // 100, 1)
+    close_threshold = close_threshold or open_threshold
+    logging.debug('open_threshold=%r, close_threshold=%r', open_threshold, close_threshold)
+
+    grey_image = ImageOps.grayscale(image)
+    grey_image = ImageOps.autocontrast(grey_image)
+
+    grey_array = numpy.array(grey_image, dtype=numpy.uint8)
+    grey_array = morphology.grey_closing(grey_array, (1, close_threshold))
+    threshold_, grey_array = cv2.threshold(grey_array, 0, 1, cv2.THRESH_OTSU)
+
+    grey_array = morphology.grey_opening(grey_array, (open_threshold, open_threshold))
+
+    # determine n-th largest component
+    labeled_array, num_features = label(grey_array)
+    features_by_size = [(len(numpy.where(labeled_array == i)[0]), i)
+                        for i in range(1, num_features + 1)]
+    features_by_size.sort(reverse=True)
+    logging.debug('features_by_size=%r', features_by_size)
+    nth_clipped = min(nth_largest - 1, len(features_by_size))
+    chosen_index = features_by_size[nth_clipped][1]
+    logging.debug('nth_largest=%r, chosen_index=%r',
+                  nth_largest, chosen_index)
+
+    # clear out all other components
+    for i in range(1, num_features + 1):
+        if i == chosen_index:
+            continue
+        grey_array[labeled_array == i] = 0
+
+    col_sum = numpy.sum(grey_array, axis=0)
+    row_sum = numpy.sum(grey_array, axis=1)
+    col_mean, col_std = col_sum.mean(), col_sum.std()
+    row_mean, row_std = row_sum.mean(), row_sum.std()
+
+    row_standard = (row_sum - row_mean) / row_std
+    col_standard = (col_sum - col_mean) / col_std
+
+    def end_points(s, std_below_mean=-1.5):
+        i, j = 0, len(s) - 1
+        for i, rs in enumerate(s):
+            if rs > std_below_mean:
+                break
+        for j in xrange(len(s) - 1, i, -1):
+            if s[j] > std_below_mean:
+                break
+        return (i, j)
+
+    # determine bounding box
+    x1, x2 = end_points(col_standard)
+    y1, y2 = end_points(row_standard)
+
+    return image.crop((x1, y1, x2, y2))
diff --git a/scan.py b/scan.py
index a887f58..b8a5b98 100644
--- a/scan.py
+++ b/scan.py
@@ -33,9 +33,7 @@ import time
 import gobject
 import gtk
 
-import Image
-import ImageChops
-import ImageColor
+from PIL import ImageOps
 import sane
 
 from sugar.activity.widgets import ActivityToolbarButton, StopButton
@@ -48,16 +46,23 @@ from sugar.graphics.toolbarbox import ToolbarButton, ToolbarBox
 from sugar.graphics import style
 from sugar.logger import trace
 
+import postproc
+
 
 class SettingsToolbar(gtk.ToolPalette):
 
     @trace()
     def __init__(self):
         gtk.ToolPalette.__init__(self)
+        self.autocontrast = False
+        self.autocontrast_cutoff = 0
+        self.autocrop_threshold = True
         self._scanner = None
         self._widget_by_option = {}
         self._toolitem_by_option = {}
         self._updating = False
+        self._static_tool_group = None
+        self._add_static_widgets()
 
     def modify_bg(self, state, color):
         gtk.ToolPalette.modify_bg(self, state, color)
@@ -70,10 +75,57 @@ class SettingsToolbar(gtk.ToolPalette):
         if self._scanner is not None:
             self._add_options()
 
+    def _add_static_widgets(self):
+        self._static_tool_group = gtk.ToolItemGroup(_('Post-processing'))
+        self._static_tool_group.show()
+        self.add(self._static_tool_group)
+        self.set_expand(self._static_tool_group, False)
+        tool_item = self._create_spin_toolitem('autocontrast_cutoff',
+                                               _('Automatic contrast cut-off threshold'),
+                                               0, 100,
+                                               _('Percentage of upper and lower'
+                                                 ' pixel values to eliminate'
+                                                 ' (clamp to lowest resp.'
+                                                 ' highest value).'))
+        tool_item.show()
+        self._static_tool_group.insert(tool_item, -1)
+        tool_item = self._create_spin_toolitem('autocrop_threshold',
+                                               _('Auto-crop threshold'),
+                                               0, 50,
+                                               _('Ignore the upper / lower pixel values for auto-cropping'))
+        tool_item.show()
+        self._static_tool_group.insert(tool_item, -1)
+
+    def _create_spin_toolitem(self, name, label, min, max, tooltip):
+        label = gtk.Label(label)
+        label.modify_fg(gtk.STATE_NORMAL,
+                        style.COLOR_WHITE.get_gdk_color())
+        value = getattr(self, name)
+        spin_adj = gtk.Adjustment(value, min, max, 1, max//10, 0)
+        spin = gtk.SpinButton(spin_adj, 0, 0)
+        spin.props.snap_to_ticks = True
+        spin.set_numeric(True)
+        spin.connect('value-changed', lambda widget:
+            setattr(self, name, int(widget.get_value())))
+        spin.show()
+
+        hbox = gtk.HBox(False, style.DEFAULT_SPACING)
+        hbox.pack_start(label, False)
+        hbox.pack_start(spin)
+        label.show()
+
+        tool_item = gtk.ToolItem()
+        tool_item.add(hbox)
+        hbox.show()
+        tool_item.set_tooltip_text(tooltip)
+        return tool_item
+
     def _clear(self):
         self._widget_by_option = {}
         self._toolitem_by_option = {}
         for widget in list(self):
+            if widget == self._static_tool_group:
+                continue
             self.remove(widget)
 
     @trace()
@@ -353,6 +405,9 @@ class ScanThread(threading.Thread):
         self._error = None
         self._ready = False
         self._dpi = None
+        self.autocontrast = True
+        self.autocontrast_cutoff = 0
+        self.autocrop_threshold = 0
 
     def set_device(self, device):
         with self._cond:
@@ -491,17 +546,15 @@ class ScanThread(threading.Thread):
         loader.close()
         return pixbuf
 
-    def autocrop(self, image):
-        if image.mode != 'RGB':
-            image = image.convert('RGB')
-
-        white_image = Image.new('RGB', image.size, ImageColor.getrgb('white'))
-        diff = ImageChops.difference(image, white_image)
-        return image.crop(diff.getbbox())
-
     def _process_image(self, image):
         logging.debug('pre-crop size: %r', image.size)
-        image = self.autocrop(image)
+        if self.autocontrast:
+            image = ImageOps.autocontrast(image, self.autocontrast_cutoff)
+
+        image = postproc.autocrop(image, self.autocrop_threshold)
+        # image = postproc.autocrop(image, self.autocrop_open_threshold,
+        #                           self.autocrop_close_threshold,
+        #                           self.autocrop_nth_largest)
         logging.debug('post-crop size: %r', image.size)
         image_file = tempfile.NamedTemporaryFile(dir=self._temp_dir,
             suffix='.png')
@@ -822,6 +875,9 @@ class ScanActivity(activity.Activity):
         if self._status == 'ready':
             self._add_msg('triggering scan')
             self._set_status('scanning')
+            self._scan_thread.autocontrast = self._settings_toolbar.autocontrast
+            self._scan_thread.autocontrast_cutoff = self._settings_toolbar.autocontrast_cutoff
+            self._scan_thread.autocrop_threshold = self._settings_toolbar.autocrop_threshold
             self._scan_thread.start_scan()
         elif self._status == 'scanning':
             self._add_msg('stopping scan')