Merge branch 'improve_matching' into 'master'

Improve matching

See merge request julia.garriga/darfix!45

.gitlab-ci.yml

 py_testproject:
   type: test
   image: docker-registry.esrf.fr/dau/x11:python3.7_buster
+  before_script:
+    - apt-get update -qq && apt-get install -y -qq ocl-icd-opencl-dev opencl-c-headers opencl-clhpp-headers opencl-headers
+    # - apt-get update -qq && apt install -qq -y ocl-icd-libopencl1
+    - export SILX_OPENCL=1
   script:                   # Run the tests from the installed module
     - python -m pip install --upgrade pip
     - pip install --upgrade setuptools wheel

darfix/core/componentsMatching.py

@@ -26,8 +26,7 @@
 
 __authors__ = ["J. Garriga"]
 __license__ = "MIT"
-__date__ = "21/01/2020"
-
+__date__ = "28/02/2020"
 
 import cv2
 import numpy
@@ -101,7 +100,8 @@ class ComponentsMatching():
         dst = numpy.linalg.norm(X - Y)  # their euclidean distances
         return dst
 
-    def match_components(self, id1=None, id2=None, method=Method.orb_feature_matching):
+    def match_components(self, id1=None, id2=None, method=Method.orb_feature_matching,
+                         tol=8):
         """
         Match components. Given the components x1,...,xn of dataset 1 and the
         components y1,...,ym of dataset 2, this function computes the pairs
@@ -122,7 +122,7 @@ class ComponentsMatching():
             id1 = 0
             id2 = 1
 
-        matches = {}
+        good = {}
         final_matches = {}
         if method == Method.orb_feature_matching:
             self.descriptors = self._create_descriptors()
@@ -133,42 +133,65 @@ class ComponentsMatching():
                 if component1.descriptor is not None:
                     for j, component2 in enumerate(self.descriptors[id2]):
                         if component2.descriptor is not None:
+
                             # Match descriptors
-                            matches[(i, j)] = bf.match(component1.descriptor,
-                                                       component2.descriptor)
+                            good[(i, j)] = numpy.array(bf.match(component1.descriptor,
+                                                       component2.descriptor))
+
+            best_v = []
             # Add matches sorted by number of matches found.
-            for x, y in sorted(matches, key=lambda match: len(matches[match]),
+            for x, y in sorted(good, key=lambda match: len(good[match]),
                                reverse=True):
                 # Only add match if neither x nor y are already in the list.
                 if x not in final_matches.keys() and y not in final_matches.values():
-                    final_matches[x] = y
+
+                    kp1 = []
+                    kp2 = []
+                    for match in good[(x, y)]:
+                        kp1 += [self.descriptors[id1][x].keypoints[match.queryIdx].pt]
+                        kp2 += [self.descriptors[id2][y].keypoints[match.trainIdx].pt]
+                    if len(kp1) > 1 and len(kp2) > 1:
+                        v = numpy.mean(numpy.array(kp2) - numpy.array(kp1), axis=0)
+                    else:
+                        v = numpy.array(kp2) - numpy.array(kp1)
+                    if not numpy.any(best_v):
+                        best_v = v
+                        final_matches[x] = y
+                    elif numpy.linalg.norm(best_v - v) < tol:
+                        final_matches[x] = y
 
         elif method == Method.sift_feature_matching:
             keypoints = self._create_sift_keypoints()
-
+            best_v = []
             mp = sift.MatchPlan()
             # Match components with id1 and id2
             for i, kp1 in enumerate(keypoints[id1]):
                 for j, kp2 in enumerate(keypoints[id2]):
                     # Match descriptors
-                    matches[(i, j)] = mp.match(kp1, kp2)
+                    good[(i, j)] = mp.match(kp1, kp2)
             # Add matches sorted by number of matches found.
-            for x, y in sorted(matches, key=lambda match: matches[match].shape[0],
+            for x, y in sorted(good, key=lambda match: good[match].shape[0],
                                reverse=True):
                 # Only add match if neither x nor y are already in the list.
                 if x not in final_matches.keys() and y not in final_matches.values():
-                    final_matches[x] = y
+                    v = numpy.array([numpy.median(good[(x, y)][:, 1].x - good[(x, y)][:, 0].x),
+                                     numpy.median(good[(x, y)][:, 1].y - good[(x, y)][:, 0].y)])
+                    if not numpy.any(best_v):
+                        best_v = v
+                        final_matches[x] = y
+                    elif numpy.linalg.norm(best_v - v) < tol:
+                        final_matches[x] = y
 
         elif method == Method.euclidean_distance:
             for i, X in enumerate(self.components[id1]):
                 for j, Y in enumerate(self.components[id2]):
-                    matches[(i, j)] = self.euclidean_distance(X, Y)
+                    good[(i, j)] = self.euclidean_distance(X, Y)
             # Add matches sorted by distance.
-            for x, y in sorted(matches, key=lambda match: matches[match]):
+            for x, y in sorted(good, key=lambda match: good[match]):
                 # Only add match if neither x nor y are already in the list.
                 if x not in final_matches.keys() and y not in final_matches.values():
                     final_matches[x] = y
-        return final_matches, matches
+        return final_matches, good
 
     def draw_matches(self, final_matches, matches, id1=None, id2=None,
                      displayMatches=False):
@@ -201,6 +224,7 @@ class ComponentsMatching():
                     assert all((isinstance(match, cv2.DMatch) for match in values)
                                for values in matches.values()), \
                         "Dictionary `matches` has to contain values of type `cv2.DMatch`"
+
                     img = cv2.drawMatches(self.descriptors[id1][i].image,
                                           self.descriptors[id1][i].keypoints,
                                           self.descriptors[id2][j].image,

darfix/core/test/test_components_matching.py

@@ -33,6 +33,8 @@ import unittest
 
 import numpy
 
+from silx.opencl.common import ocl
+
 from darfix.core.componentsMatching import ComponentsMatching, Method
 from skimage import data
 
@@ -63,8 +65,12 @@ class TestComponentsMatching(unittest.TestCase):
 
         self.assertNotEqual(ed, 0)
 
-    def test_match_components(self):
+    @unittest.skipUnless(ocl, "PyOpenCl is missing")
+    def test_sift_match(self):
+        final_matches, matches = self.componentsMatching.match_components(method=Method.sift_feature_matching)
+        self.assertEqual(final_matches[0], 1)
 
+    def test_orb_match(self):
         final_matches, matches = self.componentsMatching.match_components(method=Method.orb_feature_matching)
         self.assertEqual(final_matches[0], 1)
 
@@ -79,3 +85,9 @@ class TestComponentsMatching(unittest.TestCase):
         final_matches, matches = self.componentsMatching.match_components(method=Method.euclidean_distance)
         stack = self.componentsMatching.draw_matches(final_matches, matches)
         self.assertEqual(stack[2].shape, (512, 1024))
+
+    @unittest.skipUnless(ocl, "PyOpenCl is missing")
+    def test_draw_matches2(self):
+        final_matches, matches = self.componentsMatching.match_components(method=Method.sift_feature_matching)
+        stack = self.componentsMatching.draw_matches(final_matches, matches)
+        self.assertEqual(stack[2].shape, (512, 1024))

requirements.txt

@@ -4,3 +4,4 @@ PyQt5==5.13.2
 opencv-python
 scikit-image
 orange3==3.22.0
+pyopencl