diff --git a/wrapper_python/np_opencv_converter.h b/wrapper_python/np_opencv_converter.h
index 6d8a6aa2797226d7c24d2c43fa8a73e214524a4b..ada6020378f51c71f1c3505f9001f8dd9c6f6567 100644
--- a/wrapper_python/np_opencv_converter.h
+++ b/wrapper_python/np_opencv_converter.h
@@ -106,6 +106,10 @@ struct Mat_PyObject_converter
     using namespace boost::python;
     typedef converter::rvalue_from_python_storage< T > storage_t;
 
+    // Object is a borrowed reference, so create a handle indicating it is
+    // borrowed for proper reference counting.
+    boost::python::handle<> handle(boost::python::borrowed(obj_ptr));
+
     storage_t* the_storage = reinterpret_cast<storage_t*>( data );
     void* memory_chunk = the_storage->storage.bytes;
 
diff --git a/wrapper_python/utils/conversion.cpp b/wrapper_python/utils/conversion.cpp
index 00a8a1e570eed54645b7f973ab3c87afdeaa1e19..27b06a7dceac9866758e9928f95f1717c9d4292d 100644
--- a/wrapper_python/utils/conversion.cpp
+++ b/wrapper_python/utils/conversion.cpp
@@ -311,8 +311,11 @@ cv::Mat NDArrayConverter::toMat(const PyObject *o)
     {
 #if CV_MAJOR_VERSION == 3
       m.addref();
-      Py_INCREF(o);
+// Not needed anymore, the pointer is now managed by a borrowed handle, which
+// which will handle reference counting.
+//      Py_INCREF(o);
 #else
+// TODO: I assume this one is leaking too, but don't have CV v2 to test.
         m.refcount = refcountFromPyObject(o);
         m.addref(); // protect the original numpy array from deallocation
                     // (since Mat destructor will decrement the reference counter)