Pull request #14: CT-67 higher level 2d3dregistration demo

Merge in SUITE/public-demos from CT-67-higher-level-2d3dregistration-demo to master Squashed commit of the following: commit f2e3bf095fc164b2e2bd1fdb4c4871f4ae101a38 Author: Nathanael Schilling <[email protected]> Date: Thu Sep 30 14:21:27 2021 +0200 [CT-67] Minor change to ExamplePlugin commit 543be6544fb48af0976af3971f7c4dddc7c89707 Author: Nathanael Schilling <[email protected]> Date: Thu Sep 30 13:59:41 2021 +0200 [CT-67] Respond to reviewer comments for 2D/3D Registration demo. commit 1a9932a09d95db918f138f31a727a6d93e9181f7 Author: Nathanael Schilling <[email protected]> Date: Tue Sep 28 16:48:31 2021 +0200 [CT-67] Respond to reviewer comments commit 5e04fb4633dafbd4966e767a59b50fbbe5250f66 Author: Nathanael Schilling <[email protected]> Date: Fri Sep 10 13:32:43 2021 +0200 [CT-67] Higher-level 2D/3D registration demo
daudrain · Sep 30, 2021 · f9b7ab4 · f9b7ab4
1 parent 85ade0b
commit f9b7ab4
Show file tree

Hide file tree

Showing 15 changed files with 576 additions and 11 deletions.
diff --git a/Example2D3DRegistration/CMakeLists.txt b/Example2D3DRegistration/CMakeLists.txt
@@ -0,0 +1,48 @@
+# Define a new CMake project for the demo plugin
+project(Example2D3DRegistrationPlugin)
+cmake_minimum_required(VERSION 3.2.0)
+
+
+# Locate the ImFusion SDK. We require the CT component to
+# use the XRay2D3DRegistrationAlgorithm class.
+find_package(ImFusionLib REQUIRED COMPONENTS ImFusionCT)
+
+# Enable automatic MOC, RCC and UIC preprocessing for Qt
+set(CMAKE_AUTOMOC ON)
+set(CMAKE_AUTORCC ON)
+set(CMAKE_AUTOUIC ON)
+
+
+# Define and configure the CMake target
+set(Sources
+  Example2D3DRegistrationAlgorithm.cpp
+  Example2D3DRegistrationController.cpp
+  Example2D3DRegistrationFactory.cpp
+  Example2D3DRegistrationPlugin.cpp
+  Custom2D3DRegistrationInitialization.cpp
+  )
+set(Headers
+  Example2D3DRegistrationAlgorithm.h
+  Example2D3DRegistrationController.h
+  Example2D3DRegistrationFactory.h
+  Example2D3DRegistrationPlugin.h
+  Custom2D3DRegistrationInitialization.h
+  )
+
+# Define target library
+add_library(Example2D3DRegistrationPlugin SHARED ${Sources} ${UiHeaders} ${Headers})
+target_include_directories(Example2D3DRegistrationPlugin PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}
+  )
+# Link against the ImFusionLib and selected modules/plugins
+target_link_libraries(Example2D3DRegistrationPlugin PRIVATE
+  ImFusionLib
+  ImFusionCT
+  )
+
+# Define output target directories and provide instructions on how to launch
+# the ImFusion Suite with the built custom plugin.
+# These functions are provided by the ImFusionLib target config. 
+imfusion_set_common_target_properties()
+imfusion_provide_ide_instructions()
+
diff --git a/Example2D3DRegistration/Custom2D3DRegistrationInitialization.cpp b/Example2D3DRegistration/Custom2D3DRegistrationInitialization.cpp
@@ -0,0 +1,96 @@
+
+#include "Custom2D3DRegistrationInitialization.h"
+
+#include <ImFusion/Base/Log.h>
+#include <ImFusion/CT/XRay2D3DRegistrationInitialization.h>
+#include <ImFusion/CT/XRay2D3DRegistrationInitializationKeyPoints.h>
+#include <ImFusion/Core/Utils/Optional.h>
+
+#undef IMFUSION_LOG_DEFAULT_CATEGORY
+#define IMFUSION_LOG_DEFAULT_CATEGORY "Example2D3DRegistrationController"
+
+using namespace ImFusion;
+
+// In this constructor, we just created an instance of the key points initialization method we need
+// for the bundle adjustment later.
+Custom2D3DRegistrationInitialization::Custom2D3DRegistrationInitialization(XRay2D3DRegistrationAlgorithm& regAlg, mat4 groundTruthPose)
+	: XRay2D3DRegistrationInitialization(regAlg)
+	, m_kpAlg(std::make_unique<XRay2D3DRegistrationInitializationKeyPoints>(regAlg))
+	, m_groundTruthPose(groundTruthPose)
+{
+}
+
+// This class performs the initialization
+Utils::Optional<mat4> Custom2D3DRegistrationInitialization::initialize(ConeBeamGeometry& geom,
+																	   SharedImageSet& shots,
+																	   const SharedImageSet& volume,
+																	   MaskEditor* maskAlgorithm)
+{
+	if (m_kpAlg == nullptr)    // basic consistency check
+	{
+		LOG_ERROR("Custom initialization not correctly initialized");
+		return Utils::Optional<mat4>();
+	}
+
+	// We will configure the m_kpAlg member using the Properties interface, described in more
+	// detail in Properties.h and the XRay2D3DRegistrationInitializationKeyPoints.h header file.
+	Properties keyPointsConfig;
+
+	// The reference locations of three keypoints in world-coordinates are specified
+	vec3 keyPoint0World(0.0, 0.0, 0.0);
+	vec3 keyPoint1World(0.0, -30.0, 30.0);
+	vec3 keyPoint2World(0.0, 30.0, 30.0);
+	// We project these reference locations onto the x-ray images.
+	for (int i = 0; i < shots.size(); i++)
+	{
+		//Calculate forward projection onto the detector
+		vec2 kp0ForwardProjectioni = m_kpAlg->forwardProjection(keyPoint0World, i);
+		vec2 kp1ForwardProjectioni = m_kpAlg->forwardProjection(keyPoint1World, i);
+		vec2 kp2ForwardProjectioni = m_kpAlg->forwardProjection(keyPoint2World, i);
+
+		// Set values of the forward projections at "Shot{i}/keyPoint_kp{0,1}"
+		keyPointsConfig.setParam("Shot" + std::to_string(i) + "/keyPoint_kp0", vec3(kp0ForwardProjectioni[0], kp0ForwardProjectioni[1], 0.0));
+		keyPointsConfig.setParam("Shot" + std::to_string(i) + "/keyPoint_kp1", vec3(kp1ForwardProjectioni[0], kp1ForwardProjectioni[1], 0.0));
+		keyPointsConfig.setParam("Shot" + std::to_string(i) + "/keyPoint_kp2", vec3(kp2ForwardProjectioni[0], kp2ForwardProjectioni[1], 0.0));
+	}
+
+	// We use the ground truth pose of the volume to calculate locations of the keypoints on the volume
+	// for the correct pose. These are calculated in image coordinates of the volume.
+	auto toGroundTruth = [this](vec3 inputPoint) -> vec3 { return (m_groundTruthPose.inverse() * inputPoint.homogeneous()).hnormalized().eval(); };
+	vec3 keyPoint0Volume = toGroundTruth(keyPoint0World);
+	vec3 keyPoint1Volume = toGroundTruth(keyPoint1World);
+	vec3 keyPoint2Volume = toGroundTruth(keyPoint2World);
+
+	// The multiplication with volume.matrixToWorld() is needed as the algorithm expects key points in world coordinates.
+	// The following lambda function performs this multiplication.
+	auto toWorld = [&volume](vec3 inputPoint) -> vec3 { return (volume.matrixToWorld() * inputPoint.homogeneous()).hnormalized().eval(); };
+
+	//We add a bit of "noise" below so that the initialization is not exact
+	keyPointsConfig.setParam("Volume/keyPoint_kp0", (toWorld(keyPoint0Volume) + vec3(0.0, 1.0, 1.0)).eval());
+	keyPointsConfig.setParam("Volume/keyPoint_kp1", (toWorld(keyPoint1Volume) + vec3(-1.0, 0.0, 0.0)).eval());
+	keyPointsConfig.setParam("Volume/keyPoint_kp2", (toWorld(keyPoint2Volume) + vec3(0.0, 1.0, 1.0)).eval());
+
+	// This configures m_kpAlg.
+	m_kpAlg->configure(&keyPointsConfig);
+
+	// We now run the bundle adjustment. On failure, this returns Utils::Optional<mat4>()
+	Utils::Optional<mat4> result = m_kpAlg->initialize(geom, shots, volume, maskAlgorithm);
+
+	// We can also modify the values of the masks here. The below sets masks that do not
+	// crop away anything.
+	if (result)
+	{
+		for (int i = 0; i < shots.size(); i++)
+		{
+			// Set masks on shot i
+			shots.setFocus(i);
+			Properties p;
+			maskAlgorithm->configuration(&p);
+			// Minimum and maximum are normalized to [0,1]
+			p.setParam("min", vec3f(0.0, 0.0, -1.0));
+			p.setParam("max", vec3f(1.0, 1.0, 1.0));
+			maskAlgorithm->configure(&p);
+		}
+	}
+	return result;
+}
diff --git a/Example2D3DRegistration/Custom2D3DRegistrationInitialization.h b/Example2D3DRegistration/Custom2D3DRegistrationInitialization.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2012-2019 ImFusion GmbH, Munich, Germany. All rights reserved. */
+#pragma once
+
+#include <ImFusion/CT/XRay2D3DRegistrationInitialization.h>
+#include <ImFusion/CT/XRay2D3DRegistrationInitializationKeyPoints.h>
+
+
+using namespace ImFusion;
+
+class Custom2D3DRegistrationInitialization : public XRay2D3DRegistrationInitialization
+{
+public:
+	Custom2D3DRegistrationInitialization(XRay2D3DRegistrationAlgorithm& regAlg, mat4 groundTruthPose);
+
+	// Implement XRay2D3DRegistrationInitialization interface
+	bool canInitialize() const override { return true; }
+	Utils::Optional<mat4> initialize(ConeBeamGeometry& geom, SharedImageSet& shots, const SharedImageSet& volume, MaskEditor* maskAlgorithm) override;
+
+	// Additional getter method
+	XRay2D3DRegistrationInitializationKeyPoints* kpAlg() { return m_kpAlg.get(); };
+
+private:
+	std::unique_ptr<XRay2D3DRegistrationInitializationKeyPoints> m_kpAlg;
+	mat4 m_groundTruthPose;
+};
diff --git a/Example2D3DRegistration/Example2D3DRegistrationAlgorithm.cpp b/Example2D3DRegistration/Example2D3DRegistrationAlgorithm.cpp
@@ -0,0 +1,116 @@
+#include "Example2D3DRegistrationAlgorithm.h"
+
+#include <ImFusion/Base/DataList.h>
+#include <ImFusion/Base/ImageProcessing.h>
+#include <ImFusion/Base/Log.h>
+#include <ImFusion/Base/MemImage.h>
+#include <ImFusion/Base/Pose.h>
+#include <ImFusion/Base/SharedImage.h>
+#include <ImFusion/Base/SharedImageSet.h>
+#include <ImFusion/CT/ConeBeamData.h>
+#include <ImFusion/CT/ConeBeamSimulation.h>
+#include <ImFusion/CT/XRay2D3DRegistrationAlgorithm.h>
+
+// The following sets the log category for this file to "Example2D3DRegistration"
+#undef IMFUSION_LOG_DEFAULT_CATEGORY
+#define IMFUSION_LOG_DEFAULT_CATEGORY "Example2D3DRegistration"
+
+namespace ImFusion
+{
+	Example2D3DRegistrationAlgorithm::Example2D3DRegistrationAlgorithm(SharedImageSet* volumeIn)
+		: m_volumeIn(volumeIn)
+	{
+		//  Convert the volume to use unsigned values internally.
+		// ConeBeamSimulation currently uses the storage values of the volume,
+		// so this is needed to avoid negative values.
+		m_volumeIn->prepare();
+	}
+
+	Example2D3DRegistrationAlgorithm::~Example2D3DRegistrationAlgorithm()
+	{
+		/// Since the registration algorithm uses the projections, make sure to delete registration algorithm first.
+		m_regAlg = nullptr;
+		m_projections = nullptr;
+	}
+
+
+	bool Example2D3DRegistrationAlgorithm::createCompatible(const DataList& data, Algorithm** a)
+	{
+		// check requirements to create the algorithm. In this case, we want to take in a single volume.
+
+		if (data.size() != 1)
+			return false;
+
+		SharedImageSet* img = data.getImage(Data::VOLUME);
+		if (img == nullptr)
+			return false;
+
+		// requirements are met, create the algorithm if asked
+		if (a)
+		{
+			*a = new Example2D3DRegistrationAlgorithm(img);
+		}
+		return true;
+	}
+
+
+	// This function does all of the work of this class
+	void Example2D3DRegistrationAlgorithm::compute()
+	{
+		// Basic consistency check.
+		if (m_volumeIn == nullptr)
+		{
+			LOG_ERROR("Algorithm incorrectly initialized");
+			return;
+		}
+
+		// We create some simulated X-ray images.
+		ConeBeamSimulation simulation(*m_volumeIn->get(0));
+		// Use the current pose of the volume for the X-ray images by setting
+		// the iso matrix of the cone-beam geometry accordingly.
+		simulation.geometry().setIsoMatrix(m_volumeIn->matrixToWorld());
+		auto& geom = simulation.geometry();
+		geom.sourceDetDistance = 1000.0;
+		geom.sourcePatDistance = 500.0;
+		geom.detSizeX = 200;
+		geom.detSizeY = 200;
+		geom.angleRange = 90;
+		Properties p;
+		simulation.configuration(&p);
+		p.setParam("width", 384);
+		p.setParam("height", 512);
+		p.setParam("frames", 2);
+		p.setParam("i0", 0.0);
+		p.setParam("volPars/reconSize", 300);
+		simulation.configure(&p);
+		simulation.compute();    //<This runs the simulation
+
+		// We save the result
+		m_projections = simulation.takeOutput().extractFirst<ConeBeamData>();
+		// Reset the iso parameters
+		m_projections->geometry().setIsoMatrix(mat4::Identity());
+
+		// Save the ground truth pose of the matrix, and then move the volume
+
+		mat4 groundTruth = m_volumeIn->matrixToWorld();
+		// Translate by (2,1,1) and rotate by (0,5,1)
+		m_volumeIn->setMatrixToWorld(Pose::eulerToMat(vec3(30.0, 70.0, 1.0), vec3(0.0, 5.0, 1.0)) * groundTruth);
+
+		// Start an instance of XRay2D3DRegistrationAlgorithm with the volume and the projections.
+		// We set a custom initialization mode with an instance of Custom2D3DRegistrationInitialization
+		// that implements the initialization of the registration.
+		m_regAlg = std::make_unique<XRay2D3DRegistrationAlgorithm>(*m_projections, *m_volumeIn);
+		m_regAlg->p_initializationMode = XRay2D3DRegistrationAlgorithm::InitializationMode::Custom;
+		auto customInit = std::make_unique<Custom2D3DRegistrationInitialization>(*m_regAlg, groundTruth);
+		m_customInit = customInit.get();
+		m_regAlg->setCustomInitialization(std::move(customInit));
+	}
+
+
+	OwningDataList Example2D3DRegistrationAlgorithm::takeOutput()
+	{
+		// if we have produced some output, add it to the list
+		return OwningDataList(std::move(m_projections));
+	}
+
+}
diff --git a/Example2D3DRegistration/Example2D3DRegistrationAlgorithm.h b/Example2D3DRegistration/Example2D3DRegistrationAlgorithm.h
@@ -0,0 +1,48 @@
+/* Copyright (c) 2012-2019 ImFusion GmbH, Munich, Germany. All rights reserved. */
+/* Copyright (c) 2012-2019 ImFusion GmbH, Munich, Germany. All rights reserved. */
+
+#include "Custom2D3DRegistrationInitialization.h"
+
+#include <ImFusion/Base/Algorithm.h>
+#include <ImFusion/Base/AlgorithmListener.h>
+#include <ImFusion/CT/ConeBeamData.h>
+
+#include <memory>
+
+namespace ImFusion
+{
+	class SharedImageSet;
+	class XRay2D3DRegistrationAlgorithm;
+
+	// Demonstration of how to use the 2D/3D Registration algorithm from the SDK.
+	class Example2D3DRegistrationAlgorithm : public Algorithm
+	{
+	public:
+		// Creates the algorithm instance with an image
+		Example2D3DRegistrationAlgorithm(SharedImageSet* img);
+		~Example2D3DRegistrationAlgorithm();
+
+
+		// \name	Methods implementing the algorithm interface
+		//\{
+		// Factory method to check for applicability or to create the algorithm
+		static bool createCompatible(const DataList& data, Algorithm** a = 0);
+
+		void compute() override;
+
+		// If new data was created, make it available here
+		OwningDataList takeOutput() override;
+		//\}
+
+		// Getter and setter methods required by the controller.
+		XRay2D3DRegistrationAlgorithm* regAlg() { return m_regAlg.get(); }
+		Custom2D3DRegistrationInitialization* customInit() { return m_customInit; };
+
+
+	private:
+		SharedImageSet* m_volumeIn = nullptr;                            //< Volume
+		std::unique_ptr<ConeBeamData> m_projections;                     //< Projections
+		std::unique_ptr<XRay2D3DRegistrationAlgorithm> m_regAlg;         //< Nested Registration Algorithm
+		Custom2D3DRegistrationInitialization* m_customInit = nullptr;    //< Pointer to custom initialization object.
+	};
+}