Reduced file size and added transformations for VR

HandGenerator/HandGenerator.py

@@ -32,6 +32,7 @@ def __init__(self, parent=None):
   def setup(self):
     ScriptedLoadableModuleWidget.setup(self)
 
+    # This is setup for the GUI on the left which offers buttons to press
     self.parametersCollapsibleButton = ctk.ctkCollapsibleButton()
     self.parametersCollapsibleButton.text = "Actions"
     self.layout.addWidget(self.parametersCollapsibleButton)
@@ -50,7 +51,7 @@ def setup(self):
 
     self.generateButton = qt.QPushButton()
     self.generateButton.setDefault(False)
-    self.generateButton.text = "Generate Hands" 
+    self.generateButton.text = "Generate Hand Cylinders" 
     self.parametersFormLayout.addWidget(self.generateButton)
 
     self.generateModelButton = qt.QPushButton()
@@ -60,9 +61,10 @@ def setup(self):
 
     self.connectButton.connect('clicked(bool)', self.onConnectButtonClicked)
     self.generateButton.connect('clicked(bool)', self.generateCylinders)
-    self.generateButton.connect('clicked(bool)', self.generateModels)
+    self.generateModelButton.connect('clicked(bool)', self.generateModels)
     self.layout.addStretch(1)
 
+  # Connect to the Plus Server application. YOU MUST PRESS THIS BUTTON BEFORE GENERATING THE HANDS
   def onConnectButtonClicked(self):
     if self.connectorNode is not None: 
       self.connectorNode = None
@@ -76,12 +78,14 @@ def onConnectButtonClicked(self):
       self.connectCheck = 0  
       self.connectButton.text = 'Connected'
 
+  # Generates the Hand Cylinders, requires no resources from the Resources folder
   def generateCylinders(self):
     if self.generated == False:
       nodes = slicer.util.getNodesByClass('vtkMRMLLinearTransformNode')
       l = slicer.modules.createmodels.logic()
 
       # TODO: Make sure to render the palm as well!
+      # Generates the cylinders and tracks the hands movements
       for i in range (0, len(nodes)):
         if 'Left' in nodes[i].GetName() or 'Right' in nodes[i].GetName():
           if 'Dis' in nodes[i].GetName() or 'Int' in nodes[i].GetName() or 'Prox' in nodes[i].GetName() or 'Meta' in nodes[i].GetName():
@@ -105,7 +109,7 @@ def generateCylinders(self):
             cylinder.SetAndObserveTransformNodeID(nodes[i].GetID())
             cylinder.SetName('LHG_Cyl_'+nodes[i].GetName())            
       self.generated = True
-    else:
+    else: # Remove the models and calls the function again to generate them.
       nodes = slicer.util.getNodesByClass('vtkMRMLLinearTransformNode')
       models = slicer.util.getNodesByClass('vtkMRMLModelNode')
       n = 0
@@ -122,47 +126,121 @@ def generateCylinders(self):
       self.generated = False
       self.generateCylinders()
 
-
+  # This renders a hand model stored in the Resources folder
   def generateModels(self):
-    if self.generated == False:
-      self.nodes = slicer.util.getNodesByClass('vtkMRMLLinearTransformNode')
-      self.n = len(self.nodes)
+    self.resourcePath = os.path.dirname(os.path.abspath(__file__))
+    if self.generated == False: # If not generated then there are no models to remove so continue
+      self.nodes = slicer.util.getNodesByClass('vtkMRMLLinearTransformNode') # Obtain all of the LinearTransformNodes
+      self.n = len(self.nodes) # Keep track of the number of transforms
       l = slicer.modules.createmodels.logic()
-      mat = vtk.vtkMatrix4x4()
-      mat.SetElement(2,2,-1)
+
+      mat = vtk.vtkMatrix4x4() # This is a transformation matrix which maps the Leap Motion sensor (Tracker) to the HMD
+
+      ## This is a section of code I was trying to get working but isn't working quite yet.
+      #mat_axis_conv = vtk.vtkMatrix4x4()
+      #mat_table_to_mounted = vtk.vtkMatrix4x4()
+
+
+      # Old transforms
+      #mat.SetElement(0,0,-1)
+      #mat.SetElement(2,2,-1)
+
+
+      # Setting the transformation matrix TrackerToHMD, It is not a perfect mapping to that space
       mat.SetElement(0,0,-1)
-      self.Xflip = slicer.mrmlScene.AddNewNodeByClass('vtkMRMLLinearTransformNode')
-      self.Xflip.SetName('LHG_Xflip')
+      mat.SetElement(1,1,0.17)
+      mat.SetElement(1,2,-0.99)
+      mat.SetElement(1,3,-70)
+      mat.SetElement(2,1,-0.99)
+      mat.SetElement(2,2,-0.17)
+      mat.SetElement(2,3,-80)
+
+      ## This is a section of code I was trying to get working but isn't working quite yet.
+      # Transforms from https://developer.leapmotion.com/documentation/v4/vrar.html
+      # Must attach to HMD transforms (ie, make HMD transform the parent of this matrix
+      # Setting the axis_conversion
+      #mat_axis_conv.SetElement(2,2,-1) #Identity with position 2,2 (starting from 0) set to -1
+
+      ## This is a section of code I was trying to get working but isn't working quite yet.
+      # Setting the table_to_mounted matrix
+      #mat_table_to_mounted.SetElement(0,0,-1)
+      #mat_table_to_mounted.SetElement(1,1,0)
+      #mat_table_to_mounted.SetElement(1,2,-1)
+      #mat_table_to_mounted.SetElement(2,1,-1)
+      #mat_table_to_mounted.SetElement(2,2,0)
+      #mat_table_to_mounted.SetElement(2,3,-80)
+
+      ## This is a section of code I was trying to get working but isn't working quite yet.
+      #Multiply4x4(mat_axis_conv, mat_table_to_mounted, mat)
+      #vtk.vtkMatrix4x4.Multiply4x4(mat_axis_conv,mat_table_to_mounted, mat)
+
+
+      # Cleanup
+      # Xflip is a poorly named attribute, it *was* used to map the hand to the RAS coordinates,
+      # it was later changed to map from the RAS system to the HMD transform provided by the virtual reality
+      # module in 3D Slicer.
+      self.Xflip = slicer.mrmlScene.AddNewNodeByClass('vtkMRMLLinearTransformNode') # Add the transformation matrix to the scene
+      self.Xflip.SetName('LHG_TrackerToHMD')
       self.Xflip.SetAndObserveMatrixTransformToParent(mat)
+
+      ## This is a section of code I was trying to get working but isn't working quite yet.
+      ## Ideally, the transforms can be broken up but they seem to have performance issues as more transforms are
+      ## introduced so I've limited the number of them in my implementation.
+      #self.Xflip.SetAndObserveTransformNodeID(self.Xflip.GetID())  Set this to observe the VR.HMD transform
+      #self.HMDAlign = slicer.mrmlScene.AddNewNodeByClass('vtkMRMLLinearTransformNode')
+      #self.HMDAlign.SetName("LHG_RASToHMD")
+
       self.generated = True 
-      self.resourcePath = os.path.dirname(os.path.abspath(__file__))
-      print(self.resourcePath)
+
+      print(self.resourcePath) # This is not necessary but is sometimes useful for diagnosing problems when working with the module
 
 
+      # Looping to render each segment of the hand model
       for i in range (0, self.n):
+        # Filter for likely the only transforms that are needed, this could probably be changed to be more exclusive
         if 'Left' in self.nodes[i].GetName() or 'Right' in self.nodes[i].GetName():
           if 'Dis' in self.nodes[i].GetName() or 'Int' in self.nodes[i].GetName() or 'Prox' in self.nodes[i].GetName() or 'Meta' in self.nodes[i].GetName() or 'Palm' in self.nodes[i].GetName():
 
-
+            # This is not necessary but is sometimes useful for diagnosing problems when working with the module
             print('Resources\\' + self.nodes[i].GetName() + ".stl")
+
+            # The actual loading of the hand model and attaching the appropriate transforms.
+            # Hand model segments are labeled with the transform name. This will cause a problem in the future
+            # if 3D Slicer allows longer transformation names because currently they get truncated.
             self.tempModel = slicer.util.loadModel(os.path.join(self.resourcePath, 'Resources\\' + self.nodes[i].GetName() + ".stl"))
             self.nodes[i].SetAndObserveTransformNodeID(self.Xflip.GetID())
             self.tempModel.SetAndObserveTransformNodeID(self.nodes[i].GetID())
             self.tempModel.SetName('LHG_Seg' + self.nodes[i].GetName())
 
-    else:
-      self.nodes = slicer.util.getNodesByClass('vtkMRMLLinearTransformNode')
+    else: # There are models to remove so we delete them first. We also check if Xflip was properly created or not.
+      self.nodes = slicer.util.getNodesByClass('vtkMRMLLinearTransformNode') # Get the LinearTransforms in the scene
       self.n = len(self.nodes)
-      self.models = slicer.util.getNodesByClass('vtkMRMLModelNode')
-      if self.Xflip is None:
+      self.models = slicer.util.getNodesByClass('vtkMRMLModelNode') # Get the models currently in the scene
+      if self.Xflip is None: # If Xflip is not defined then create it
         mat = vtk.vtkMatrix4x4()
-        mat.SetElement(2,2,-1)
+        # Old transforms
+        #mat.SetElement(0,0,-1)
+        #mat.SetElement(2,2,-1)
+
+        # New transforms, these are decent but not a perfect mapping from virtual to real space
         mat.SetElement(0,0,-1)
+        mat.SetElement(1,1,0.17)
+        mat.SetElement(1,2,-0.99)
+        mat.SetElement(1,3,-70)
+        mat.SetElement(2,1,-0.99)
+        mat.SetElement(2,2,-0.17)
+        mat.SetElement(2,3,-80)
         self.Xflip = slicer.mrmlScene.AddNewNodeByClass('vtkMRMLLinearTransformNode')
-        self.Xflip.SetName('LHG_Xflip')
+        self.Xflip.SetName('LHG_TrackerToHMD')
         self.Xflip.SetAndObserveMatrixTransformToParent(mat)
+
+      # Remove the models
       for j in range(0, len(self.models)): 
-        slicer.mrmlScene.RemoveNode(self.models[j])
+        if 'LHG_' in self.models[j].GetName():
+          slicer.mrmlScene.RemoveNode(self.models[j])
+
+      # This part is re-rendering the hand, it is the exact same loop from the true statement and should probably be rearranged 
+      # for shorter code.
       for i in range (0, self.n):
         if 'Left' in self.nodes[i].GetName() or 'Right' in self.nodes[i].GetName():
           if 'Dis' in self.nodes[i].GetName() or 'Int' in self.nodes[i].GetName() or 'Prox' in self.nodes[i].GetName() or 'Meta' in self.nodes[i].GetName():

HandGenerator/PlusDeviceSet_Server_LeapMotionTracker Both.xml

@@ -8,7 +8,7 @@
       AcquisitionRate="1000"
       Id="TrackerDevice"
       Type="LeapMotion"
-      LeapHMDPolicy="FALSE"
+      LeapHMDPolicy="TRUE"
       ToolReferenceFrame="Tracker" >
       <DataSources>
         <DataSource Type="Video" Id="LeftCamera"/>

HandGenerator/PlusDeviceSet_Server_LeapMotionTracker Left.xml

@@ -8,7 +8,7 @@
       AcquisitionRate="1000"
       Id="TrackerDevice"
       Type="LeapMotion"
-      LeapHMDPolicy="FALSE"
+      LeapHMDPolicy="TRUE"
       ToolReferenceFrame="Tracker" >
       <DataSources>
         <DataSource Type="Video" Id="LeftCamera"/>

HandGenerator/README.txt

@@ -8,10 +8,11 @@
 What you need to run this module:
 1. 3D Slicer version 4. This works with builds from November 2019 and January 2020.
 2. PlusToolkit PlusServer. I don't know exactly which version was used but as long as it supports the Leap Motion sensor it should work.
-        In the case that it doesn't work, it might be too old a version to read in the data about finger lengths and widths, you can edit 
-        the scripted python module for Slicer on the lines that throw errors.
 3. OpenIGTLinkIF module/extension in Slicer. This can be installed through the 3D Slicer extension manager.
-
+4. A Leap Motion Sensor from https://www.ultraleap.com/product/leap-motion-controller/
+5. The Leap Motion SDK kit may be necessary. If so, install the (Orion) V4 SDK.
+Optional: The "SlicerVirtualReality" module can be used with this module and a head mounted display (virtual reality setup) to see your hand
+        in the virtual reality space where it should appear. This can be installed through the 3D Slicer extension manager.
 
 Steps to run the module:
 1. Add the "PlusDeviceSet_Server_LeapMotionTracker Right/Left/Both.xml" files that are part of this module to wherever you want them to be 
@@ -20,11 +21,11 @@ Steps to run the module:
 3. Select the directory that you stored the .xml files in step 1 where it says "Device set configuration directory:"
 4. Select the file that you wish to run for the configuration under "Device set:". These should have the name
         "PlusServer: LeapMotion Right/Left/Both Hand/Hands". 
-5. Make sure the Leap Motion sensor device is connected to your computer. This is done with a USB cable.
-6. Click the "Launch Server" button. Make sure that it boots up correctly.
+5. Make sure the Leap Motion sensor device is connected to your computer (or head mounted display). This is done with a USB cable.
+6. Click the "Launch Server" button. Make sure that it boots up correctly, this should be displayed in centre message box.
 7. Start 3D Slicer
 8. Make sure the module is added via "Application Settings" under the "Edit" tab. Instruction can be found on the Slicer discourse wiki.
-9. Select the "LeapMotionHandGenerator" module, it should be located in the "Examples" category unless the file has been edited.
+9. Select the "Hand Generator" module, it should be located in the "IGT" category.
 
 *WORKING IN THE MODULE*
 
@@ -33,16 +34,14 @@ Steps to run the module:
 11. Place your hand(s) within view of the Leap Motion sensor and then press the "Generate Hands" button.
 12. Have fun.
 
+*OPTIONAL VIRTUAL REALITY FUNCTIONALITY*
+1. Once the hands are rendered (by following the steps above), go to the Virtual Reality module and check the toggle box for the HMD transform.
+2. Navigate to the "Data" module and select "Transform hierarchy".
+3. Move the "LHG_TrackerToHMD" so that the "VR.HMD" transform is the parent of the "LHG_TrackerToHMD" transform
 
 
 ***KNOWN BUGS***
-At the time of making this README file, I (Thomas) know of one fatal bug in the program.
-1. DO NOT press the "Generate Hands" button twice. If you do the module fails and no hands will be displayed. This can be fixed by making sure
-    the version of PlusServer can properly track the finger length and width data or by removing that feature from the python module.
-
-2. If you press the "Generate Hands" button with no hands in view of the sensor no hand will be generated, even if you put the hands in 
-    view after pressing the button. To generate the hands in this case you should restart Slicer, proceed from Step 10 in the "Steps to run 
-    the module:" section, and make sure your hand is within view of the Leap Motion sensor.
-
-Note: Hopefully these bugs won't be an issue in the future.
-
+Sometimes the Leap Motion sensor will not generate the hands despite them being in view. If this happens you can take your hands out of view 
+of the Leap Motion sensor for a few seconds and place them back and then press the "Generate ..." again. This problem is due to Leap Motion 
+failing to pick up transformational data, moving your hands slightly while in view of the sensor and attempting to regenerate them tends to
+resolve this issue.