Skip to content

Commit

Permalink
Fix double teardown, implement eye image and external tracking module…
Browse files Browse the repository at this point in the history 
… support for it, converted lip image to float array
  • Loading branch information
@benaclejames
benaclejames committed Jun 6, 2022
1 parent 89ac7ac commit daa61aa
Show file tree
Hide file tree
Showing 12 changed files with 163 additions and 77 deletions.
7 changes: 3 additions & 4 deletions VRCFT Module Example/ExternalTrackingModule.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ public class ExternalExtTrackingModule : ExtTrackingModule
{
// Synchronous module initialization. Take as much time as you need to initialize any external modules. This runs in the init-thread
public override (bool SupportsEye, bool SupportsLip) Supported => (true, true);
public override (bool UtilizingEye, bool UtilizingLip) Utilizing { get; set; }

public override (bool eyeSuccess, bool lipSuccess) Initialize(bool eye, bool lip)
{
Expand All @@ -65,13 +64,13 @@ public override Action GetUpdateThreadFunc()
}

// The update function needs to be defined separately in case the user is running with the --vrcft-nothread launch parameter
public override void Update()
public void Update()
{
Console.WriteLine("Updating inside external module.");

if (Utilizing.UtilizingEye)
if (Status.EyeState == ModuleState.Active)
Console.WriteLine("Eye data is being utilized.");
if (Utilizing.UtilizingLip)
if (Status.LipState == ModuleState.Active)
Console.WriteLine("Lip data is being utilized.");
}

Expand Down
4 changes: 2 additions & 2 deletions VRCFaceTracking/MainStandalone.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,9 @@ public static void Initialize()

if (_relevantParamsCount <= 0)
continue;

UnifiedTrackingData.OnUnifiedDataUpdated.Invoke(UnifiedTrackingData.LatestEyeData,
UnifiedTrackingData.LatestLipShapes);
UnifiedTrackingData.LatestLipData);

var bundle = new OscBundle(ConstructMessages(_relevantParams));

Expand Down
2 changes: 1 addition & 1 deletion VRCFaceTracking/MainWindow.xaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@

</TabItem>
<TabItem Header="Eye Tracking" Foreground="White">
<Image Margin="22,22,22,22"/>
<Image x:Name="EyeImage" Height="478" Margin="22,0,22,0"/>
</TabItem>
<TabItem Header="Lip Tracking" Foreground="White">
<Image x:Name="LipImage" Height="478" Margin="22,0,22,0"/>
Expand Down
30 changes: 22 additions & 8 deletions VRCFaceTracking/MainWindow.xaml.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,24 +71,40 @@ public MainWindow()
while (!MainStandalone.MasterCancellationTokenSource.IsCancellationRequested)
{
Thread.Sleep(10);
Dispatcher.BeginInvoke(new ThreadStart(() => UpdateLipImage()));
Dispatcher.BeginInvoke(new ThreadStart(() =>
{
UpdateEyeImage();
UpdateLipImage();
}));
}
}).Start();
}

void UpdateLipImage()
{
if (UnifiedTrackingData.Image == IntPtr.Zero) // If the image is not initialized
if (UnifiedTrackingData.LatestLipData.ImageData == null) // If the image is not initialized
return;

byte[] managedArray = new byte[800*400];
Marshal.Copy(UnifiedTrackingData.Image, managedArray, 0, 800*400);
var bitmap = new WriteableBitmap(800, 400, 96, 96, PixelFormats.Gray8, null);
bitmap.WritePixels(new Int32Rect(0, 0, 800, 400), managedArray, 800, 0);
var bitmap = new WriteableBitmap(UnifiedTrackingData.LatestLipData.ImageSize.x, UnifiedTrackingData.LatestLipData.ImageSize.y, 96, 96, PixelFormats.Gray8, null);
bitmap.WritePixels(new Int32Rect(0, 0, UnifiedTrackingData.LatestLipData.ImageSize.x, UnifiedTrackingData.LatestLipData.ImageSize.y), UnifiedTrackingData.LatestLipData.ImageData, 800, 0);

// Set the WPF image name LipImage
LipImage.Source = bitmap;
}

void UpdateEyeImage()
{
if (UnifiedTrackingData.LatestEyeData.ImageData == null) // If the image is not initialized
return;

var bitmap = new WriteableBitmap(UnifiedTrackingData.LatestEyeData.ImageSize.x,
UnifiedTrackingData.LatestEyeData.ImageSize.y, 96, 96, PixelFormats.Gray8, null);
bitmap.WritePixels(new Int32Rect(0, 0, UnifiedTrackingData.LatestEyeData.ImageSize.x,
UnifiedTrackingData.LatestEyeData.ImageSize.y), UnifiedTrackingData.LatestEyeData.ImageData, UnifiedTrackingData.LatestEyeData.ImageSize.x, 0);

// Set the WPF image name EyeImage
EyeImage.Source = bitmap;
}

public ObservableCollection<Tuple<string, string>> ConsoleOutput => Logger.ConsoleOutput;

Expand Down Expand Up @@ -118,7 +134,6 @@ private void PauseClickEyes(object sender, RoutedEventArgs e)
return;

UnifiedLibManager.EyeStatus = UnifiedLibManager.EyeStatus == ModuleState.Idle ? ModuleState.Active : ModuleState.Idle;
Logger.Msg(UnifiedLibManager.EyeStatus == ModuleState.Idle ? "Eyes Paused" : "Eyes Unpaused");
}

private void PauseClickMouth(object sender, RoutedEventArgs e)
Expand All @@ -127,7 +142,6 @@ private void PauseClickMouth(object sender, RoutedEventArgs e)
return;

UnifiedLibManager.LipStatus = UnifiedLibManager.LipStatus == ModuleState.Idle ? ModuleState.Active : ModuleState.Idle;
Logger.Msg(UnifiedLibManager.LipStatus == ModuleState.Idle ? "Mouth Paused" : "Mouth Unpaused");
}

private void UpdateLogo(ModuleState eyeState, ModuleState lipState)
Expand Down
38 changes: 17 additions & 21 deletions VRCFaceTracking/Params/Lip/LipShapeConversion.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,41 +6,41 @@ namespace VRCFaceTracking.Params.Lip
{
public interface ICombinedShape
{
float GetBlendedLipShape(Dictionary<LipShape_v2, float> inputMap);
float GetBlendedLipShape(float[] inputMap);
}

public class PositiveNegativeShape : ICombinedShape
{
private readonly LipShape_v2 _positiveShape, _negativeShape;
private readonly int _positiveShape, _negativeShape;
private float _positiveCache, _negativeCache;
private bool _steps;

public PositiveNegativeShape(LipShape_v2 positiveShape, LipShape_v2 negativeShape, bool steps = false)
{
_positiveShape = positiveShape;
_negativeShape = negativeShape;
_positiveShape = (int)positiveShape;
_negativeShape = (int)negativeShape;
_steps = steps;
}

public float GetBlendedLipShape(Dictionary<LipShape_v2, float> inputMap)
public float GetBlendedLipShape(float[] inputMap)
{
if (inputMap.TryGetValue(_positiveShape, out var positiveResult)) _positiveCache = positiveResult;
if (inputMap.TryGetValue(_negativeShape, out var negativeResult)) _negativeCache = negativeResult * -1;
_positiveCache = inputMap[_positiveShape];
_negativeCache = inputMap[_negativeShape] * -1;
return _steps ? (_positiveCache - _negativeCache) - 1 : _positiveCache + _negativeCache;
}
}

public class PositiveNegativeAveragedShape : ICombinedShape
{
private readonly LipShape_v2[] _positiveShapes, _negativeShapes;
private readonly int[] _positiveShapes, _negativeShapes;
private readonly float[] _positiveCache, _negativeCache;
private readonly int _positiveCount, _negativeCount;
private readonly bool _useMax;

public PositiveNegativeAveragedShape(LipShape_v2[] positiveShapes, LipShape_v2[] negativeShapes)
{
_positiveShapes = positiveShapes;
_negativeShapes = negativeShapes;
_positiveShapes = positiveShapes.Select(s => (int)s).ToArray();
_negativeShapes = negativeShapes.Select(s => (int)s).ToArray();
_positiveCache = new float[positiveShapes.Length];
_negativeCache = new float[negativeShapes.Length];
_positiveCount = positiveShapes.Length;
Expand All @@ -49,45 +49,41 @@ public PositiveNegativeAveragedShape(LipShape_v2[] positiveShapes, LipShape_v2[]

public PositiveNegativeAveragedShape(LipShape_v2[] positiveShapes, LipShape_v2[] negativeShapes, bool useMax)
{
_positiveShapes = positiveShapes;
_negativeShapes = negativeShapes;
_positiveShapes = positiveShapes.Select(s => (int)s).ToArray();
_negativeShapes = negativeShapes.Select(s => (int)s).ToArray();
_positiveCache = new float[positiveShapes.Length];
_negativeCache = new float[negativeShapes.Length];
_positiveCount = positiveShapes.Length;
_negativeCount = negativeShapes.Length;
_useMax = useMax;
}

public float GetBlendedLipShape(Dictionary<LipShape_v2, float> inputMap)
public float GetBlendedLipShape(float[] inputMap)
{
if (!_useMax)
{
float positive = 0;
float negative = 0;

for (int i = 0; i < _positiveCount; i++) {
if (inputMap.TryGetValue(_positiveShapes[i], out var positiveResult))
_positiveCache[i] = positiveResult;
_positiveCache[i] = inputMap[_positiveShapes[i]];
positive += _positiveCache[i];
}

for (int i = 0; i < _negativeCount; i++) {
if (inputMap.TryGetValue(_negativeShapes[i], out var negativeResult))
_negativeCache[i] = negativeResult * -1;
_negativeCache[i] = inputMap[_negativeShapes[i]] * -1;
negative += _negativeCache[i];
}

return (positive / _positiveCount) + (negative / _negativeCount);
}

for (int i = 0; i < _positiveCount; i++) {
if (inputMap.TryGetValue(_positiveShapes[i], out var positiveResult))
_positiveCache[i] = positiveResult;
_positiveCache[i] = inputMap[_positiveShapes[i]];
}

for (int i = 0; i < _negativeCount; i++) {
if (inputMap.TryGetValue(_negativeShapes[i], out var negativeResult))
_negativeCache[i] = negativeResult;
_negativeCache[i] = inputMap[_negativeShapes[i]];
}

return _positiveCache.Max() + (-1) * _negativeCache.Max();
Expand Down
4 changes: 2 additions & 2 deletions VRCFaceTracking/Params/Lip/LipShapeMerger.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,11 @@ public static class LipShapeMerger

private static IEnumerable<EParam> GetOptimizedLipParameters() => MergedShapes
.Select(shape => new EParam((eye, lip) =>
shape.Value.GetBlendedLipShape(lip), shape.Key, 0.0f)).ToList();
shape.Value.GetBlendedLipShape(lip.LatestShapes), shape.Key, 0.0f)).ToList();

private static IEnumerable<EParam> GetAllLipShapes() =>
((LipShape_v2[]) Enum.GetValues(typeof(LipShape_v2))).ToList().Select(shape =>
new EParam((eye, lip) => lip.TryGetValue(shape, out var outValue) ? outValue : (float?) null,
new EParam((eye, lip) => lip.LatestShapes[(int)shape],
shape.ToString(), 0.0f));
}
}
10 changes: 5 additions & 5 deletions VRCFaceTracking/Params/ParamContainers.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ namespace VRCFaceTracking.Params
{
public class FloatParameter : OSCParams.FloatBaseParam, IParameter
{
public FloatParameter(Func<EyeTrackingData, Dictionary<LipShape_v2, float>, float?> getValueFunc,
public FloatParameter(Func<EyeTrackingData, LipTrackingData, float?> getValueFunc,
string paramName)
: base(paramName) =>
UnifiedTrackingData.OnUnifiedDataUpdated += (eye, lip) =>
Expand All @@ -24,7 +24,7 @@ public FloatParameter(Func<EyeTrackingData, Dictionary<LipShape_v2, float>, floa

public class XYParameter : XYParam, IParameter
{
public XYParameter(Func<EyeTrackingData, Dictionary<LipShape_v2, float>, Vector2?> getValueFunc, string xParamName, string yParamName)
public XYParameter(Func<EyeTrackingData, LipTrackingData, Vector2?> getValueFunc, string xParamName, string yParamName)
: base(new OSCParams.FloatBaseParam(xParamName), new OSCParams.FloatBaseParam(yParamName)) =>
UnifiedTrackingData.OnUnifiedDataUpdated += (eye, lip) =>
{
Expand All @@ -45,7 +45,7 @@ public XYParameter(Func<EyeTrackingData, Vector2> getValueFunc, string xParamNam

public class BoolParameter : OSCParams.BoolBaseParam, IParameter
{
public BoolParameter(Func<EyeTrackingData, Dictionary<LipShape_v2, float>, bool?> getValueFunc,
public BoolParameter(Func<EyeTrackingData, LipTrackingData, bool?> getValueFunc,
string paramName) : base(paramName) =>
UnifiedTrackingData.OnUnifiedDataUpdated += (eye, lip) =>
{
Expand All @@ -67,7 +67,7 @@ public OSCParams.BaseParam[] GetBase()

public class BinaryParameter : OSCParams.BinaryBaseParameter, IParameter
{
public BinaryParameter(Func<EyeTrackingData, Dictionary<LipShape_v2, float>, float?> getValueFunc,
public BinaryParameter(Func<EyeTrackingData, LipTrackingData, float?> getValueFunc,
string paramName) : base(paramName)
{
UnifiedTrackingData.OnUnifiedDataUpdated += (eye, lip) =>
Expand Down Expand Up @@ -99,7 +99,7 @@ public class EParam : IParameter
private readonly IParameter[] _parameter;
private readonly string Name;

public EParam(Func<EyeTrackingData, Dictionary<LipShape_v2, float>, float?> getValueFunc, string paramName, float minBoolThreshold = 0.5f, bool skipBinaryParamCreation = false)
public EParam(Func<EyeTrackingData, LipTrackingData, float?> getValueFunc, string paramName, float minBoolThreshold = 0.5f, bool skipBinaryParamCreation = false)
{
var paramLiterals = new List<IParameter>
{
Expand Down
7 changes: 2 additions & 5 deletions VRCFaceTracking/TrackingLibs/SRanipal/Lip/SRanipal_Lip_v2.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ public class SRanipal_Lip_v2
public const int ANIPAL_TYPE_LIP_V2 = 3;

public const int ImageWidth = 800, ImageHeight = 400, ImageChannel = 1;
public const int WeightingCount = 37;
public const int WeightingCount = 36;
private static Error LastUpdateResult = Error.FAILED;
public static LipData_v2 LipData;
private static Dictionary<LipShape_v2, float> Weightings;
Expand All @@ -33,10 +33,7 @@ private static unsafe bool UpdateData()
{
for (int i = 0; i < WeightingCount; ++i)
{
unsafe
{
Weightings[(LipShape_v2)i] = LipData.prediction_data.blend_shape_weight[i];
}
Weightings[(LipShape_v2)i] = LipData.prediction_data.blend_shape_weight[i];
}
}
return LastUpdateResult == Error.WORK;
Expand Down
Loading

0 comments on commit daa61aa

Please sign in to comment.