Integrate Whisper CPP and write a wrapper module in Aprapipes

base/CMakeLists.txt

@@ -1,11 +1,14 @@
 cmake_minimum_required(VERSION 3.22)
 
-OPTION(ENABLE_LINUX "Use this switch to enable LINUX" ON)
-OPTION(ENABLE_CUDA "Use this switch to enable CUDA" ON)
+OPTION(ENABLE_LINUX "Use this switch to enable LINUX" OFF)
+OPTION(ENABLE_CUDA "Use this switch to enable CUDA" OFF)
 OPTION(ENABLE_ARM64 "Use this switch to enable ARM64" OFF)
-OPTION(ENABLE_WINDOWS "Use this switch to enable WINDOWS" OFF)
+OPTION(ENABLE_WINDOWS "Use this switch to enable WINDOWS" ON)
+OPTION(ENABLE_WHISPER "Use Whisper ASR" ON)
 
 set(VCPKG_INSTALL_OPTIONS "--clean-after-build")
+set(VCPKG_OVERLAY_PORTS "${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/custom-overlay")
+
 IF(ENABLE_CUDA)
 	add_compile_definitions(APRA_CUDA_ENABLED)
 ENDIF(ENABLE_CUDA)
@@ -38,7 +41,10 @@ project(APRAPIPES)
 message(STATUS $ENV{PKG_CONFIG_PATH}">>>>>> PKG_CONFIG_PATH")
 
 find_package(PkgConfig REQUIRED)
-
+
+IF(ENABLE_WHISPER)        
+	find_package(whisper CONFIG REQUIRED)
+ENDIF(ENABLE_WHISPER)
 
 find_package(Boost COMPONENTS system thread filesystem serialization log chrono unit_test_framework REQUIRED)
 find_package(JPEG REQUIRED)
@@ -281,9 +287,7 @@ SET(IP_FILES
 	src/OverlayFactory.cpp
 	src/TestSignalGeneratorSrc.cpp
 )
-
-
-
+
 SET(IP_FILES_H
 	include/HistogramOverlay.h
 	include/CalcHistogramCV.h
@@ -308,6 +312,10 @@ SET(IP_FILES_H
 	include/Overlay.h
 )
 
+IF(ENABLE_WHISPER)
+	set(IP_FILES ${IP_FILES} src/WhisperStreamTransform.cpp )
+	set(IP_FILES_H ${IP_FILES_H} include/WhisperStreamTransform.h )
+ENDIF(ENABLE_WHISPER)
 
 
 SET(CUDA_CORE_FILES
@@ -461,6 +469,10 @@ ${NVCODEC_INCLUDE_DIR}
 
 # aprapipes Unit Tests
 
+IF(ENABLE_WHISPER)
+	SET(WHISPER_UT_FILES test/whisper_asr_tests.cpp)
+ENDIF(ENABLE_WHISPER)
+
 IF (ENABLE_ARM64)
 	SET(ARM64_UT_FILES
 		test/jpegencoderl4tm_tests.cpp
@@ -563,6 +575,7 @@ SET(UT_FILES
 	test/testSignalGeneratorSrc_tests.cpp
 	${ARM64_UT_FILES}
 	${CUDA_UT_FILES}
+	${WHISPER_UT_FILES}
 )
 
 IF(ENABLE_LINUX)
@@ -609,6 +622,10 @@ target_link_libraries(aprapipesut
   sfml-audio
   )
 
+IF(ENABLE_WHISPER)
+    target_link_libraries(aprapipesut whisper::whisper)
+ENDIF(ENABLE_WHISPER)
+
 IF(ENABLE_WINDOWS)
 	file(COPY ${RUNTIME_DLLS} DESTINATION Debug/)
 	file(COPY ${RUNTIME_DLLS} DESTINATION Release/)

base/fix-vcpkg-json.ps1

@@ -1,8 +1,18 @@
 #inplace fixing of a vcpkg file
-param([String]$fileName='vcpkg.json', [switch]$removeOpenCV,  [switch]$removeCUDA, [switch]$onlyOpenCV)
+param([String]$fileName='vcpkg.json', [switch]$removeOpenCV,  [switch]$removeCUDA, [switch]$onlyOpenCV, [switch]$removeWhisper)
 
 $v = Get-Content $fileName -raw | ConvertFrom-Json
 
+if($removeWhisper.IsPresent){
+    $result = $v.overrides | Where-Object { $_.name -ne 'whisper' } 
+    # Check if the result is $null or empty and set it to an empty array if it is
+    if (-not $result) {
+        $result = @()
+    }
+    $v.overrides = $result
+    $v.dependencies = $v.dependencies | Where-Object { $_ -ne 'whisper'}
+}
+
 if ($removeCUDA.IsPresent)
 {
     $v.dependencies |

base/include/FrameMetadata.h

@@ -50,7 +50,8 @@ class FrameMetadata {
 		HEVC_DATA, //H265
 		MOTION_VECTOR_DATA,
 		OVERLAY_INFO_IMAGE,
-		FACE_LANDMARKS_INFO
+		FACE_LANDMARKS_INFO,
+		TEXT
 	};
 
 	enum MemType

base/include/Mp4WriterSinkUtils.h

@@ -1,4 +1,7 @@
 #include <ctime>
+#include <chrono>
+#include <string>
+#include <boost/filesystem.hpp>
 
 class Mp4WriterSinkUtils
 {

base/include/WhisperStreamTransform.h

@@ -0,0 +1,50 @@
+#pragma once
+
+#include "Module.h"
+
+// size of audio to process should be a parameter. 
+// Cache variable to collect frames for processing
+
+class WhisperStreamTransformProps : public ModuleProps
+{
+public:
+	enum DecoderSamplingStrategy {
+		GREEDY,      //WHISPER_SAMPLING_GREEDY
+		BEAM_SEARCH //WHISPER_SAMPLING_BEAM_SEARCH
+	};
+	WhisperStreamTransformProps(
+		DecoderSamplingStrategy _samplingStrategy,
+		std::string _modelPath,
+		int _bufferSize) : samplingStrategy(_samplingStrategy),
+		modelPath(_modelPath),
+		bufferSize(_bufferSize)
+	{}
+    DecoderSamplingStrategy samplingStrategy;
+    std::string modelPath;
+	int bufferSize;
+};
+
+class WhisperStreamTransform  : public Module
+{
+
+public:
+	WhisperStreamTransform(WhisperStreamTransformProps _props);
+	virtual ~WhisperStreamTransform();
+	bool init();
+	bool term();
+	void setProps(WhisperStreamTransformProps& props);
+	WhisperStreamTransformProps getProps();
+
+protected:
+	bool process(frame_container& frames);
+	bool processSOS(frame_sp& frame);
+	bool validateInputPins();
+	bool validateOutputPins();
+	void addInputPin(framemetadata_sp& metadata, string& pinId);
+	bool handlePropsChange(frame_sp& frame);
+
+private:
+	void setMetadata(framemetadata_sp& metadata);
+	class Detail;
+	boost::shared_ptr<Detail> mDetail;
+};

base/src/WhisperStreamTransform.cpp

@@ -0,0 +1,180 @@
+#include "WhisperStreamTransform.h"
+#include "FrameMetadata.h"
+#include "FrameMetadataFactory.h"
+#include "Frame.h"
+#include "Logger.h"
+#include "Utils.h"
+#include "whisper.h"
+#include "SFML/Config.hpp"
+
+class WhisperStreamTransform::Detail
+{
+public:
+	Detail(WhisperStreamTransformProps& _props) : mProps(_props)
+	{
+	}
+	~Detail() {}
+
+	void setProps(WhisperStreamTransformProps& props)
+	{
+		mProps = props;
+	}
+
+public:
+	framemetadata_sp mOutputMetadata;
+	std::string mOutputPinId;
+	std::vector<float> inputAudioBuffer;
+	WhisperStreamTransformProps mProps;
+	int mFrameType;
+	whisper_context *mWhisperContext = NULL;
+	whisper_full_params mWhisperFullParams;
+	whisper_context_params mWhisperContextParams;
+};
+
+WhisperStreamTransform::WhisperStreamTransform(WhisperStreamTransformProps _props) : Module(TRANSFORM, "WhisperStreamTransform", _props)
+{
+	mDetail.reset(new Detail(_props));
+}
+
+WhisperStreamTransform::~WhisperStreamTransform() {}
+
+bool WhisperStreamTransform::validateInputPins()
+{
+	if (getNumberOfInputPins() != 1)
+	{
+		LOG_ERROR << "<" << getId() << ">::validateInputPins size is expected to be 1. Actual<" << getNumberOfInputPins() << ">";
+		return false;
+	}
+
+	framemetadata_sp metadata = getFirstInputMetadata();
+
+	FrameMetadata::FrameType frameType = metadata->getFrameType();
+	if (frameType != FrameMetadata::AUDIO)
+	{
+		LOG_ERROR << "<" << getId() << ">::validateInputPins input frameType is expected to be Audio. Actual<" << frameType << ">";
+		return false;
+	}
+
+	return true;
+}
+
+bool WhisperStreamTransform::validateOutputPins()
+{
+	if (getNumberOfOutputPins() != 1)
+	{
+		LOG_ERROR << "<" << getId() << ">::validateOutputPins size is expected to be 1. Actual<" << getNumberOfOutputPins() << ">";
+		return false;
+	}
+
+	framemetadata_sp metadata = getFirstOutputMetadata();
+	FrameMetadata::FrameType frameType = metadata->getFrameType();
+	if (frameType != FrameMetadata::TEXT)
+	{
+		LOG_ERROR << "<" << getId() << ">::validateOutputPins input frameType is expected to be TEXT. Actual<" << frameType << ">";
+		return false;
+	}
+
+	return true;
+}
+
+void WhisperStreamTransform::addInputPin(framemetadata_sp& metadata, string& pinId)
+{
+	Module::addInputPin(metadata, pinId);
+	mDetail->mOutputMetadata = framemetadata_sp(new FrameMetadata(FrameMetadata::FrameType::TEXT));
+	mDetail->mOutputMetadata->copyHint(*metadata.get());
+	mDetail->mOutputPinId = addOutputPin(mDetail->mOutputMetadata);
+}
+
+bool WhisperStreamTransform::init()
+{
+	//intialize model
+	auto samplingStrategy = whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY;
+	switch (mDetail->mProps.samplingStrategy)
+	{
+		case WhisperStreamTransformProps::DecoderSamplingStrategy::GREEDY:
+			samplingStrategy = whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY;
+			break;
+		case WhisperStreamTransformProps::DecoderSamplingStrategy::BEAM_SEARCH:
+			samplingStrategy = whisper_sampling_strategy::WHISPER_SAMPLING_BEAM_SEARCH;
+			break;
+		default:
+			samplingStrategy = whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY;
+	}
+	mDetail->mWhisperFullParams = whisper_full_default_params(samplingStrategy);
+	mDetail->mWhisperContextParams = whisper_context_default_params();
+	mDetail->mWhisperContext = whisper_init_from_file_with_params(mDetail->mProps.modelPath.c_str(), mDetail->mWhisperContextParams);
+	return Module::init();
+}
+
+bool WhisperStreamTransform::term()
+{
+	whisper_free_context_params(&mDetail->mWhisperContextParams);
+	whisper_free_params(&mDetail->mWhisperFullParams);
+	whisper_free(mDetail->mWhisperContext);
+	return Module::term();
+}
+
+bool WhisperStreamTransform::process(frame_container& frames)
+{
+	auto frame = frames.begin()->second;
+	sf::Int16* constFloatPointer = static_cast<sf::Int16*>(frame->data());
+	int numberOfSamples = frame->size() / 2;
+	for (int index = 0; index < numberOfSamples; index++) {
+		mDetail->inputAudioBuffer.push_back((float)constFloatPointer[index]/ 32768.0f);
+	}
+	if (mDetail->inputAudioBuffer.size() < mDetail->mProps.bufferSize) {
+		return true;
+	}
+	whisper_full(
+		mDetail->mWhisperContext,
+		mDetail->mWhisperFullParams,
+		mDetail->inputAudioBuffer.data(),
+		mDetail->inputAudioBuffer.size()
+	);
+	std::string output = "";
+	const int n_segments = whisper_full_n_segments(mDetail->mWhisperContext);
+	for (int i = 0; i < n_segments; ++i) {
+		const char* text = whisper_full_get_segment_text(mDetail->mWhisperContext, i);
+		output += text;
+	}
+	mDetail->inputAudioBuffer.clear();
+	auto outFrame = makeFrame(output.length());
+	memcpy(outFrame->data(), output.c_str(), output.length());
+	frames.insert(make_pair(mDetail->mOutputPinId, outFrame));
+	send(frames);
+	return true;
+}
+
+void WhisperStreamTransform::setMetadata(framemetadata_sp& metadata)
+{
+	if (!metadata->isSet())
+	{
+		return;
+	}
+}
+
+bool WhisperStreamTransform::processSOS(frame_sp& frame)
+{
+	auto metadata = frame->getMetadata();
+	setMetadata(metadata);
+	return true;
+}
+
+WhisperStreamTransformProps WhisperStreamTransform::getProps()
+{
+	fillProps(mDetail->mProps);
+	return mDetail->mProps;
+}
+
+bool WhisperStreamTransform::handlePropsChange(frame_sp& frame)
+{
+	WhisperStreamTransformProps props(mDetail->mProps.samplingStrategy, mDetail->mProps.modelPath,32000);
+	auto ret = Module::handlePropsChange(frame, props);
+	mDetail->setProps(props);
+	return ret;
+}
+
+void WhisperStreamTransform::setProps(WhisperStreamTransformProps& props)
+{
+	Module::addPropsToQueue(props);
+}