Skip to content

Commit

Permalink
Merge pull request #258 from AugmentOS-Community/better-feedback
Browse files Browse the repository at this point in the history
virtual wearable
  • Loading branch information
alex1115alex authored Mar 8, 2025
2 parents e731958 + 9e5c29a commit b1ce04f
Show file tree
Hide file tree
Showing 36 changed files with 1,422 additions and 48 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,14 @@ export class SessionService {

addTranscriptSegment(userSession: ExtendedUserSession, segment: TranscriptSegment): void {
if (userSession) {
// Add new segment
userSession.transcript.segments.push(segment);

// Prune old segments (older than 30 minutes)
const thirtyMinutesAgo = new Date(Date.now() - 30 * 60 * 1000);
userSession.transcript.segments = userSession.transcript.segments.filter(
seg => seg.timestamp && new Date(seg.timestamp) >= thirtyMinutesAgo
);
}
}

Expand Down Expand Up @@ -189,9 +196,31 @@ export class SessionService {
if (userSession.lc3Service) {
userSession.lc3Service.cleanup();
}

// Clean up subscription history for this session
const subscriptionService = require('./subscription.service').default;
if (subscriptionService && typeof subscriptionService.removeSessionSubscriptionHistory === 'function') {
subscriptionService.removeSessionSubscriptionHistory(sessionId);
}

// Clear transcript history
if (userSession.transcript && userSession.transcript.segments) {
userSession.transcript.segments = [];
}

// Clean other data structures
if (userSession.bufferedAudio) {
userSession.bufferedAudio = [];
}

this.activeSessions.delete(sessionId);
userSession.logger.info(`[Ended session] ${sessionId}`);

// Suggest garbage collection if available
if (global.gc) {
console.log('🧹 Running garbage collection after ending session');
global.gc();
}
}

getAllSessions(): ExtendedUserSession[] {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,29 @@ import { StreamType, ExtendedStreamType, isLanguageStream, UserSession, parseLan
console.log(`Removed all subscriptions for ${packageName} in session ${userSession.sessionId}`);
}
}

/**
* Removes all subscription history for a session
* Used when a session is being killed to free memory
* @param sessionId - User session identifier
*/
removeSessionSubscriptionHistory(sessionId: string): void {
// Find all keys that start with this session ID
const keysToRemove: string[] = [];

for (const key of this.history.keys()) {
if (key.startsWith(`${sessionId}:`)) {
keysToRemove.push(key);
}
}

// Remove all history entries for this session
keysToRemove.forEach(key => {
this.history.delete(key);
});

console.log(`Removed subscription history for session ${sessionId} (${keysToRemove.length} entries)`);
}

/**
* Checks if a TPA has a specific subscription
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -313,9 +313,11 @@ export class TranscriptionService {
private updateTranscriptHistory(userSession: ExtendedUserSession, event: ConversationTranscriptionEventArgs, isFinal: boolean): void {
const segments = userSession.transcript.segments;
const hasInterimLast = segments.length > 0 && !segments[segments.length - 1].isFinal;
// Only save engligh transcriptions.
// Only save English transcriptions.
if (event.result.language !== 'en-US') return;

const currentTime = new Date();

if (isFinal) {
if (hasInterimLast) {
segments.pop();
Expand All @@ -324,7 +326,7 @@ export class TranscriptionService {
resultId: event.result.resultId,
speakerId: event.result.speakerId,
text: event.result.text,
timestamp: new Date(),
timestamp: currentTime,
isFinal: true
});
} else {
Expand All @@ -333,19 +335,25 @@ export class TranscriptionService {
resultId: event.result.resultId,
speakerId: event.result.speakerId,
text: event.result.text,
timestamp: new Date(),
timestamp: currentTime,
isFinal: false
};
} else {
segments.push({
resultId: event.result.resultId,
speakerId: event.result.speakerId,
text: event.result.text,
timestamp: new Date(),
timestamp: currentTime,
isFinal: false
});
}
}

// Prune old segments (older than 30 minutes)
const thirtyMinutesAgo = new Date(currentTime.getTime() - 30 * 60 * 1000);
userSession.transcript.segments = segments.filter(
seg => seg.timestamp && new Date(seg.timestamp) >= thirtyMinutesAgo
);
}
}

Expand Down
43 changes: 38 additions & 5 deletions augmentos_cloud/packages/utils/src/lc3/LC3Service3.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,16 +93,20 @@ export class LC3Service {
if (!this.initialized || !this.decoder) {
await this.initialize();
}

let localInputData: Uint8Array | null = null;

try {
const numFrames = Math.floor(audioData.byteLength / this.frameBytes);
const totalSamples = numFrames * this.frameSamples;
const outputBuffer = new ArrayBuffer(totalSamples * 2); // 16-bit PCM
const outputView = new DataView(outputBuffer);
const inputData = new Uint8Array(audioData);
localInputData = new Uint8Array(audioData);
let outputOffset = 0;

for (let i = 0; i < numFrames; i++) {
this.decoder!.frame.set(
inputData.subarray(i * this.frameBytes, (i + 1) * this.frameBytes)
localInputData.subarray(i * this.frameBytes, (i + 1) * this.frameBytes)
);
this.decoder!.decode();
for (let j = 0; j < this.frameSamples; j++) {
Expand All @@ -114,17 +118,46 @@ export class LC3Service {
outputOffset += 2;
}
}

// Update last used timestamp
if (this.decoder) {
this.decoder.lastUsed = Date.now();
}

return outputBuffer;
} catch (error) {
console.error('❌ Error decoding LC3 audio:', error);
return null;
} finally {
// Release references to input data to help GC
// This is safe even if there was an error
localInputData = null;
}
}

cleanup(): void {
this.initialized = false;
this.lc3Exports = null;
this.decoder = null;
try {
if (this.decoder && this.lc3Exports) {
// Force garbage collection of the ArrayBuffer views
this.decoder.samples = new Float32Array(0);
this.decoder.frame = new Uint8Array(0);

// If WebAssembly instances support explicit cleanup in future, add it here

// Log memory usage before clearing references
if (global.gc) {
console.log('🧹 Running garbage collection for LC3Service cleanup');
global.gc();
}
}
} catch (error) {
console.error('Error during LC3Service cleanup:', error);
} finally {
// Clear all references to allow garbage collection
this.initialized = false;
this.lc3Exports = null;
this.decoder = null;
}
}
}

Expand Down
31 changes: 31 additions & 0 deletions augmentos_core/CLAUDE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# AugmentOS Core Development Guide

## Build Commands
- `./gradlew build` - Full project build
- `./gradlew assembleDebug` - Build debug APK
- `./gradlew test` - Run unit tests
- `./gradlew androidTest` - Run instrumentation tests

## Environment Setup
- Java SDK 17 required
- AugmentOS_Core depends on "SmartGlassesManager" repo being adjacent

## Code Style Guidelines
- Classes: PascalCase (e.g., `WebSocketManager`)
- Methods: camelCase (e.g., `isConnected()`)
- Constants: UPPER_SNAKE_CASE (e.g., `MAX_RETRY_ATTEMPTS`)
- Member variables: camelCase with m prefix (e.g., `mService`)
- Indentation: 2 spaces
- Javadoc comments for public methods and classes

## Error Handling
- Log errors with `Log.e(TAG, "Error message", e)`
- Use callbacks for asynchronous error propagation
- Handle all checked exceptions
- Provide user feedback for permissions issues

## Architecture
- Service-based design with `AugmentosService` as main component
- Fragment-based UI with Navigation Component
- EventBus for component communication
- WebSocket for server communication
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package com.augmentos.augmentos_core.smarterglassesmanager.smartglassescommunicators;

import android.content.Context;
import android.graphics.Bitmap;
import android.util.Log;

import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.GlassesBluetoothSearchDiscoverEvent;
import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.TextToSpeechEvent;
import com.augmentos.augmentos_core.smarterglassesmanager.supportedglasses.SmartGlassesDevice;
import com.augmentos.augmentos_core.smarterglassesmanager.utils.SmartGlassesConnectionState;

import org.greenrobot.eventbus.EventBus;

public class VirtualSGC extends SmartGlassesCommunicator {
private static final String TAG = "WearableAi_AndroidWearableSGC";


Context context;
SmartGlassesDevice smartGlassesDevice;

public VirtualSGC(Context context, SmartGlassesDevice smartGlassesDevice){
super();
mConnectState = SmartGlassesConnectionState.DISCONNECTED;
this.smartGlassesDevice = smartGlassesDevice;
}

public void setFontSizes(){
}

public void connectToSmartGlasses(){
connectionEvent(SmartGlassesConnectionState.CONNECTED);
}

public void blankScreen(){
}

public void displayRowsCard(String[] rowStrings){

}

@Override
public void destroy() {
mConnectState = SmartGlassesConnectionState.DISCONNECTED;
this.context = null;
this.smartGlassesDevice = null;
Log.d(TAG, "VirtualSGC destroyed successfully.");
}


public void displayReferenceCardSimple(String title, String body){}

public void displayReferenceCardImage(String title, String body, String imgUrl){}

public void displayBulletList(String title, String [] bullets){}

public void displayBulletList(String title, String [] bullets, int lingerTime){}

public void displayTextWall(String text){}
public void displayDoubleTextWall(String textTop, String textBottom){}

public void stopScrollingTextViewMode() {
}

public void startScrollingTextViewMode(String title){
}

public void scrollingTextViewIntermediateText(String text){
}

public void scrollingTextViewFinalText(String text){
}

public void showHomeScreen(){
}

public void displayPromptView(String prompt, String [] options){
}

public void displayTextLine(String text){}

@Override
public void displayBitmap(Bitmap bmp) {

}

@Override
public void displayCustomContent(String json) {}

@Override
public void findCompatibleDeviceNames() {
EventBus.getDefault().post(new GlassesBluetoothSearchDiscoverEvent(smartGlassesDevice.deviceModelName, "NOTREQUIREDSKIP"));
this.destroy();
}

public void showNaturalLanguageCommandScreen(String prompt, String naturalLanguageArgs){}

public void updateNaturalLanguageCommandScreen(String naturalLanguageArgs){}

public void setFontSize(SmartGlassesFontSize fontSize){}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.NewAsrLanguagesEvent;
import com.augmentos.augmentos_core.smarterglassesmanager.smartglassescommunicators.SmartGlassesFontSize;
import com.augmentos.augmentos_core.smarterglassesmanager.comms.MessageTypes;
import com.augmentos.augmentos_core.smarterglassesmanager.supportedglasses.special.VirtualWearable;
import com.augmentos.augmentoslib.events.BulletPointListViewRequestEvent;
import com.augmentos.augmentoslib.events.CenteredTextViewRequestEvent;
import com.augmentos.augmentoslib.events.DoubleTextWallViewRequestEvent;
Expand Down Expand Up @@ -420,7 +421,8 @@ public static SmartGlassesDevice getSmartGlassesDeviceFromModelName(String model
new VuzixShield(),
new InmoAirOne(),
new TCLRayNeoXTwo(),
new AudioWearable()
new AudioWearable(),
new VirtualWearable()
)
);

Expand Down Expand Up @@ -582,6 +584,11 @@ public void changeMicrophoneState(boolean isMicrophoneEnabled) {
Log.d(TAG, "Want to changing microphone state to " + isMicrophoneEnabled);
Log.d(TAG, "Force core onboard mic: " + getForceCoreOnboardMic(this.getApplicationContext()));

if(smartGlassesRepresentative == null || smartGlassesRepresentative.smartGlassesDevice == null) {
Log.d(TAG, "Cannot change microphone state: smartGlassesRepresentative or smartGlassesDevice is null");
return;
}

// If we're trying to turn ON the microphone
if (isMicrophoneEnabled) {
// Cancel any pending turn-off operations
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.AudioChunkNewEvent;
import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.DisableBleScoAudioEvent;
import com.augmentos.augmentos_core.smarterglassesmanager.smartglassescommunicators.VirtualSGC;
import com.augmentos.augmentos_core.smarterglassesmanager.smartglassescommunicators.special.SelfSGC;
import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.LC3AudioChunkNewEvent;
import com.augmentos.augmentoslib.events.DisplayCustomContentRequestEvent;
Expand Down Expand Up @@ -131,6 +132,8 @@ private SmartGlassesCommunicator createCommunicator() {
return new AndroidSGC(context, smartGlassesDevice, dataObservable);
case AUDIO_WEARABLE_GLASSES:
return new AudioWearableSGC(context, smartGlassesDevice);
case VIRTUAL_WEARABLE:
return new VirtualSGC(context, smartGlassesDevice);
case ULTRALITE_MCU_OS_GLASSES:
return new UltraliteSGC(context, smartGlassesDevice, lifecycleOwner);
case EVEN_REALITIES_G1_MCU_OS_GLASSES:
Expand Down Expand Up @@ -297,7 +300,7 @@ public void onHomeScreenEvent(HomeScreenEvent receivedEvent){
@Subscribe
public void onTextWallViewEvent(TextWallViewRequestEvent receivedEvent){
if (smartGlassesCommunicator != null) {
Log.d(TAG, "SINGLE TEXT WALL BOOM");
// Log.d(TAG, "SINGLE TEXT WALL BOOM");
smartGlassesCommunicator.displayTextWall(receivedEvent.text);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,6 @@ public enum SmartGlassesOperatingSystem {
ULTRALITE_MCU_OS_GLASSES,
EVEN_REALITIES_G1_MCU_OS_GLASSES,
INMO_GO_MCU_OS_GLASSES,
AUDIO_WEARABLE_GLASSES
AUDIO_WEARABLE_GLASSES,
VIRTUAL_WEARABLE
}
Loading

0 comments on commit b1ce04f

Please sign in to comment.