From 5d8eea72998986302a98a0ffd779145ad3fefa65 Mon Sep 17 00:00:00 2001 From: "Jade (Rose) Rowland" Date: Sun, 21 Apr 2024 17:29:29 -0400 Subject: [PATCH 1/2] fixed --- packages/core/util.mjs | 5 + packages/desktopbridge/oscbridge.mjs | 6 +- packages/midi/midi.mjs | 6 +- packages/osc/osc.mjs | 7 +- src-tauri/src/oscbridge.rs | 222 ++++++++++++++------------- 5 files changed, 130 insertions(+), 116 deletions(-) diff --git a/packages/core/util.mjs b/packages/core/util.mjs index 2c7b3d712..9ab9bc7c6 100644 --- a/packages/core/util.mjs +++ b/packages/core/util.mjs @@ -61,6 +61,11 @@ export const valueToMidi = (value, fallbackValue) => { return fallbackValue; }; +// used to schedule external event like midi and osc out +export const getEventOffsetMs = (targetTime, currentTime) => { + return (targetTime - currentTime) * 1000; +}; + /** * @deprecated does not appear to be referenced or invoked anywhere in the codebase * @noAutocomplete diff --git a/packages/desktopbridge/oscbridge.mjs b/packages/desktopbridge/oscbridge.mjs index 935ec9ff4..81366367b 100644 --- a/packages/desktopbridge/oscbridge.mjs +++ b/packages/desktopbridge/oscbridge.mjs @@ -1,8 +1,8 @@ -import { parseNumeral, Pattern } from '@strudel/core'; +import { parseNumeral, Pattern, getEventOffsetMs } from '@strudel/core'; import { Invoke } from './utils.mjs'; Pattern.prototype.osc = function () { - return this.onTrigger(async (time, hap, currentTime, cps = 1) => { + return this.onTrigger(async (time, hap, currentTime, cps = 1, targetTime) => { hap.ensureObjectValue(); const cycle = hap.wholeOrPart().begin.valueOf(); const delta = hap.duration.valueOf(); @@ -13,7 +13,7 @@ Pattern.prototype.osc = function () { const params = []; - const timestamp = Math.round(Date.now() + (time - currentTime) * 1000); + const timestamp = Math.round(Date.now() + getEventOffsetMs(targetTime, currentTime)); Object.keys(controls).forEach((key) => { const val = controls[key]; diff --git a/packages/midi/midi.mjs b/packages/midi/midi.mjs index 89812b86c..ef5c792aa 100644 --- a/packages/midi/midi.mjs +++ b/packages/midi/midi.mjs @@ -5,7 +5,7 @@ This program is free software: you can redistribute it and/or modify it under th */ import * as _WebMidi from 'webmidi'; -import { Pattern, isPattern, logger, ref } from '@strudel/core'; +import { Pattern, getEventOffsetMs, isPattern, logger, ref } from '@strudel/core'; import { noteToMidi } from '@strudel/core'; import { Note } from 'webmidi'; // if you use WebMidi from outside of this package, make sure to import that instance: @@ -120,9 +120,9 @@ Pattern.prototype.midi = function (output) { const device = getDevice(output, WebMidi.outputs); hap.ensureObjectValue(); //magic number to get audio engine to line up, can probably be calculated somehow - const latency = 0.034; + const latencyMs = 34; // passing a string with a +num into the webmidi api adds an offset to the current time https://webmidijs.org/api/classes/Output - const timeOffsetString = `+${(targetTime - currentTime + latency) * 1000}`; + const timeOffsetString = `${getEventOffsetMs(targetTime, currentTime) + latencyMs}`; // destructure value let { note, nrpnn, nrpv, ccn, ccv, midichan = 1, midicmd, gain = 1, velocity = 0.9 } = hap.value; diff --git a/packages/osc/osc.mjs b/packages/osc/osc.mjs index 14206e1d2..5552a1c14 100644 --- a/packages/osc/osc.mjs +++ b/packages/osc/osc.mjs @@ -6,7 +6,7 @@ This program is free software: you can redistribute it and/or modify it under th import OSC from 'osc-js'; -import { logger, parseNumeral, Pattern } from '@strudel/core'; +import { logger, parseNumeral, Pattern, getEventOffsetMs } from '@strudel/core'; let connection; // Promise function connect() { @@ -44,7 +44,7 @@ function connect() { * @returns Pattern */ Pattern.prototype.osc = function () { - return this.onTrigger(async (time, hap, currentTime, cps = 1) => { + return this.onTrigger(async (time, hap, currentTime, cps = 1, targetTime) => { hap.ensureObjectValue(); const osc = await connect(); const cycle = hap.wholeOrPart().begin.valueOf(); @@ -56,7 +56,8 @@ Pattern.prototype.osc = function () { const keyvals = Object.entries(controls).flat(); // time should be audio time of onset // currentTime should be current time of audio context (slightly before time) - const offset = (time - currentTime) * 1000; + const offset = getEventOffsetMs(targetTime, currentTime); + // timestamp in milliseconds used to trigger the osc bundle at a precise moment const ts = Math.floor(Date.now() + offset); const message = new OSC.Message('/dirt/play', ...keyvals); diff --git a/src-tauri/src/oscbridge.rs b/src-tauri/src/oscbridge.rs index 6060cd9dc..c8dcf03af 100644 --- a/src-tauri/src/oscbridge.rs +++ b/src-tauri/src/oscbridge.rs @@ -1,22 +1,22 @@ -use rosc::{ encoder, OscTime }; -use rosc::{ OscMessage, OscPacket, OscType, OscBundle }; +use rosc::{encoder, OscTime}; +use rosc::{OscBundle, OscMessage, OscPacket, OscType}; use std::net::UdpSocket; -use std::time::Duration; -use std::sync::Arc; -use tokio::sync::{ mpsc, Mutex }; use serde::Deserialize; +use std::sync::Arc; use std::thread::sleep; +use std::time::Duration; +use tokio::sync::{mpsc, Mutex}; use crate::loggerbridge::Logger; pub struct OscMsg { - pub msg_buf: Vec, - pub timestamp: u64, + pub msg_buf: Vec, + pub timestamp: u64, } pub struct AsyncInputTransmit { - pub inner: Mutex>>, + pub inner: Mutex>>, } const UNIX_OFFSET: u64 = 2_208_988_800; // 70 years in seconds @@ -25,133 +25,141 @@ const NANOS_PER_SECOND: f64 = 1.0e9; const SECONDS_PER_NANO: f64 = 1.0 / NANOS_PER_SECOND; pub fn init( - logger: Logger, - async_input_receiver: mpsc::Receiver>, - mut async_output_receiver: mpsc::Receiver>, - async_output_transmitter: mpsc::Sender> + logger: Logger, + async_input_receiver: mpsc::Receiver>, + mut async_output_receiver: mpsc::Receiver>, + async_output_transmitter: mpsc::Sender>, ) { - tauri::async_runtime::spawn(async move { async_process_model(async_input_receiver, async_output_transmitter).await }); - let message_queue: Arc>> = Arc::new(Mutex::new(Vec::new())); - /* ........................................................... - Listen For incoming messages and add to queue - ............................................................*/ - let message_queue_clone = Arc::clone(&message_queue); - tauri::async_runtime::spawn(async move { - loop { - if let Some(package) = async_output_receiver.recv().await { - let mut message_queue = message_queue_clone.lock().await; - let messages = package; - for message in messages { - (*message_queue).push(message); - } - } - } - }); - - let message_queue_clone = Arc::clone(&message_queue); - tauri::async_runtime::spawn(async move { - /* ........................................................... - Open OSC Ports - ............................................................*/ - let sock = UdpSocket::bind("127.0.0.1:57122").unwrap(); - let to_addr = String::from("127.0.0.1:57120"); - sock.set_nonblocking(true).unwrap(); - sock.connect(to_addr).expect("could not connect to OSC address"); - + tauri::async_runtime::spawn(async move { + async_process_model(async_input_receiver, async_output_transmitter).await + }); + let message_queue: Arc>> = Arc::new(Mutex::new(Vec::new())); /* ........................................................... - Process queued messages + Listen For incoming messages and add to queue ............................................................*/ - - loop { - let mut message_queue = message_queue_clone.lock().await; - - message_queue.retain(|message| { - let result = sock.send(&message.msg_buf); - if result.is_err() { - logger.log( - format!("OSC Message failed to send, the server might no longer be available"), - "error".to_string() - ); + let message_queue_clone = Arc::clone(&message_queue); + tauri::async_runtime::spawn(async move { + loop { + if let Some(package) = async_output_receiver.recv().await { + let mut message_queue = message_queue_clone.lock().await; + let messages = package; + for message in messages { + (*message_queue).push(message); + } + } } - return false; - }); + }); - sleep(Duration::from_millis(1)); - } - }); + let message_queue_clone = Arc::clone(&message_queue); + tauri::async_runtime::spawn(async move { + /* ........................................................... + Open OSC Ports + ............................................................*/ + let sock = UdpSocket::bind("127.0.0.1:57122").unwrap(); + let to_addr = String::from("127.0.0.1:57120"); + sock.set_nonblocking(true).unwrap(); + sock.connect(to_addr) + .expect("could not connect to OSC address"); + + /* ........................................................... + Process queued messages + ............................................................*/ + + loop { + let mut message_queue = message_queue_clone.lock().await; + + message_queue.retain(|message| { + let result = sock.send(&message.msg_buf); + if result.is_err() { + logger.log( + format!( + "OSC Message failed to send, the server might no longer be available" + ), + "error".to_string(), + ); + } + return false; + }); + + sleep(Duration::from_millis(1)); + } + }); } pub async fn async_process_model( - mut input_reciever: mpsc::Receiver>, - output_transmitter: mpsc::Sender> + mut input_reciever: mpsc::Receiver>, + output_transmitter: mpsc::Sender>, ) -> Result<(), Box> { - while let Some(input) = input_reciever.recv().await { - let output = input; - output_transmitter.send(output).await?; - } - Ok(()) + while let Some(input) = input_reciever.recv().await { + let output = input; + output_transmitter.send(output).await?; + } + Ok(()) } #[derive(Deserialize)] pub struct Param { - name: String, - value: String, - valueisnumber: bool, + name: String, + value: String, + valueisnumber: bool, } #[derive(Deserialize)] pub struct MessageFromJS { - params: Vec, - timestamp: u64, - target: String, + params: Vec, + timestamp: u64, + target: String, } // Called from JS #[tauri::command] pub async fn sendosc( - messagesfromjs: Vec, - state: tauri::State<'_, AsyncInputTransmit> + messagesfromjs: Vec, + state: tauri::State<'_, AsyncInputTransmit>, ) -> Result<(), String> { - let async_proc_input_tx = state.inner.lock().await; - let mut messages_to_process: Vec = Vec::new(); - for m in messagesfromjs { - let mut args = Vec::new(); - for p in m.params { - args.push(OscType::String(p.name)); - if p.valueisnumber { - args.push(OscType::Float(p.value.parse().unwrap())); - } else { - args.push(OscType::String(p.value)); - } - } + let async_proc_input_tx = state.inner.lock().await; + let mut messages_to_process: Vec = Vec::new(); + for m in messagesfromjs { + let mut args = Vec::new(); + for p in m.params { + args.push(OscType::String(p.name)); + if p.valueisnumber { + args.push(OscType::Float(p.value.parse().unwrap())); + } else { + args.push(OscType::String(p.value)); + } + } - let duration_since_epoch = Duration::from_millis(m.timestamp) + Duration::new(UNIX_OFFSET, 0); + let duration_since_epoch = + Duration::from_millis(m.timestamp) + Duration::new(UNIX_OFFSET, 0); - let seconds = u32 - ::try_from(duration_since_epoch.as_secs()) - .map_err(|_| "bit conversion failed for osc message timetag")?; + let seconds = u32::try_from(duration_since_epoch.as_secs()) + .map_err(|_| "bit conversion failed for osc message timetag")?; - let nanos = duration_since_epoch.subsec_nanos() as f64; - let fractional = (nanos * SECONDS_PER_NANO * TWO_POW_32).round() as u32; + let nanos = duration_since_epoch.subsec_nanos() as f64; + let fractional = (nanos * SECONDS_PER_NANO * TWO_POW_32).round() as u32; - let timetag = OscTime::from((seconds, fractional)); + let timetag = OscTime::from((seconds, fractional)); - let packet = OscPacket::Message(OscMessage { - addr: m.target, - args, - }); + let packet = OscPacket::Message(OscMessage { + addr: m.target, + args, + }); - let bundle = OscBundle { - content: vec![packet], - timetag, - }; + let bundle = OscBundle { + content: vec![packet], + timetag, + }; - let msg_buf = encoder::encode(&OscPacket::Bundle(bundle)).unwrap(); + let msg_buf = encoder::encode(&OscPacket::Bundle(bundle)).unwrap(); - let message_to_process = OscMsg { - msg_buf, - timestamp: m.timestamp, - }; - messages_to_process.push(message_to_process); - } + let message_to_process = OscMsg { + msg_buf, + timestamp: m.timestamp, + }; + messages_to_process.push(message_to_process); + } - async_proc_input_tx.send(messages_to_process).await.map_err(|e| e.to_string()) + async_proc_input_tx + .send(messages_to_process) + .await + .map_err(|e| e.to_string()) } From 57ad278137f4797fe8dfbd5666b5451a806529b5 Mon Sep 17 00:00:00 2001 From: "Jade (Rose) Rowland" Date: Sun, 21 Apr 2024 17:52:22 -0400 Subject: [PATCH 2/2] fixed osc server --- packages/core/util.mjs | 4 ++-- packages/desktopbridge/midibridge.mjs | 6 +++--- packages/midi/midi.mjs | 3 +-- packages/osc/server.js | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/core/util.mjs b/packages/core/util.mjs index 9ab9bc7c6..e8e403988 100644 --- a/packages/core/util.mjs +++ b/packages/core/util.mjs @@ -62,8 +62,8 @@ export const valueToMidi = (value, fallbackValue) => { }; // used to schedule external event like midi and osc out -export const getEventOffsetMs = (targetTime, currentTime) => { - return (targetTime - currentTime) * 1000; +export const getEventOffsetMs = (targetTimeSeconds, currentTimeSeconds) => { + return (targetTimeSeconds - currentTimeSeconds) * 1000; }; /** diff --git a/packages/desktopbridge/midibridge.mjs b/packages/desktopbridge/midibridge.mjs index 07a5f963f..471c826ea 100644 --- a/packages/desktopbridge/midibridge.mjs +++ b/packages/desktopbridge/midibridge.mjs @@ -1,5 +1,5 @@ import { Invoke } from './utils.mjs'; -import { Pattern, noteToMidi } from '@strudel/core'; +import { Pattern, getEventOffsetMs, noteToMidi } from '@strudel/core'; const ON_MESSAGE = 0x90; const OFF_MESSAGE = 0x80; @@ -9,8 +9,8 @@ Pattern.prototype.midi = function (output) { return this.onTrigger((time_deprecate, hap, currentTime, cps, targetTime) => { let { note, nrpnn, nrpv, ccn, ccv, velocity = 0.9, gain = 1 } = hap.value; //magic number to get audio engine to line up, can probably be calculated somehow - const latency = 0.034; - const offset = (targetTime - currentTime + latency) * 1000; + const latencyMs = 34; + const offset = getEventOffsetMs(targetTime, currentTime) + latencyMs; velocity = Math.floor(gain * velocity * 100); const duration = Math.floor((hap.duration.valueOf() / cps) * 1000 - 10); const roundedOffset = Math.round(offset); diff --git a/packages/midi/midi.mjs b/packages/midi/midi.mjs index ef5c792aa..3ce894d64 100644 --- a/packages/midi/midi.mjs +++ b/packages/midi/midi.mjs @@ -122,8 +122,7 @@ Pattern.prototype.midi = function (output) { //magic number to get audio engine to line up, can probably be calculated somehow const latencyMs = 34; // passing a string with a +num into the webmidi api adds an offset to the current time https://webmidijs.org/api/classes/Output - const timeOffsetString = `${getEventOffsetMs(targetTime, currentTime) + latencyMs}`; - + const timeOffsetString = `+${getEventOffsetMs(targetTime, currentTime) + latencyMs}`; // destructure value let { note, nrpnn, nrpv, ccn, ccv, midichan = 1, midicmd, gain = 1, velocity = 0.9 } = hap.value; diff --git a/packages/osc/server.js b/packages/osc/server.js index 42722d285..7e0b90592 100644 --- a/packages/osc/server.js +++ b/packages/osc/server.js @@ -4,7 +4,7 @@ Copyright (C) 2022 Strudel contributors - see . */ -const OSC = require('osc-js'); +import OSC from 'osc-js'; const config = { receiver: 'ws', // @param {string} Where messages sent via 'send' method will be delivered to, 'ws' for Websocket clients, 'udp' for udp client