Skip to content
This repository has been archived by the owner on Apr 18, 2024. It is now read-only.

Commit

Permalink
feat: LSDV-5252: Audio player segment visual identification (#1494)
Browse files Browse the repository at this point in the history 
* feat: LSDV-5298: Paragraphs must sync seek, play and pause with Audio in both directions

* removing unused SyncMixin

* Allow playingId to correctly track over time

* updating tests for sync with contextual scrolling on/off

* removing unused TimeSync class

* adding test for contextual scrolling state change during playback

* feat: LSDV-5252: Audio player segment visual identification

* fix region loading

* adding tests to ensure audio regions do not regress

* fix the loading of audio when not synced

* trimming down tests ran with both flag states to keep runtime in check

* trying to reduce gh action times as it is taking way too long

* fix e2e name, was treating as two inputs

* linting

* we don't want node_modules in the artifacts

* download path not required since tar has dir within

* remove node_modules from artifact upload of cypress test suite

* linting

* don't need the download paths now

* leave snapshots for comparison

* add snapshots for testing audio paragraph segment loading

* update test helpers

* update to include image snapshot coverage reports

* update snapshots

* update snapshots

* fix px ratio

* check just the canvas drawing area to avoid text difference

* update snapshots

* update snapshots

* update to use percentage image diff

* fix parallel coverage reports

* fix custom tasks

* add audio playback tests with paragraph segment audio highlighting

* fix linting

* fix visual match naming options

* remove diff files

* put FF setting in a before each
  • Loading branch information
@bmartel
bmartel authored Jul 12, 2023
1 parent 0807819 commit 9a1a0d3
Show file tree
Hide file tree
Showing 22 changed files with 737 additions and 234 deletions.
320 changes: 168 additions & 152 deletions e2e/tests/audio/audio-regions.test.js
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
const { FFlagMatrix, FFlagScenario } = require('../../utils/feature-flags');

Feature('Audio Regions');

const config = `
Expand Down Expand Up @@ -73,210 +75,224 @@ const annotations = [
const params = { annotations: [{ id: 'test', result: annotations }], config, data };
const paramsSpeech = { annotations: [{ id: 'test', result: [] }], config: configSpeech, data };

Scenario('Check if regions are selected', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
LabelStudio.setFeatureFlags({
ff_front_dev_2715_audio_3_280722_short: true,
});
I.amOnPage('/');
FFlagMatrix([
'fflag_feat_front_lsdv_e_278_contextual_scrolling_short',
], function(flags) {

LabelStudio.init(params);
FFlagScenario('Check if regions are selected', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
LabelStudio.setFeatureFlags({
ff_front_dev_2715_audio_3_280722_short: true,
...flags,
});
I.amOnPage('/');

await AtAudioView.waitForAudio();
await AtAudioView.lookForStage();
LabelStudio.init(params);

AtSidebar.seeRegions(1);
await AtAudioView.waitForAudio();
await AtAudioView.lookForStage();

// creating a new region
I.pressKey('1');
AtAudioView.dragAudioElement(160,80);
I.pressKey('u');
AtSidebar.seeRegions(1);

AtSidebar.seeRegions(2);
// creating a new region
I.pressKey('1');
AtAudioView.dragAudioElement(160,80);
I.pressKey('u');

AtAudioView.clickAt(170);
AtSidebar.seeSelectedRegion();
AtAudioView.clickAt(170);
AtSidebar.dontSeeSelectedRegion();
AtAudioView.dragAudioElement(170,40);
AtSidebar.seeSelectedRegion();
AtAudioView.clickAt(220);
AtSidebar.dontSeeSelectedRegion();
});
AtSidebar.seeRegions(2);

Scenario('Check if multiple regions are working changing labels', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
LabelStudio.setFeatureFlags({
ff_front_dev_2715_audio_3_280722_short: true,
AtAudioView.clickAt(170);
AtSidebar.seeSelectedRegion();
AtAudioView.clickAt(170);
AtSidebar.dontSeeSelectedRegion();
AtAudioView.dragAudioElement(170,40);
AtSidebar.seeSelectedRegion();
AtAudioView.clickAt(220);
AtSidebar.dontSeeSelectedRegion();
});
I.amOnPage('/');

LabelStudio.init(paramsSpeech);
// Don't need to test this for both scenarios of flags, as it is the same code and is verified in the above test
if (!flags['fflag_feat_front_lsdv_e_278_contextual_scrolling_short']) {
FFlagScenario('Check if multiple regions are working changing labels', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
LabelStudio.setFeatureFlags({
ff_front_dev_2715_audio_3_280722_short: true,
...flags,
});
I.amOnPage('/');

await AtAudioView.waitForAudio();
await AtAudioView.lookForStage();
LabelStudio.init(paramsSpeech);

for (let i = 0; i < 20; i++) {
// creating a new region
I.pressKey('1');
AtAudioView.dragAudioElement((40 * i) + 10,30);
AtAudioView.clickAt((40 * i) + 20);
I.pressKey('2');
I.pressKey('1');
I.pressKey('u');
}
await AtAudioView.waitForAudio();
await AtAudioView.lookForStage();

AtSidebar.seeRegions(20);
for (let i = 0; i < 20; i++) {
// creating a new region
I.pressKey('1');
AtAudioView.dragAudioElement((40 * i) + 10,30);
AtAudioView.clickAt((40 * i) + 20);
I.pressKey('2');
I.pressKey('1');
I.pressKey('u');
}

for (let i = 0; i < 20; i++) {
// creating a new region
AtAudioView.clickAt((40 * i) + 20);
AtSidebar.seeSelectedRegion();
I.pressKey('u');
}
AtSidebar.seeRegions(20);

AtSidebar.seeRegions(20);
for (let i = 0; i < 20; i++) {
// creating a new region
AtAudioView.clickAt((40 * i) + 20);
AtSidebar.seeSelectedRegion();
I.pressKey('u');
}

I.pressKey('u');
AtSidebar.seeRegions(20);

AtSidebar.dontSeeSelectedRegion();
});
I.pressKey('u');

Scenario('Can select a region below a hidden region', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
LabelStudio.setFeatureFlags({
ff_front_dev_2715_audio_3_280722_short: true,
});
I.amOnPage('/');
AtSidebar.dontSeeSelectedRegion();
});

LabelStudio.init(paramsSpeech);
FFlagScenario('Can select a region below a hidden region', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
LabelStudio.setFeatureFlags({
ff_front_dev_2715_audio_3_280722_short: true,
...flags,
});
I.amOnPage('/');

await AtAudioView.waitForAudio();
await AtAudioView.lookForStage();
LabelStudio.init(paramsSpeech);

// create a new region
I.pressKey('1');
AtAudioView.dragAudioElement(50, 80);
I.pressKey('u');
await AtAudioView.waitForAudio();
await AtAudioView.lookForStage();

AtSidebar.seeRegions(1);
// create a new region
I.pressKey('1');
AtAudioView.dragAudioElement(50, 80);
I.pressKey('u');

// create a new region above the first one
I.pressKey('2');
AtAudioView.dragAudioElement(49, 81);
I.pressKey('u');
AtSidebar.seeRegions(1);

AtSidebar.seeRegions(2);
// create a new region above the first one
I.pressKey('2');
AtAudioView.dragAudioElement(49, 81);
I.pressKey('u');

// click on the top-most region visible to select it
AtAudioView.clickAt(50);
AtSidebar.seeSelectedRegion('Noise');
AtSidebar.seeRegions(2);

// hide the region
AtSidebar.hideRegion('Noise');
// click on the top-most region visible to select it
AtAudioView.clickAt(50);
AtSidebar.seeSelectedRegion('Noise');

// click on the region below the hidden one to select it
AtAudioView.clickAt(50);
AtSidebar.seeSelectedRegion('Speech');
});
// hide the region
AtSidebar.hideRegion('Noise');

Scenario('Selecting a region brings it to the front of the stack', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
LabelStudio.setFeatureFlags({
ff_front_dev_2715_audio_3_280722_short: true,
});
I.amOnPage('/');
// click on the region below the hidden one to select it
AtAudioView.clickAt(50);
AtSidebar.seeSelectedRegion('Speech');
});

LabelStudio.init(paramsSpeech);
FFlagScenario('Selecting a region brings it to the front of the stack', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
LabelStudio.setFeatureFlags({
ff_front_dev_2715_audio_3_280722_short: true,
...flags,
});
I.amOnPage('/');

await AtAudioView.waitForAudio();
await AtAudioView.lookForStage();
LabelStudio.init(paramsSpeech);

// create a new region
I.pressKey('1');
AtAudioView.dragAudioElement(50, 80);
I.pressKey('u');
await AtAudioView.waitForAudio();
await AtAudioView.lookForStage();

AtSidebar.seeRegions(1);
// create a new region
I.pressKey('1');
AtAudioView.dragAudioElement(50, 80);
I.pressKey('u');

// create a new region above the first one
I.pressKey('2');
AtAudioView.dragAudioElement(49, 81);
I.pressKey('u');
AtSidebar.seeRegions(1);

AtSidebar.seeRegions(2);
// create a new region above the first one
I.pressKey('2');
AtAudioView.dragAudioElement(49, 81);
I.pressKey('u');

// click on the top-most region visible to select it
AtAudioView.clickAt(50);
AtSidebar.seeSelectedRegion('Noise');
AtSidebar.seeRegions(2);

// Select the bottom most region to bring it to the top
AtSidebar.clickRegion('Speech');
AtSidebar.seeSelectedRegion('Speech');
// click on the top-most region visible to select it
AtAudioView.clickAt(50);
AtSidebar.seeSelectedRegion('Noise');

// click on the overlapping region will deselect it, which shows that it is now the top in the list
AtAudioView.clickAt(50);
AtSidebar.dontSeeSelectedRegion('Speech');
AtSidebar.dontSeeSelectedRegion('Noise');
// Select the bottom most region to bring it to the top
AtSidebar.clickRegion('Speech');
AtSidebar.seeSelectedRegion('Speech');

// click on the overlapping region will select the top item of the list, which will now be the item which was brought to the front by the original interaction.
AtAudioView.clickAt(50);
AtSidebar.seeSelectedRegion('Speech');
});
// click on the overlapping region will deselect it, which shows that it is now the top in the list
AtAudioView.clickAt(50);
AtSidebar.dontSeeSelectedRegion('Speech');
AtSidebar.dontSeeSelectedRegion('Noise');

Scenario('Delete region by pressing delete hotkey', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
LabelStudio.setFeatureFlags({
ff_front_dev_2715_audio_3_280722_short: true,
});
I.amOnPage('/');
// click on the overlapping region will select the top item of the list, which will now be the item which was brought to the front by the original interaction.
AtAudioView.clickAt(50);
AtSidebar.seeSelectedRegion('Speech');
});

LabelStudio.init(params);
FFlagScenario('Delete region by pressing delete hotkey', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
LabelStudio.setFeatureFlags({
ff_front_dev_2715_audio_3_280722_short: true,
...flags,
});
I.amOnPage('/');

await AtAudioView.waitForAudio();
await AtAudioView.lookForStage();
LabelStudio.init(params);

AtSidebar.seeRegions(1);
await AtAudioView.waitForAudio();
await AtAudioView.lookForStage();

// creating a new region
AtAudioView.dragAudioElement(160,80);
AtSidebar.seeRegions(1);

I.pressKey('Delete');
// creating a new region
AtAudioView.dragAudioElement(160,80);

I.pressKey('1');
I.pressKey('Delete');

AtSidebar.seeRegions(1);
});
I.pressKey('1');

Scenario('Check if there are ghost regions', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
LabelStudio.setFeatureFlags({
ff_front_dev_2715_audio_3_280722_short: true,
});
I.amOnPage('/');
AtSidebar.seeRegions(1);
});

FFlagScenario('Check if there are ghost regions', async function({ I, LabelStudio, AtAudioView, AtSidebar }) {
LabelStudio.setFeatureFlags({
ff_front_dev_2715_audio_3_280722_short: true,
...flags,
});
I.amOnPage('/');

LabelStudio.init(paramsSpeech);
LabelStudio.init(paramsSpeech);

await AtAudioView.waitForAudio();
await AtAudioView.lookForStage();
await AtAudioView.waitForAudio();
await AtAudioView.lookForStage();

// creating a new region
I.pressKey('1');
AtAudioView.dragAudioElement(300,80);
I.pressKey('u');
// creating a new region
I.pressKey('1');
AtAudioView.dragAudioElement(300,80);
I.pressKey('u');


// creating a ghost region
I.pressKey('1');
AtAudioView.dragAudioElement(160,80, false);
I.pressKey('1');
I.wait(1);
I.pressMouseUp();
I.wait(1);
// creating a ghost region
I.pressKey('1');
AtAudioView.dragAudioElement(160,80, false);
I.pressKey('1');
I.wait(1);
I.pressMouseUp();
I.wait(1);

// checking if the created region is selected
AtAudioView.clickAt(310);
AtSidebar.seeSelectedRegion();
// checking if the created region is selected
AtAudioView.clickAt(310);
AtSidebar.seeSelectedRegion();

// trying to select the ghost region, if there is no ghost region, the region will keep selected
// as ghost region is not selectable and impossible to change the label, the created region will be deselected if there is a ghost region created.
AtAudioView.clickAt(170);
I.pressKey('2');
AtSidebar.seeSelectedRegion();
// trying to select the ghost region, if there is no ghost region, the region will keep selected
// as ghost region is not selectable and impossible to change the label, the created region will be deselected if there is a ghost region created.
AtAudioView.clickAt(170);
I.pressKey('2');
AtSidebar.seeSelectedRegion();

AtSidebar.seeRegions(2);
AtSidebar.seeRegions(2);
});
}
});
3 changes: 2 additions & 1 deletion src/components/TimeDurationControl/TimeBox.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ export const TimeBox: FC<TimerProps> = ({
inverted = false,
readonly = false,
onChange,
...props
}) => {
const inputRef = React.createRef<HTMLInputElement>();
const [currentInputTime, setCurrentInputTime] = useState<string | number | undefined>(value);
Expand Down Expand Up @@ -100,7 +101,7 @@ export const TimeBox: FC<TimerProps> = ({
};

return (
<Block name="time-box" mod={{ inverted, sidepanel }}>
<Block name="time-box" mod={{ inverted, sidepanel }} {...props}>
{renderInputTime()}
</Block>
);
Expand Down
Loading

0 comments on commit 9a1a0d3

Please sign in to comment.