Skip to content

Commit

Permalink
adding support for captions import
Browse files Browse the repository at this point in the history
possibility to import captions along side video, and have those converted into hypertranscript to use in autoEdit
  • Loading branch information
pietrop committed Jul 13, 2018
1 parent a3896d9 commit f41e968
Show file tree
Hide file tree
Showing 13 changed files with 6,424 additions and 32 deletions.
2 changes: 2 additions & 0 deletions electron/db.js
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,8 @@ DB.create = function(model, success, error){
// destFolder:"/media",
tmpWorkFolder: tmpMediaFolder,
destFolder: mediaFolder,
// for caption file conversion
captionFilePath: newElement.captionFilePath,
keys: {
watson: window.IBMWatsonKeys(),
speechmatics: window.SpeechmaticsKeys(),
Expand Down
13 changes: 13 additions & 0 deletions electron/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,19 @@
});
}

window.openCaptionsFileDiaglogue = function(cb){
dialog.showOpenDialog(
{
properties: ['openFile'],
filters: [
{ name: 'All Files', extensions: ['srt']}
]
},
function(fileName){
if(cb){cb(fileName)};
});
}

}
</script>

Expand Down
2 changes: 2 additions & 0 deletions lib/app/models/transcription.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ module.exports = Backbone.Model.extend({
audioFile: undefined,
processedAudio: false,
processedVideo: false,
// for caption fiel converion
captionFilePath: "",
// status is marked as false by default and turned to true when transcription has been processed
// could changed as status marked as null if there's an issue
// so that can have 3 options. not set yet, gone wrong, success.
Expand Down
25 changes: 16 additions & 9 deletions lib/app/templates/transcription_form_template.html.ejs
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,10 @@
<div class="form-group">
<label for="fileUpload">Choose an audio or video file to transcribe</label>
<% if(window.process !== undefined ){%>
<button id="btnElectronInputMediaFile" class="btn btn-default btn-xs" class="very-sweet-looking">Choose a File</button>
<input id="inputElectronInputMediaFile" name="file" type="file" disabled="disabled" style=" visibility: hidden;"/>
<p class="help-block" id="inputFilePreview" style="word-wrap:break-word;" ></p>
<% }else{%>
<input name="file" type="file" id="inputMediaFile">
<% }%>
</div>
Expand All @@ -49,7 +44,7 @@
<label class="radio-inline"><input class="languageRadio" type="radio" id="revOption" name="optradio" value="rev" >Rev </label><br>
<label class="radio-inline"><input class="languageRadio" type="radio" id="genelteOption" value="gentle" name="optradio">Gentle/Kaldi (offline/experimental)</label><br>
<label class="radio-inline"><input class="languageRadio" type="radio" id="pocketSphinxOption" name="optradio" value="pocketsphinx">Pocketsphinx (offline/experimental) </label><br>
<!-- <label class="radio-inline"><input class="languageRadio" type="radio" id="captionsOptionSrt" name="optradio" value="srt">Caption File <code>.srt</code> </label><br> -->
<label class="radio-inline"><input class="languageRadio" type="radio" id="captionsOption" name="optradio" value="captions">Caption File <code>.srt</code> </label><br>
<!-- <label class="radio-inline"><input class="languageRadio" type="radio" id="captionsOptionVtt" name="optradio" value="vtt">Caption File <code>.vtt</code> </label><br> -->

</div>
Expand All @@ -62,7 +57,7 @@
<li role="presentation"><a href="#rev" aria-controls="rev" role="tab" data-toggle="tab">Rev</a></li>
<li role="presentation"><a href="#gentle" aria-controls="gentle" role="tab" data-toggle="tab">Gentle</a></li>
<li role="presentation"><a href="#pocketsphinx" aria-controls="pocketsphinx" role="tab" data-toggle="tab">Pocketsphinx</a></li>
<!-- <li role="presentation"><a href="#srt" aria-controls="captions" role="tab" data-toggle="tab"><code>.srt</code></a></li> -->
<li role="presentation"><a href="#captions" aria-controls="captions" role="tab" data-toggle="tab"><code>.srt</code></a></li>
<!-- <li role="presentation"><a href="#vtt" aria-controls="captions" role="tab" data-toggle="tab"><code>.vtt</code></a></li> -->
</ul>

Expand Down Expand Up @@ -191,11 +186,23 @@
<i> Does not require extra setup, only supports US English for now, only works on Mac OSX</i>
</div>

<!-- <div role="tabpanel" class="tab-pane" id="srt">
<div role="tabpanel" class="tab-pane" id="captions">
<br>
<label for="languageModel">Captions <code>.srt</code> file:</label><br>
<i> coming soon <code>.srt</code></i>
</div> -->
<!-- srt -->
<div class="form-group">
<label for="fileUpload">Choose an audio or video file to transcribe</label>
<% if(window.process !== undefined ){%>
<button id="btnElectronInputCaptionFile" class="btn btn-default btn-xs" class="very-sweet-looking">Choose Caption File</button>
<input id="inputElectronInputCaptionFile" name="file" type="file" disabled="disabled" style=" visibility: hidden;"/>
<p class="help-block" id="inputCaptionFilePreview" style="word-wrap:break-word;" ></p>
<% }else{%>
<input name="file" type="file" id="inputCaptionFile">
<% }%>
</div>

</div>
<!--
<div role="tabpanel" class="tab-pane" id="vtt">
<br>
Expand Down
65 changes: 44 additions & 21 deletions lib/app/views/transcription_form_view.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ module.exports = Backbone.View.extend({
},
events :{
'click #btnElectronInputMediaFile':'electronGetFilePath',
'click #btnElectronInputCaptionFile':'electronGetCaptionFilePath',
'click #submitBtn': 'save',
'keypress .form-control': 'onEnterListener',
'change input[type=radio]': 'changedRadio'
Expand Down Expand Up @@ -80,6 +81,25 @@ module.exports = Backbone.View.extend({

},

electronGetCaptionFilePath: function(e){
e.preventDefault();

var self = this;
window.openCaptionsFileDiaglogue(function(fileName){
console.log(fileName);
self.captionFilePath = fileName[0];
console.log(self.captionFilePath );
document.getElementById("inputCaptionFilePreview").innerHTML = self.captionFilePath;
// console.log(document.getElementById("title").value);
// if(document.getElementById("title").value ===""){
// // self.newFilePath.split("/")
// document.getElementById("title").value = path.basename(self.newFilePath);
// }
});

},


save: function(e){
console.log( e.target);
//TODO: there might be a better way to get values from a form in backbone?
Expand All @@ -103,7 +123,7 @@ module.exports = Backbone.View.extend({
var newLanguage = '';

var newFilePath = this.newFilePath;

var captionFilePath = this.captionFilePath;

var radios = document.querySelectorAll('.languageRadio');
var sttEngine ;
Expand Down Expand Up @@ -131,8 +151,8 @@ module.exports = Backbone.View.extend({
newLanguage = document.querySelector('#languageModelSpeechmatics').value;
}else if(sttEngine === 'rev'){
newLanguage = 'rev';
}else if(sttEngine === 'srt'){
newLanguage = 'srt';
}else if(sttEngine === 'captions'){
newLanguage = 'captions';
}else if(sttEngine === 'gentle'){
newLanguage = 'gentle';
}else if(sttEngine === 'pocketsphinx'){
Expand All @@ -149,8 +169,8 @@ module.exports = Backbone.View.extend({
if(sttEngine === "ibm" || sttEngine === "speechmatics"){
if(navigator.onLine){
console.log("SPEECHMATICS-Transcription form ", newTitle, newDescription, newFilePath, newLanguage, sttEngine);

this.model.save({title: newTitle,
this.model.save({
title: newTitle,
description: newDescription,
videoUrl:newFilePath,
languageModel: newLanguage,
Expand All @@ -171,22 +191,25 @@ module.exports = Backbone.View.extend({
}

//captions can be used offline, parse captions with parserComposer module in transcriber option.
}else if(sttEngine === "srt"){
// console.log("SRT-Transcription form ", newTitle, newDescription, newFilePath, newLanguage, sttEngine);
// this.model.save({title: newTitle,
// description: newDescription,
// videoUrl:newFilePath,
// languageModel: newLanguage,
// sttEngine: sttEngine},
// {
// success: function(mode, response, option){
// Backbone.history.navigate("transcriptions", {trigger:true});
// },
// error: function(model, xhr,options){
// var errors = JSON.parse(xhr.responseText).errors;
// alert("ops, something when wrong with saving the transcription:" + errors);
// }
// });
}else if(sttEngine === "captions"){
console.log("SRT-Transcription form ", newTitle, newDescription, newFilePath, newLanguage, sttEngine);
this.model.save({
title: newTitle,
description: newDescription,
videoUrl:newFilePath,
captionFilePath: captionFilePath,
languageModel: newLanguage,
sttEngine: sttEngine
},
{
success: function(mode, response, option){
Backbone.history.navigate("transcriptions", {trigger:true});
},
error: function(model, xhr,options){
var errors = JSON.parse(xhr.responseText).errors;
alert("ops, something when wrong with saving the transcription:" + errors);
}
});


//pocketsphinx and Gentle handled as fallback cases.
Expand Down
2 changes: 2 additions & 0 deletions lib/interactive_transcription_generator/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,8 @@ var generate = function(config) {
languageModel: config.languageModel,
sttEngine: config.sttEngine,
revOrderNumber: config.revOrderNumber,
// for caption file conversion
captionFilePath: config. captionFilePath,

callback: function(error, respTranscriptJson){
console.info("---> Done transcribing: "+videoFile, respTranscriptJson);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
const fs = require('fs');
const srtJsonToWordLinesJson = require('./srtJsonToWordLinesJson.js');
const srtJsonWordLinesJsonToautoEditJson = require('./srtJsonWordLinesJsonToautoEditJson.js');

function convert(captionsFilePath){
const captionsString = openFile(captionsFilePath);
var srtLineJson = convertSrtToJson(captionsString);
var srtLineJsonSecondsTimecodes = convertTimecodesToSeconds(srtLineJson);
var result = srtJsonToWordLinesJson(srtLineJsonSecondsTimecodes);
result = srtJsonWordLinesJsonToautoEditJson(result);
return result;
}



function convertTimecodesToSeconds(srtLineJsonArray){
var result = srtLineJsonArray.map((lineObject)=>{
return {
startTime :timecodesToSeconds(lineObject.startTime),
endTime : timecodesToSeconds(lineObject.endTime),
id: lineObject.id,
text: lineObject.text
};
})
return result;
}

/**
* @todo: this function is repeated from Rev conversion. absract as own module
* @param {stirng} timecodeStringHHMMSSFFF, eg '00:33:19,470'
* as described in their documentation https://www.rev.com/api/attachmentsgetcontent
* timestamp format is hh:mm:sss,fff
* where fff represents milliseconds
* @returns {number} - timecode in seconds
*/
function timecodesToSeconds(timecodeStringHHMMSSFFF){
var tcArray = timecodeStringHHMMSSFFF.split(":");
var hoursInSeconds = parseInt(tcArray[0],10) * 60 * 60;
var minutesInSeconds = parseInt(tcArray[1],10) * 60;
// seconds and milliseconds are joined together for this calculation by replacing the comma with a full stop.
var secondsAndMilliseconds = parseFloat(tcArray[2].replace(/,/,'.'));
return hoursInSeconds+minutesInSeconds+secondsAndMilliseconds;
}


// convert srt
function convertSrtToJson(captionsString){
//split srt string into array. where each element it's a line of the srt file.
var srtArray = captionsString.split("\n");
//define regex for recognising components of the srt file. number. timecodes, words in a line, empty space between lines
//line counter regex
var oneDigit = /^[0-9]+$/;
//timecode regex
// "00:00:06,500 --> 00:00:10,790" there seems to be some cases where the milliseconds have 2 digits
var twoTimeCodes = /\d{2}:\d{2}:\d{2},\d{2,3} --> \d{2}:\d{2}:\d{2},\d{2,3}/
var words = /\w/;

//setup data structure to save results as as array of line objects.
var result = []
// initialise first line object outside of the loop ensure persistency for the different attributes across file lines.
var lineO = {};

srtArray.forEach((line)=>{
//issue a round charatcher windwos/mac \n \r messing up the code.
if(oneDigit.test(line.split("\r")[0])){
//line object is reset when we encounter a new srt line counter, rather then after the line text, ebcause number of line text can be variable.
lineO = {};
lineO.id = line;
}else if (twoTimeCodes.test(line)) {
var timecodes = line.split(" --> ")
lineO.startTime = timecodes[0]
lineO.endTime = timecodes[1]
//because number of lines in srt is variable, 1 or 2 generally, but we don't want to be opionionated about that just in case. and because We split on `\n` then needs to do a bunch of different passages to add all the text. so initialising the text as empty string after the timecode var is best way to add that attribute.
lineO.text ="";
}else if(words.test(line)){
lineO.text +=line+"\n";
}else{
//also save line when done
result.push(lineO);
}
})
return result;
}

function openFile(filePath){
return fs.readFileSync(filePath).toString('utf8');
}

module.exports = convert;

// var sampleSrt = '/Users/pietropassarelli/Desktop/Demo_media/BBC_interview/23 glas blower INT.srt'
// convert(sampleSrt)
Loading

0 comments on commit f41e968

Please sign in to comment.