diff --git a/README.md b/README.md index 8b461d5..5baa447 100644 --- a/README.md +++ b/README.md @@ -1 +1,131 @@ -# LilypondToBandVideoConverter \ No newline at end of file +# LilypondToBandVideoConverter + +## Introduction + +The LilypondToBandVideoConverter is a python script that orchestrates +existing command line tools to convert a music piece written in the +lilypond notation to + +- a *PDF score* of the whole piece, + +- several *PDF voice extracts*, + +- a *MIDI file with all voices* (with some preprocessing + applied for humanization), + +- *audio mix files* with several subsets of voices (specified + by configuration), and + +- *video files* for several output devices visualizing the + score notation pages and having the mixes as mutually + selectable audio tracks as backing tracks. + +For processing a piece one must have + +- a *lilypond fragment file* with the score information + containing specific lilypond identifiers, and + +- a *configuration file* giving details like the voices + occuring in the piece, their associated midi instrument, + target audio volume, list of mutable voices for the audio + tracks etc. + +Based on those files the python script -- together with some +open-source command-line software like ffmpeg or fluidsynth -- +produces all the target files either incrementally or altogether. + +The tool-chain has several processing phases that can be run as +required and produce the several outputs incrementally. The following +figure shows the phases and their results and how the phases depend on +each other. + +![LilypondToBandVideoConverter phases](lilytobvc-flow.png) + +The files (in yellow) are generated by the phases (in magenta), the +configuration file (in green) and the lilypond fragment file (in blue) +are the only manual inputs into the processing chain. + +Those phases are: + + - *extract:* generates PDF notation files for single voices as + extracts (might use compacted versions if specified), + + - *score:* generates a single PDF file containing all voices as a + score, + + - *midi:* generates a MIDI file containing all voices with specified + instruments, pan positions and volumes, + + - *silentvideo:* generates (intermediate) silent videos containing + the score pages for several output video file kinds (with + configurable resolution and size), + + - *rawaudio:* generates unprocessed (intermediate) audio files for + all the instrument voices from the midi tracks, + + - *refinedaudio:* generates (intermediate) audio files for all the + instrument voices with additional audio processing applied, + + - *mix:* generates final compressed audio files with submixes of all + instrument voices based on the refined audio files with a + specified volume balance and some subsequent mastering audio + processing (where the submix variants are configurable), and + + - *finalvideo:* generates a final video file with all submixes as + selectable audio tracks and with a measure indication as subtitle + +## Installation and Requirements + +The script and its components are written in python and can be +installed as a single python package. The package requires either +Python 2.7 or Python 3.3 or later. + +Additionally the following software has to be available: + +- *[lilypond][]*: for generating the score pdf, voice + extract pdfs, the raw midi file and the score images used + in the video files, + +- *[ffmpeg][]*: for video generation and video + postprocessing, + +- *[fluidsynth][]*: for generation of voice audio files from + a midi file plus some soundfont (e.g. [FluidR3_GM.sf3][]), + +- *[sox][]*: for instrument-specific postprocessing of audio + files for the target mix files as well as the mixdown, + and + +Optionally the following software is also used: + +- *[qaac][]*: the AAC-encoder for the final audio mix file + compression. + +- *[mp4box][]*: the MP4 container packaging software + +The location of all those commands as well as a few other +settings has to be defined in a global configuration file +for the LilypondToBandVideoConverter. + +Installation is done from the PyPi repository via + + pip install lilypondToBandVideoConverter + +Make sure that the scripts directory of python is in the path for +executables on your platform. + +## Further Information + +A longer description is available [here][notation-video] and the +detailed manual is available [here]. + +[ffmpeg]: http://ffmpeg.org/ +[FluidR3_GM.sf3]: https://github.com/musescore/MuseScore/raw/2.1/share/sound/FluidR3Mono_GM.sf3 +[fluidsynth]: http://www.fluidsynth.org/ +[here]: http://www.tensi.eu/thomas/iPod/lilypondToBandVideoConverter.pdf +[lilypond]: http://lilypond.org/ +[lilypondFileSyntax]: http://tensi.eu/thomas +[mp4box]: https://gpac.wp.imt.fr/mp4box/mp4box-documentation/ +[notation-video]: http://www.tensi.eu/thomas/iPod/notation-video.html +[qaac]: https://sites.google.com/site/qaacpage/ +[sox]: http://sox.sourceforge.net/ diff --git a/config/ltbvc-global.cfg b/config/ltbvc-global.cfg index 450a01f..40f8110 100644 --- a/config/ltbvc-global.cfg +++ b/config/ltbvc-global.cfg @@ -1,16 +1,30 @@ -- -*- mode: Conf; coding: utf-8-unix -*- -- global configuration file for ltbvc.py and submodules +-- note that the paths defined here must be adapted for your local +-- configuration -- ####################### -- # AUXILIARY VARIABLES # -- ####################### -_programDirectory = "C:/Programme_TT/Multimedia" +_programDirectory = "C:/Program Files/Multimedia" _audioProgramDirectory = _programDirectory "/Audio" _midiProgramDirectory = _programDirectory "/MIDI" _videoProgramDirectory = _programDirectory "/Video" -_targetVideoDirectory = "C:/VideoDateien" +_targetVideoDirectory = "C:/Video Files" +_tempDirectory = "C:/temp" + +-- path of directory for the soundfonts +_soundFontDirectoryPath = _midiProgramDirectory "/soundfonts" + +_soundFonts = _soundFontDirectoryPath "/FluidR3_GM.SF2" + +_soxTargetFormat="-b 32" + +-- sox command +_soxCommand = \ + _audioProgramDirectory "/sox/sox.exe --buffer 100000 --multi-threaded" -- ########################### -- # CONFIGURATION VARIABLES # @@ -21,15 +35,19 @@ _targetVideoDirectory = "C:/VideoDateien" -- ============== -- location of aac encoder command (optional, otherwise ffmpeg is used) -aacCommandLine = _audioProgramDirectory \ - "/QuicktimeAACEncoder/qaac64.exe" \ - " -V100 -i $1 -o $2" +aacCommandLine = \ + _audioProgramDirectory \ + "/QuicktimeAACEncoder/qaac64.exe" " -V100 ${infile} -o ${outfile}" -- location of ffmpeg command ffmpegCommand = _videoProgramDirectory "/ffmpeg/bin/ffmpeg.exe" --- location of fluidsynth command -fluidsynthCommand = _midiProgramDirectory "/QSynth/fluidsynth.exe" +-- complete midi to wav rendering command line +-- " -f " _tempDirectory "/fluidsynthsettings.txt" +midiToWavRenderingCommandLine = \ + _midiProgramDirectory "/QSynth/fluidsynth.exe -n -i -g 1" \ + " -R 0" \ + " -F ${outfile} " _soundFonts " ${infile}" -- location of lilypond command lilypondCommand = _midiProgramDirectory "/LilyPond/usr/bin/lilypond.exe" @@ -37,19 +55,26 @@ lilypondCommand = _midiProgramDirectory "/LilyPond/usr/bin/lilypond.exe" -- location of mp4box command mp4boxCommand = _videoProgramDirectory "/MP4Box/mp4box.exe" --- location of sox command -soxCommandLinePrefix = _audioProgramDirectory "/sox/sox.exe" \ - " --buffer 100000 --multi-threaded" +-- audio processing command lines +audioProcessor = "{" \ + "mixingCommandLine:" \ + "'" _soxCommand " -m [-v ${factor} ${infile} ] "\ + _soxTargetFormat " ${outfile}' ," \ + "amplificationEffect: 'gain ${amplificationLevel}'," \ + "paddingCommandLine:" \ + "'" _soxCommand " ${infile} " \ + _soxTargetFormat " ${outfile} pad ${duration}' ," \ + "refinementCommandLine:" \ + "'" _soxCommand " ${infile} " \ + _soxTargetFormat " ${outfile} ${effects}'" \ + "}" -- ====================== -- === FILE LOCATIONS === -- ====================== -- path of file containing the processing log -loggingFilePath = "/temp/logs/lilypondToBandVideoConverter.log" - --- path of directory for the soundfonts -soundFontDirectoryPath = _midiProgramDirectory "/soundfonts" +loggingFilePath = _tempDirectory "/logs/lilypondToBandVideoConverter.log" -- path of directory where all generated files go targetDirectoryPath = "generated" @@ -58,7 +83,7 @@ targetDirectoryPath = "generated" intermediateFileDirectoryPath = "temp" -- path of directory for temporary audio files -tempAudioDirectoryPath = "/temp/MIDI-Rendering-Demo/_current" +tempAudioDirectoryPath = _tempDirectory "/MIDI-Rendering-Demo/_current" -- path of temporary lilypond file tempLilypondFilePath = "temp/temp.ly" @@ -76,9 +101,10 @@ tempLilypondFilePath = "temp/temp.ly" -- defines video scaling factor for antialiasing and frame rate for -- the target video -_directoryA = "'" _targetVideoDirectory "/_ipod-Videos'" -_directoryB = "'" _targetVideoDirectory "/Notenvideos'" +_directoryA = "'" _targetVideoDirectory "/_ipad_videos'" +_directoryB = "'" _targetVideoDirectory "/_ipod_videos-temp'" _directoryC = "'" _targetVideoDirectory "/_internet-Videos'" +_directoryD = "'" _targetVideoDirectory "/_notation_videos'" -- subtitleColor = 0X8800FFFF _subtitleColor = "2281766911" @@ -98,14 +124,15 @@ videoTargetMap = "{" \ " subtitleFontSize: 20," \ " subtitlesAreHardcoded: true }," \ \ - "lumia: { resolution: 282," \ - " height: 770," \ - " width: 1280," \ - " topBottomMargin: 3," \ - " leftRightMargin: 5," \ + "ipod: { resolution: 163," \ + " height: 240," \ + " width: 320," \ + " topBottomMargin: 2," \ + " leftRightMargin: 2," \ " scalingFactor: 4," \ " frameRate: 10.0," \ - " systemSize: 15," \ + " systemSize: 10," \ + " mediaType: 'Movie'," \ " subtitleColor: " _subtitleColor "," \ " subtitleFontSize: 30," \ " subtitlesAreHardcoded: true }," \ @@ -120,6 +147,18 @@ videoTargetMap = "{" \ " systemSize: 25," \ " subtitleColor: " _subtitleColor "," \ " subtitleFontSize: 20," \ + " subtitlesAreHardcoded: true }," \ + \ + "lumia: { resolution: 282," \ + " height: 770," \ + " width: 1280," \ + " topBottomMargin: 3," \ + " leftRightMargin: 5," \ + " scalingFactor: 4," \ + " frameRate: 10.0," \ + " systemSize: 15," \ + " subtitleColor: " _subtitleColor "," \ + " subtitleFontSize: 30," \ " subtitlesAreHardcoded: true }" \ "}" @@ -131,14 +170,19 @@ videoFileKindMap = "{" \ " fileNameSuffix: '-i-v'," \ " voiceNameList: 'vocals' }," \ \ - "lumia: { target: lumia," \ + "ipod: { target: ipod," \ " directoryPath: " _directoryB "," \ - " fileNameSuffix: '-l-h'," \ + " fileNameSuffix: '-i-h'," \ " voiceNameList: 'vocals' }," \ \ "internet: { target: internet," \ " directoryPath: " _directoryC "," \ " fileNameSuffix: '-n-h'," \ + " voiceNameList: 'vocals' }," \ + \ + "lumia: { target: lumia," \ + " directoryPath: " _directoryD "," \ + " fileNameSuffix: '-l-h'," \ " voiceNameList: 'vocals' }" \ "}" @@ -146,8 +190,4 @@ videoFileKindMap = "{" \ -- === MISC === -- ============ --- comma-separated list of soundfont names (all located in soundfont --- directory) --- "FluidR3_GM.SF2" - -soundFontNames = "FluidR3_GM.SF2, Ultimate_Drums.sf2" +lilypondVersion = "2.19.82" diff --git a/config/notationSettings.cfg b/config/notationSettings.cfg new file mode 100644 index 0000000..1d954e7 --- /dev/null +++ b/config/notationSettings.cfg @@ -0,0 +1,81 @@ +-- -*- mode: Conf; coding: utf-8-unix -*- +-- this configuration file contains several mappings in the +-- context of the notation phases of the +-- lilypondToBandVideoConverter + +-- ######################### +-- # AUXILIARY DEFINITIONS # +-- ######################### + +-- special staff definitions for voices, default is "Staff" +_voiceNameToStaffListMapExtract = "{" + "drums : DrumStaff," + "guitarExtended : Staff/TabStaff," + "keyboard : Staff/Staff," + "percussion : DrumStaff" +"}" + +_voiceNameToStaffListMapMIDI = "{" + "drums : DrumStaff," + "guitarExtended : Staff," + "keyboard : Staff/Staff," + "percussion : DrumStaff" +"}" + +_voiceNameToStaffListMapScore = "{" + "drums : DrumStaff," + "guitarExtended : Staff," + "keyboard : Staff/Staff," + "percussion : DrumStaff" +"}" + +-- special clef definitions for voices, default is "G" +_voiceNameToClefMap = "{" + "bass : bass_8," + "drums : ''," + "guitar : G_8," + "guitarExtended : G_8," + "guitarExtendedTop : G_8," + "guitarTop : G_8," + "keyboardBottom : bass," + "percussion : ''" +"}" + +-- ######################## +-- # EXPORTED DEFINITIONS # +-- ######################## + +-- mapping from voice name to short name in score file +voiceNameToScoreNameMap = "{" + "bass : bs," + "bgVocals : bvc," + "drums : dr," + "guitar : gtr," + "guitarExtended : gtr," + "keyboard : kb," + "keyboardSimple : kb," + "organ : org," + "percussion : prc," + "piano : pia," + "strings : str," + "synthesizer : syn," + "vocals : voc" +"}" + +-- staff definitions for phases and voices: all phases use the +-- same definitions +phaseAndVoiceNameToStaffListMap = "{" + "extract :" _voiceNameToStaffListMapExtract "," + "midi :" _voiceNameToStaffListMapMIDI "," + "score :" _voiceNameToStaffListMapScore "," + "video :" _voiceNameToStaffListMapScore +"}" + +-- special clef definitions for phases and voices: all phases use the +-- same definitions +phaseAndVoiceNameToClefMap = "{" + "extract :" _voiceNameToClefMap "," + "midi :" _voiceNameToClefMap "," + "score :" _voiceNameToClefMap "," + "video :" _voiceNameToClefMap +"}" diff --git a/config/soundProcessorCommands.cfg b/config/soundProcessorCommands.cfg index 2c8e3c6..806a339 100644 --- a/config/soundProcessorCommands.cfg +++ b/config/soundProcessorCommands.cfg @@ -21,261 +21,399 @@ -- common effects -- ============== -_overdriveExtreme = \ - " norm -10" \ +_limiter = + " compand 0.001,0.01 6:-6,0,-5.994 +0" + +-- + +_overdriveExtreme = + " gain -10" " overdrive 30 0" -- -_overdriveHard = \ - " norm -6" \ +_overdriveHard = + " gain -6" " overdrive 12 0" -- -_overdriveSoft = \ - " norm -4" \ +_overdriveSoft = + " gain -4" " overdrive 5 0" -- -_overdriveUltraHard = \ - " norm -15" \ +_overdriveUltraHard = + " gain -15" " overdrive 25 0" +-- + +_leslieSlow = + " chorus 0.7 0.9 55 0.4 0.25 2 −t" + " tremolo 1 50" + +-- + +_reverbLight = " reverb " + +-- + +-- pitch shift up by exactly one octave +_addOctaveUp = + " pitch +1200 ->A;" + " mix 0 -> -3 A-> ->B;" + " B->" + +-- ---------- + +-- imperfect pitch shift by 1195 cents (instead of 1200) +_addOctaveUpImperfect = + " pitch +1195 ->A;" + " mix 0 -> -3 A-> ->B;" + " B->" + +-- ---------- + +-- pitch shift by exactly one octave +_addOctaveDown = + " pitch -1200 ->A;" + " mix 0 -> -3 A-> ->B;" + " B->" + +-- ---------- + +-- pitch shift by exactly one octave +_addOctaveDownImperfect = + " pitch -1195 ->A;" + " mix 0 -> -3 A-> ->B;" + " B->" + +-- === loudspeaker emulation === +_impulsResponse = + _impulseResponseDirectory "/loudspeaker_impulse_response.txt" +-- half the number of samples in IR minus 1 +_sampleCount = "1102s" + +_loudspeakerEmulation = + " gain -12" + " pad " _sampleCount + " fir " _impulsResponse + -- ======== -- = BASS = -- ======== -_bassPostshape = \ - " highpass -2 60 norm -6" \ - " equalizer 300 1q +10 norm -12" \ - " lowpass -2 1500 norm -6" +_bassPostshape = + " highpass -2 41 gain -6" + " equalizer 200 2q +8 gain -8" + " equalizer 300 1q +10 gain -12" + " equalizer 1500 1.5q +7 gain -12" + " lowpass -2 4000 gain -6" -- --- compand 0.015,0.15 6:-16,-16,-4 0 -70 0.2 -_bassPreprocess = \ - " norm -12" \ - " highpass -2 40" \ - " lowpass -2 2k" \ - " tee" + +_bassPreprocess = + " gain +6" + " highpass -2 40" + " lowpass -2 2k" -- -_bassPostprocess = \ - " tee" \ - " norm -24" \ - " equalizer 150 4o +10" \ - " lowpass -2 600 1.2o" +_bassPostprocess = + " highpass -2 40 2o" + " equalizer 164 1.4o +10" + " equalizer 1.6k 2o +4" + " lowpass -2 2k 1o" --- --- +-- ---------- +-- ---------- -soundStyleBassAcoustic = \ - " highpass -2 40 norm -6" \ - " lowpass -2 2k norm -6" \ - " compand 0.15,0.5 6:-30,-30,-6 -8 -30 0.2" +soundStyleBassAcoustic = + " gain -6" + " highpass -2 41" + " lowpass -2 2k" + " compand 0.15,0.5 6:-30,0,-24 +12" -- -soundStyleBassExtreme = \ - _bassPreprocess \ - _overdriveExtreme \ +soundStyleBassExtreme = + _bassPreprocess + _overdriveExtreme + " gain -10" _bassPostprocess -- -soundStyleBassHard = \ - _bassPreprocess \ - _overdriveHard \ +soundStyleBassHard = + _bassPreprocess + _overdriveHard _bassPostprocess -- -soundStyleBassSoft = \ - _bassPreprocess \ - _overdriveSoft \ - " tee" \ - " bass -40 80 1.2q norm -12" \ - " equalizer 200 2q +11 norm -12" \ +soundStyleBassDeep = + "gain +6" + _addOctaveDown + soundStyleBassHard + +-- + +soundStyleBassSoft = + _bassPreprocess + _overdriveSoft + " bass -40 41 1.2q gain -12" + " equalizer 200 2q +11 gain -12" " treble -40 500 1.2q" -- -soundStyleBassStd = \ - _bassPreprocess \ - " gain -l 5" \ +soundStyleBassStd = + _bassPreprocess + " compand 0.001,0.01 6:-6,0,-5.994 +5" _bassPostprocess -- ============ -- = BGVOCALS = -- ============ -soundStyleBgvocalsStd = \ - " chorus 0.5 0.9" \ - " 50 0.4 0.25 2 -t" \ - " 60 0.32 0.4 2.3 -t" \ +soundStyleBgvocalsStd = + " gain +6" + " chorus 0.5 0.9" + " 50 0.4 0.25 2 -t" + " 60 0.32 0.4 2.3 -t" " 40 0.3 0.3 1.3 -s" -- ========= -- = DRUMS = -- ========= -_drumsCompressExtreme = \ - " mcompand" \ - " '0.03,0.15 6:-20,-20,-2 12' 300" \ - " '0.04,0.15 6:-12,-12,-3 6' 1400" \ - " '0.04,0.15 6:-25,-25,-3 4' 4000" \ - " '0.10,0.50 6:-28,-28,-7 9' " +_drumsCompressExtreme = + " mcompand" + " '0.03,0.15 6:-20,0,-7.5 12' 150" + " '0.04,0.15 6:-12,0,-9 6' 1500" + " '0.04,0.15 6:-25,0,-22.5 4' 4000" + " '0.10,0.50 6:-28,0,-21 9' " -- -_drumsCompressHard = \ - " mcompand" \ - " '0.03,0.15 6:-20,-20,-3 12' 300" \ - " '0.04,0.15 6:-12,-12,-3 5' 1400" \ - " '0.04,0.15 6:-25,-25,-3 4' 4000" \ - " '0.10,0.50 6:-28,-28,-7 9' " +_drumsCompressHard = + " mcompand" + " '0.03,0.15 6:-18,0,-15 4' 300" + " '0.03,0.15 6:-18,0,-15 -7' 1500" + " '0.04,0.15 6:-18,0,-16.8 10' 2500" + " '0.30,0.50 6:-60,0,0 0' " -- -_drumsPostprocess = \ - " equalizer 60 1.4o +8" \ - " equalizer 300 1.4o +9" \ - " equalizer 4000 1.4o +3" +_drumsCompressUltra = + " mcompand" + " '0.03,0.15 6:-30,0,-26 12' 150" + " '0.04,0.15 6:-10,0,-9 6' 1000" + " '0.04,0.15 6:-25,0,-22.5 -3' 4000" + " '0.10,0.50 6:-28,0,-21 0' " -- + +_drumsPostprocess = + " equalizer 300 1.4o +9" +-- " treble -10 4500 1o" + -- -soundStyleDrumsExtreme = \ - " gain -20" \ - _drumsCompressExtreme \ +_drumsPostprocessUltra = + " equalizer 60 1.4o +8" + " equalizer 500 1.4o +9" + " equalizer 5000 1.4o +3" + +-- ---------- +-- ---------- + +soundStyleDrumsExtreme = + " gain -14" + _drumsCompressExtreme _drumsPostprocess -- -soundStyleDrumsStd = \ - " norm -20" \ +soundStyleDrumsStd = + " gain +6" _drumsPostprocess -- -soundStyleDrumsHard = \ - " norm -15" \ - _drumsCompressHard \ +soundStyleDrumsHard = + " gain +6" + _drumsCompressHard + " gain -4" _drumsPostprocess +-- + +soundStyleDrumsUltra = + " gain +6" + _drumsCompressUltra + _drumsPostprocessUltra + -- ========== -- = GUITAR = -- ========== -_guitarPreprocessingCompressor = \ - " compand 0.04,0.5 6:-25,-20,-5 -10 -90 0.02" +_guitarPreprocessingCompressor = + " compand 0.04,0.5 6:-20,0,-15 +8" --- +-- ---------- +-- ---------- -soundStyleGuitarCrunch = \ - " highpass -1 100 norm -6" \ - _guitarPreprocessingCompressor \ - " tee" \ +soundStyleGuitarCrunch = + " highpass -2 82" + _guitarPreprocessingCompressor " overdrive 10 40" -- -soundStyleGuitarDrive = \ - " norm -6" \ - " highpass -2 80 2o" \ - _guitarPreprocessingCompressor \ - " norm -8" \ - " highpass -2 100 2o" \ - " equalizer 500 2o +6" \ - " lowpass -2 3000 2o" \ - " tee" \ - " norm -5" \ - " overdrive 25 0 " \ - " norm -8" \ - " equalizer 200 2o +10" \ - " lowpass -2 1100 2o" +soundStyleGuitarCrunchhard = + " gain +6" + " compand 0.04,0.5 6:-28,0,-21 +10" + " overdrive 15 0" + " gain -6" + " lowpass -2 4k 2o" + " highpass -2 82 1.5o" + " equalizer 900 1.5o +4" -- -soundStyleGuitarHard = \ - " highpass -1 80" \ - " compand 0.3,0.5 6:-85,-60,-120 -15 -90 0.02" \ - " norm -3" \ - " tee" \ +soundStyleGuitarDrive = + " highpass -2 82 2o" + _guitarPreprocessingCompressor + " gain -8" + " highpass -2 82 2o" + " equalizer 500 2o +6" + " lowpass -2 3k 2o" + " gain +5" + " overdrive 25 0 " + " gain -8" + " equalizer 200 2o +10" + " lowpass -2 1.1k 2o" + +-- + +soundStyleGuitarHard = + " highpass -2 82" + " gain +6" + " compand 0.3,0.5 6:-60,0,-48 +20 -90 0.02" + " gain -3" _overdriveHard - -- " compand .1,.2 -inf,-50.1,-inf,-50,-50 0 -90 .1" - -- " compand .3,.5 6:-70,-60,-15 -15 -80 .02" +-- " compand .1,.2 -inf,-50.1,-inf,-50,-50 0 -90 .1" + -- " compand .3,.5 6:-60,0,-45 20 -80 .02" -- " equalizer 300 1q +4" -- -soundStyleGuitarStd = \ - " highpass -1 100 norm -6" \ +soundStyleGuitarStd = + " highpass -1 100" _guitarPreprocessingCompressor -- ============ -- = KEYBOARD = -- ============ -_keyboardHardCompress = \ - " norm -6" \ - " highpass -1 100" \ - " compand 0.05,0.3 6:-18,-18,-3 -8 -90 0.02" +_keyboardHardCompress = + " highpass -1 100" + " compand 0.05,0.3 6:-18,0,-15 8" -- -_keyboardPrecompress = \ - " compand 0.02,0.04 6:-70,-60,-40 -15" +_keyboardPrecompress = + " gain +6" + " compand 0.02,0.04 6:-60,0,-20 +15" -- -_keyboardSoftCompress = \ - " norm -6" \ - " highpass -1 500" \ - " compand 0.005,0.1 6:-20,-18,-3 -8 -90 0.02" +_keyboardSoftCompress = + " highpass -1 500" + " compand 0.005,0.1 6:-18,0,-9 +5" --- --- +-- ---------- +-- ---------- -soundStyleKeyboardCrunch = \ - _keyboardHardCompress \ +soundStyleKeyboardCrunch = + _keyboardHardCompress _overdriveHard -- -soundStyleKeyboardCompressed = \ +soundStyleKeyboardCompressed = _keyboardSoftCompress -- -soundStyleKeyboardStd = \ - _keyboardPrecompress \ +soundStyleKeyboardLeslie = + soundStyleKeyboardCrunch + _leslieSlow + +-- + +soundStyleKeyboardPhase = + _keyboardPrecompress + " phaser 0.6 0.66 3 0.6 0.5 -t" + " highpass -1 500" + +-- + +soundStyleKeyboardSoftcrunch = + _keyboardPrecompress + _overdriveSoft " highpass -1 500" -- -soundStyleKeyboardPhase = \ - _keyboardPrecompress \ - " phaser 0.6 0.66 3 0.6 0.5 -t" \ +soundStyleKeyboardStd = + _keyboardPrecompress " highpass -1 500" +-- ============== +-- = PERCUSSION = +-- ============== + +soundStylePercussionStd = + _drumsPostprocess + -- =========== -- = STRINGS = -- =========== -soundStyleStringsStd = \ - " compand 0.002,0.04 6:-70,-60,-20 -20" \ +_stringsCompress = + " gain +6" + " compand 0.002,0.04 6:-60,0,-40 20" + +-- ---------- + +soundStyleStringsStd = + _stringsCompress + " highpass -2 500" + +-- + +soundStyleStringsPhase = + _stringsCompress + " phaser 0.6 0.66 3 0.6 0.5 -t" " highpass -2 500" -- =============== -- = SYNTHESIZER = -- =============== -soundStyleSynthesizerPhase = \ - _keyboardPrecompress \ - " phaser 0.6 0.66 1 0.6 0.1 -t" \ - " phaser 0.6 0.66 1 0.6 0.1 -t" \ +soundStyleSynthesizerPhase = + _keyboardPrecompress + " phaser 0.6 0.66 1 0.6 0.1 -t" + " phaser 0.6 0.66 1 0.6 0.1 -t" " highpass -1 500" + +-- ---------- + +soundStyleSynthesizerStd = + _keyboardPrecompress + " highpass -1 800" diff --git a/config/styleHumanization.cfg b/config/styleHumanization.cfg new file mode 100644 index 0000000..a650a44 --- /dev/null +++ b/config/styleHumanization.cfg @@ -0,0 +1,73 @@ +-- -*- mode: Conf; coding: utf-8-unix -*- +-- style humanization strategies for MidiTransformer script; +-- +-- The strategy describes how how the timing and the velocity may be +-- changed depending on the position of a note within a measure. +-- +-- The velocity factor is a factor that is applied to the existing +-- velocity and then the slack is the possible variation. E.g. a +-- velocity factor of 0.8 with a slack of 10 means that the original +-- velocity is reduced by 20% and then a random offset in the interval +-- [-10, 10] is applied. The random distribution is quadratic around the +-- center. +-- +-- The timing is a variation as a factor of thirtysecond beats. +-- E.g. a value of 0.25 means that the timing may be off by at most a +-- 128th before or after the beat (25% of a thirtysecond note). When +-- the percentage has an "A" prefix (for "ahead"), the offset is always +-- negative, for a "B" prefix (for "behind") it is always positive. +-- +-- Additionally there is a global setting for instrument dependent +-- scaling factor on the velocity and timing variation. A drum is +-- very tight and has a factor of 1.0 each which means that the +-- calculated variation is taken directly; a bass is slightly more +-- loose and has a factor of 1.1 for velocity and 1.05 for timing +-- which means that the calculated variations are scaled accordingly. +-- +-- by Dr. Thomas Tensi, 2017 + +countInMeasureCount = 2 + +-- ============================================================ + +-- global setting for instrument dependent scaling factor on the +-- velocity and timing variation +voiceNameToVariationFactorMap = "{ bass : 1.1/1.05," + " drums : 1/1," + " guitar : 1.1/1.2," + " keyboard : 1.5/1.2," + " percussion : 1/1," + " strings : 1.5/1.2," + " vocals : 1.25/1.25 }" + +-- ============================================================ + +humanizationStyleBeatStd = + "{ 0.00: 1.05/0, 0.25: 0.95/0.3, 0.50: 1.25/0, 0.75: 0.95/0.3," + " OTHER: 0.85/B0.2," + " RASTER: 0.03125, SLACK:0.1 }" + +humanizationStyleBeatTight = + "{ 0.00: 1.25/0, 0.25: 1/0.1, 0.50: 1.15/0.1, 0.75: 1.05/0.1," + " OTHER: 0.95/B0.2," + " RASTER: 0.03125, SLACK:0.1 }" + +humanizationStyleDefault = + "{ 0.00: 1/0, 0.25: 1/0, 0.50: 1/0, 0.75: 1/0," + " OTHER: 1/0," + " RASTER: 0.03125, SLACK:0 }" + +humanizationStyleReggae = + "{ 0.00: 1/0.2, 0.25: 0.95/0.2, 0.50: 1.1/0, 0.75: 0.95/0.2," + " OTHER: 0.85/B0.25," + " RASTER: 0.03125, SLACK:0.1 }" + +humanizationStyleRockHard = + "{ 0.00: 0.95/0.2, 0.25: 1.1/0, 0.50: 0.9/0.2, 0.75: 1.05/0," + " OTHER: 0.85/B0.25," + " RASTER : 0.03125, SLACK : 0.1 }" + +humanizationStyleRockHardHalfTime = + "{ 0.00: 0.95/0.2, 0.5: 1.15/0," + " OTHER: 0.8/B0.25," + " RASTER : 0.03125, SLACK : 0.1 }" diff --git a/demo/globalconfig-post.txt b/demo/globalconfig-post.txt index 1260c1c..d01e130 100644 --- a/demo/globalconfig-post.txt +++ b/demo/globalconfig-post.txt @@ -1,16 +1,16 @@ -audioTrackList = "{" \ - "all : { audioGroupList : base/voc/gtr," \ - " audioFileTemplate : '$'," \ - " songNameTemplate : '$ [ALL]'," \ - " albumName : '$'," \ - " description : 'all voices'," \ - " languageCode : deu," \ - " voiceNameToAudioLevelMap : "_voiceNameToAudioLevelMap"}," \ - "novocals : { audioGroupList : base/gtr," \ - " audioFileTemplate : '$-v'," \ - " songNameTemplate : '$ [-V]'," \ - " albumName : '$ [-V]'," \ - " description : 'no vocals'," \ - " languageCode : eng," \ - " voiceNameToAudioLevelMap : "_voiceNameToAudioLevelMap"}" \ +audioTrackList = "{" + "all : { audioGroupList : base/voc/gtr," + " audioFileTemplate : '$'," + " songNameTemplate : '$ [ALL]'," + " albumName : '$'," + " description : 'all voices'," + " languageCode : deu," + " voiceNameToMixSettingsMap : "_voiceNameToMixSettingsMap"}," + "novocals : { audioGroupList : base/gtr," + " audioFileTemplate : '$-v'," + " songNameTemplate : '$ [-V]'," + " albumName : '$ [-V]'," + " description : 'no vocals'," + " languageCode : eng," + " voiceNameToMixSettingsMap : "_voiceNameToMixSettingsMap"}" "}" diff --git a/demo/globalconfig-pre.txt b/demo/globalconfig-pre.txt index 264c728..dc19a11 100644 --- a/demo/globalconfig-pre.txt +++ b/demo/globalconfig-pre.txt @@ -5,8 +5,8 @@ _programDirectory = "/usr/local" _soundFonts = _programDirectory "/soundfonts/FluidR3_GM.SF2" aacCommandLine = _programDirectory "/qaac -V100 -i ${infile} -o ${outfile}" -midiToWavRenderingCommandLine = \ - _programDirectory "/fluidsynth.exe -n -i -g 1 -R 0" \ +midiToWavRenderingCommandLine = + _programDirectory "/fluidsynth.exe -n -i -g 1 -R 0" " -F ${outfile} " _soundFonts " ${infile}" --==================== @@ -25,48 +25,48 @@ targetFileNamePrefix = "test-" --==================== _voiceNameToStaffListMap = "{ drums : DrumStaff }" -_voiceNameToClefMap = "{" \ - "bass : bass_8, drums : '', guitar : G_8" \ +_voiceNameToClefMap = "{" + "bass : bass_8, drums : '', guitar : G_8" "}" -phaseAndVoiceNameToStaffListMap = "{" \ - "extract :" _voiceNameToStaffListMap "," \ - "midi :" _voiceNameToStaffListMap "," \ - "score :" _voiceNameToStaffListMap "," \ +phaseAndVoiceNameToStaffListMap = "{" + "extract :" _voiceNameToStaffListMap "," + "midi :" _voiceNameToStaffListMap "," + "score :" _voiceNameToStaffListMap "," "video :" _voiceNameToStaffListMap "}" -phaseAndVoiceNameToClefMap = "{" \ - "extract :" _voiceNameToClefMap "," \ - "midi :" _voiceNameToClefMap "," \ - "score :" _voiceNameToClefMap "," \ +phaseAndVoiceNameToClefMap = "{" + "extract :" _voiceNameToClefMap "," + "midi :" _voiceNameToClefMap "," + "score :" _voiceNameToClefMap "," "video :" _voiceNameToClefMap "}" -voiceNameToChordsMap = "{" \ - "vocals : s/v, bass : e, guitar : e" \ +voiceNameToChordsMap = "{" + "vocals : s/v, bass : e, guitar : e" "}" --==================== -- VIDEO OUTPUT --==================== -videoTargetMap = "{" \ - "tablet: { resolution: 132," \ - " height: 1024," \ - " width: 768," \ - " topBottomMargin: 5," \ - " leftRightMargin: 10," \ - " scalingFactor: 4," \ - " frameRate: 10.0," \ - " mediaType: 'Music Video'," \ - " systemSize: 25," \ - " subtitleColor: 2281766911," \ - " subtitleFontSize: 20," \ +videoTargetMap = "{" + "tablet: { resolution: 132," + " height: 1024," + " width: 768," + " topBottomMargin: 5," + " leftRightMargin: 10," + " scalingFactor: 4," + " frameRate: 10.0," + " mediaType: 'Music Video'," + " systemSize: 25," + " subtitleColor: 2281766911," + " subtitleFontSize: 20," " subtitlesAreHardcoded: true } }" -videoFileKindMap = "{" \ - "tabletVocGtr: { target: tablet," \ - " fileNameSuffix: '-tblt-vg'," \ - " directoryPath: './mediaFiles' ," \ +videoFileKindMap = "{" + "tabletVocGtr: { target: tablet," + " fileNameSuffix: '-tblt-vg'," + " directoryPath: './mediaFiles' ," " voiceNameList: 'vocals, guitar' } }" --==================== @@ -75,8 +75,8 @@ videoFileKindMap = "{" \ albumArtFilePath = "demo.jpg" -audioGroupToVoicesMap = "{" \ - " base : bass/keyboard/strings/drums/percussion," \ - " voc : vocals/bgVocals," \ - " gtr : guitar" \ +audioGroupToVoicesMap = "{" + " base : bass/keyboard/strings/drums/percussion," + " voc : vocals/bgVocals," + " gtr : guitar" "}" diff --git a/demo/wonderful_song-config.txt b/demo/wonderful_song-config.txt index 015ab1d..5b1a06b 100644 --- a/demo/wonderful_song-config.txt +++ b/demo/wonderful_song-config.txt @@ -18,7 +18,7 @@ panPositionList = " C, 0.5L, 0.6R, 0.1L" reverbLevelList = " 0.2, 0.0, 0.0, 0.0" soundVariantList = "SIMPLE, CRUNCH, CRUNCH, GRIT" -_voiceNameToAudioLevelMap = \ +_voiceNameToMixSettingMap = "{ vocals : -4, bass : 0, guitar : -6, drums : -2 }" -- preprocessing @@ -27,14 +27,14 @@ voiceNameToLyricsMap = "{ vocals : e2/s2/v }" -- humanization countInMeasureCount = 2 -humanizationStyleRockHard = \ - "{ 0.00: 0.95/A0.1, 0.25: 1.15/0," \ - " 0.50: 0.98/A0.2, 0.75: 1.1/0," \ - " OTHER: 0.85/0.2," \ +humanizationStyleRockHard = + "{ 0.00: 0.95/A0.1, 0.25: 1.15/0," + " 0.50: 0.98/A0.2, 0.75: 1.1/0," + " OTHER: 0.85/0.2," " SLACK:0.1, RASTER: 0.03125 }" humanizedVoiceNameSet = "bass, guitar, drums" -measureToHumanizationStyleNameMap = \ +measureToHumanizationStyleNameMap = "{ 1 : humanizationStyleRockHard }" -- tempo @@ -44,17 +44,17 @@ measureToTempoMap = "{ 1 : 90 }" -- audio postprocessing -- .................... -soundStyleBassCrunch = \ - " compand 0.03,0.1 6:-20,0,-15" \ - " highpass -2 60 1o lowpass -2 800 1o equalizer 120 1o +3" \ +soundStyleBassCrunch = + " compand 0.03,0.1 6:-20,0,-15" + " highpass -2 60 1o lowpass -2 800 1o equalizer 120 1o +3" " reverb 60 100 20 100 10" soundStyleDrumsGrit = "overdrive 4 0 reverb 25 50 60 100 40" -soundStyleGuitarCrunch = \ - " compand 0.01,0.1 6:-10,0,-7.5 -6" \ - " overdrive 30 0 gain -10" \ - " highpass -2 300 0.5o lowpass -1 1200" \ +soundStyleGuitarCrunch = + " compand 0.01,0.1 6:-10,0,-7.5 -6" + " overdrive 30 0 gain -10" + " highpass -2 300 0.5o lowpass -1 1200" " reverb 40 50 50 100 30" soundStyleVocalsSimple = " overdrive 5 20" diff --git a/doc/Makefile b/doc/Makefile index 0ad68e0..4ddbac5 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -62,6 +62,9 @@ figures/humanizationGerman.mps: ltbvcFigure-6.mps figures/humanization.mps: ltbvcFigure-7.mps cp $< $@ +figures/panGraph.mps: ltbvcFigure-8.mps + cp $< $@ + #-- $(TARGETPDF): $(LATEXSOURCE) $(TARGETFIGURELIST) diff --git a/doc/figures/d.png b/doc/figures/d.png new file mode 100644 index 0000000..e0e7a40 Binary files /dev/null and b/doc/figures/d.png differ diff --git a/doc/figures/demo-extract.png b/doc/figures/demo-extract.png new file mode 100644 index 0000000..faeb806 Binary files /dev/null and b/doc/figures/demo-extract.png differ diff --git a/doc/figures/demo-score.png b/doc/figures/demo-score.png new file mode 100644 index 0000000..7113a13 Binary files /dev/null and b/doc/figures/demo-score.png differ diff --git a/doc/figures/demo-video.png b/doc/figures/demo-video.png new file mode 100644 index 0000000..efd85ed Binary files /dev/null and b/doc/figures/demo-video.png differ diff --git a/doc/figures/panGraph.png b/doc/figures/panGraph.png new file mode 100644 index 0000000..3aad0db Binary files /dev/null and b/doc/figures/panGraph.png differ diff --git a/doc/lilypondToBandVideoConverter.ltx b/doc/lilypondToBandVideoConverter.ltx index 4cfb5bd..451eccb 100644 --- a/doc/lilypondToBandVideoConverter.ltx +++ b/doc/lilypondToBandVideoConverter.ltx @@ -15,6 +15,7 @@ \usepackage{microtype} % for better justification, requires later % engine of LaTeX like e.g. pdflatex + \usepackage{lmodern} % scalable font %==================== % LOCAL DEFINITIONS @@ -143,7 +144,7 @@ \newcommand{\hyperlink}[1]{\textsf{\small \color{blue}#1}} \newcommand{\ltbvc}{Lilypond\-To\-Band\-Video\-Converter} \newcommand{\ltbvcCommand}{lilypondToBVC} - \newcommand{\ltbvcVersion}{1.1} + \newcommand{\ltbvcVersion}{1.1.1} \newcommand{\macro}[1]{\textbackslash #1} \newcommand{\mandatory}{\textbf{MANDATORY}} \newcommand{\meta}[1]{\guillemotleft #1\guillemotright} @@ -328,11 +329,13 @@ \newcommand{\DESCaudioProcessorMixingCommandLine}{% audio processor command line for mixing audio files with volume - factors containing \placeholder{factor}, \placeholder{infile} and - \placeholder{outfile} as placeholders; the group of factor and - infile is embraced by parentheses ("[]") and will be repeated - depending on the number of infiles with the parentheses removed; - if missing, mixing will be done by (slow) internal routines + factors containing \placeholder{factor}, \placeholder{pan}, + \placeholder{infile} and \placeholder{outfile} as placeholders; the + group of factor and infile is embraced by parentheses ("[]") and + will be repeated depending on the number of infiles with the + parentheses removed; if missing, mixing will be done by (slow) + internal routines, if ``pan'' is not specified as a placeholder, an + internal panning via ffmpeg is done } \newcommand{\DESCaudioProcessorAmplificationEffect}{% @@ -407,12 +410,15 @@ dollar-sign) } -\newcommand{\DESCaudioTrackVoiceNameToAudioLevelMap}{% - mapping from voice names to volume factors used for mixing the - refined audio files into cumulated audio file for given track; the - factors are decimal values in decibels (where 0.0 means that the - refined voice file is taken without change with a conversion of - \(10^{{\rm dBValue}/20}\)) +\newcommand{\DESCaudioTrackVoiceNameToMixSettingMap}{% + mapping from voice names to volume factors and pan positions used + for mixing the refined audio files into cumulated audio file for + given track with both elements separated by a slash; the factors + are decimal values in decibels (where 0.0 means that the refined + voice file is taken without change with a conversion of \(10^{{\rm + dBValue}/20}\)), the pan position is given as a decimal value + between 0 and 1 with suffix ``R'' or ``L'' (for right/left) or the + character ``C'' (for center) } \newcommand{\DESCaudioTrackList}{% @@ -509,12 +515,6 @@ 127) followed by a colon } -\newcommand{\DESCmidiPanList}{% - list of pan positions per voice as a decimal value between 0 and 1 - with suffix ``R'' or ``L'' (for right/left) or the character ``C'' - (for center) -} - \newcommand{\DESCmidiToWavRenderingCommandLine}{% command line for rendering command from MIDI file to WAV audio file (typically ``fluidsynth'' with parameters for input @@ -534,6 +534,12 @@ instead } +\newcommand{\DESCpanPositionList}{% + list of pan positions per voice as a decimal value between 0 and 1 + with suffix ``R'' or ``L'' (for right/left) or the character ``C'' + (for center) +} + \newcommand{\DESCparallelTrack}{% specification of an audio file name, a volume factor (in decibels) and an offset (in seconds) relative to the start of the song for an @@ -667,9 +673,9 @@ } \newcommand{\DESCvideoTargetScalingFactor}{% - the factor by which width and height are multiplied for lilypond - image rendering to be downscaled accordingly by the video renderer - (an integer); this is used for antialiasing + the factor (an integer) by which width and height are multiplied for + lilypond image rendering to be later downscaled for a better edge + smoothing via antialiasing } \newcommand{\DESCvideoTargetSubtitleColor}{% @@ -1156,12 +1162,8 @@ Each configuration file has a simple line-oriented syntax as follows: \item Leading and trailing whitespace in a line is ignored. Other whitespace is only interpreted as token separator. - \item A line starting with a comment marker ``-\relax-'' or completely - empty is ignored. - - \item A line ending with a continuation marker - ``\textvisiblespace\textbackslash'' is combined with the - following line with the continuation marker discarded. + \item A line starting with a comment marker ``{\tt -\,-}'' + (double-dash) is ignored. \item Each relevant line starts with an identifier followed by an equal sign and the associated value. The associated value may @@ -1171,14 +1173,22 @@ Each configuration file has a simple line-oriented syntax as follows: variable will replace that value. \item An identifier is a sequence of lower- and uppercase letters or - underscores. - - \item One may define such variables arbitrarily. - - \item An integer literal is a digit sequence, a decimal value is a digit - sequence with at most one decimal point, a boolean value is either - the string ``true'' or ``false'' and a string value is a - character sequence enclosed by double quotes. Two double + underscores and signifies a variable. One may define such + variables arbitrarily. + + \item Several physical lines are collected into a single logical + value assignment line until either an empty line (with only + whitespace) or a new assigment line is encountered. + + \item A line may end with a continuation marker + ``\textbackslash''. That marker is discarded + and the line is combined into the previous logical assignment + line (if any). + + \item An integer literal is a digit sequence, a decimal value is a + digit sequence with at most one decimal point, a boolean value + is either the string ``true'' or ``false'' and a string value is + a character sequence enclosed by double quotes. Two double quotes within a string are interpreted as a double quote character. @@ -1657,7 +1667,7 @@ configuration variables could look like that: ffmpegCommand = "/usr/local/ffmpeg" lilypondCommand = "/usr/local/lilypond" lilypondVersion = "2.18.2" - midiToWavRenderingCommandLine = \ + midiToWavRenderingCommandLine = "/usr/local/fluidsynth ${infile} ${outfile}" \end{configurationFileCode} @@ -1672,15 +1682,15 @@ figure~\ref{figure:configFileAudioProcessorVariables}. \begin{configurationFileCode} _sox = "/usr/local/sox --buffer 100000 --multi-threaded" - audioProcessor = \ - "{ redirector: '->'," \ - " chainSeparator: ';'," \ - " amplificationEffect: 'gain ${amplificationLevel}'," \ - " mixingCommandLine: '" _sox \ - " -m [-v ${factor} ${infile} ] ${outfile}'," \ - " paddingCommandLine: '" _sox \ - " ${infile} ${outfile} pad ${duration}'," \ - " refinementCommandLine: '" _sox \ + audioProcessor = + "{ redirector: '->'," + " chainSeparator: ';'," + " amplificationEffect: 'gain ${amplificationLevel}'," + " mixingCommandLine: '" _sox + " -m [-v ${factor} ${infile} ] ${outfile}'," + " paddingCommandLine: '" _sox + " ${infile} ${outfile} pad ${duration}'," + " refinementCommandLine: '" _sox " ${infile} ${outfile} ${effects}' }" \end{configurationFileCode} @@ -1928,19 +1938,19 @@ To reduce the mental complexity we first define a map from voice name to staff by the following configuration file lines \begin{configurationFileCode} - _voiceNameToStaffListMap = \ - "{ drums : DrumStaff," \ - " keyboard : PianoStaff," \ + _voiceNameToStaffListMap = + "{ drums : DrumStaff," + " keyboard : PianoStaff," " percussion : DrumStaff }" \end{configurationFileCode} that are reused in the mapping from phase name \begin{configurationFileCode} - phaseAndVoiceNameToStaffListMap = \ - "{ extract : " _voiceNameToStaffListMap "," \ - " midi : " _voiceNameToStaffListMap "," \ - " score : " _voiceNameToStaffListMap "," \ + phaseAndVoiceNameToStaffListMap = + "{ extract : " _voiceNameToStaffListMap "," + " midi : " _voiceNameToStaffListMap "," + " score : " _voiceNameToStaffListMap "," " video : " _voiceNameToStaffListMap "}" \end{configurationFileCode} @@ -1995,11 +2005,11 @@ clef definition (it must be a bass clef). A typical definition might be given as follows: \begin{configurationFileCode} - _voiceNameToClefMap = \ - "{ bass" : 'bass_8', " \ - " drums" : ''," \ - " guitar" : 'G_8'," \ - " keyboardBottom" : 'bass'," \ + _voiceNameToClefMap = + "{ bass" : 'bass_8', " + " drums" : ''," + " guitar" : 'G_8'," + " keyboardBottom" : 'bass'," " percussion" : '' }" \end{configurationFileCode} @@ -2013,10 +2023,10 @@ our case ---~as above~--- the mapping is identical for all phases, but, of course, individual definitions per phase are possible. \begin{configurationFileCode} - phaseAndVoiceNameToClefMap = \ - "{ extract : " _voiceNameToClefMap "," \ - " midi : " _voiceNameToClefMap "," \ - " score : " _voiceNameToClefMap "," \ + phaseAndVoiceNameToClefMap = + "{ extract : " _voiceNameToClefMap "," + " midi : " _voiceNameToClefMap "," + " score : " _voiceNameToClefMap "," " video : " _voiceNameToClefMap "}" \end{configurationFileCode} @@ -2120,17 +2130,17 @@ identification by filling the variable \embeddedCode{voiceNameToScoreNameMap}. A possible setting is: \begin{configurationFileCode} - voiceNameToScoreNameMap = \ - "{ bass : bs," \ - " bgVocals : bvc," \ - " drums : dr," \ - " guitar : gtr," \ - " keyboard : kb," \ - " keyboardSimple : kb," \ - " organ : org," \ - " percussion : prc," \ - " strings : str," \ - " synthesizer : syn," \ + voiceNameToScoreNameMap = + "{ bass : bs," + " bgVocals : bvc," + " drums : dr," + " guitar : gtr," + " keyboard : kb," + " keyboardSimple : kb," + " organ : org," + " percussion : prc," + " strings : str," + " synthesizer : syn," " vocals : voc }" \end{configurationFileCode} @@ -2219,8 +2229,8 @@ directory given by \embeddedCode{targetDirectoryPath} with name {\DESCmidiVolumeList} {see text} - \varDescriptionLong{midiPanList} - {\DESCmidiPanList} + \varDescriptionLong{panPositionList} + {\DESCpanPositionList} {see text} \end{variableDescriptionTableLong} @@ -2239,7 +2249,7 @@ For example, the following settings in the configuration file midiChannelList = " 1, 2, 10 " midiInstrumentList = " 54, 2:29, 16 " midiVolumeList = " 90, 60, 110 " - midiPanList = " C, 0.5L, 0.1R" + panPositionList = " C, 0.5L, 0.1R" \end{configurationFileCode} define vocals to be a synth vox in the center with 3/4 volume, the @@ -2276,7 +2286,7 @@ variation, because there is a maximum and minimum velocity. Our example would result in \begin{configurationFileCode} - voiceNameToVariationFactorMap = "{ drums: 1.0/1.0," \ + voiceNameToVariationFactorMap = "{ drums: 1.0/1.0," " vocals: 1.5/1.2}" \end{configurationFileCode} @@ -2405,10 +2415,10 @@ variation here) and some emphasis on the second beat. In the configuration file it might look like \begin{configurationFileCode} - humanizationStyleRockHard = \ - "{ 0.00: 1/0.2, 0.25: 1.15/0," \ - " 0.50: 0.95/0.2, 0.75: 1.1/0," \ - " OTHER: 0.9/B0.25," \ + humanizationStyleRockHard = + "{ 0.00: 1/0.2, 0.25: 1.15/0," + " 0.50: 0.95/0.2, 0.75: 1.1/0," + " OTHER: 0.9/B0.25," " RASTER : 0.03125, SLACK : 0.1 }" \end{configurationFileCode} @@ -2430,8 +2440,8 @@ reggae on drums against a rumba on bass. So the style map in the configuration file might look like \begin{configurationFileCode} - measureToHumanizationStyleNameMap = \ - "{ 1 : humanizationStyleRockHard," \ + measureToHumanizationStyleNameMap = + "{ 1 : humanizationStyleRockHard," " 45 : humanizationStyleBeat}" \end{configurationFileCode} @@ -2609,24 +2619,24 @@ So we have two configuration file variables: So a video target definition for a single midrange tablet could look like this: \begin{configurationFileCode} - videoTargetMap = \ - "{" \ - " tablet:" \ - " { fileNameSuffix: '-i-v'," \ - " targetVideoDirectoryPath: '/pathto/tablet'," \ - " resolution: 132," \ - " height: 1024," \ - " width: 768," \ - " topBottomMargin: 5," \ - " leftRightMargin: 10," \ - " systemSize: 25," \ - " ffmpegPresetName: 'mydevice'," \ - " scalingFactor: 4," \ - " frameRate: 10," \ - " mediaType: 'TV Show'," \ - " subtitleColor: 2281766911," \ - " subtitleFontSize: 20," \ - " subtitlesAreHardcoded: false }" \ + videoTargetMap = + "{" + " tablet:" + " { fileNameSuffix: '-i-v'," + " targetVideoDirectoryPath: '/pathto/tablet'," + " resolution: 132," + " height: 1024," + " width: 768," + " topBottomMargin: 5," + " leftRightMargin: 10," + " systemSize: 25," + " ffmpegPresetName: 'mydevice'," + " scalingFactor: 4," + " frameRate: 10," + " mediaType: 'TV Show'," + " subtitleColor: 2281766911," + " subtitleFontSize: 20," + " subtitlesAreHardcoded: false }" "}" \end{configurationFileCode} @@ -2646,13 +2656,13 @@ separate track. Based on the video target definition given above a video file kind definition could look like this: \begin{configurationFileCode} - videoFileKindMap = \ - "{" \ - " tabletVocGtr:" \ - " { target: tablet," \ - " fileNameSuffix: '-i-v'," \ - " directoryPath: '/pathto/xyz'," \ - " voiceNameList: 'vocals, guitar' }" \ + videoFileKindMap = + "{" + " tabletVocGtr:" + " { target: tablet," + " fileNameSuffix: '-i-v'," + " directoryPath: '/pathto/xyz'," + " voiceNameList: 'vocals, guitar' }" "}" \end{configurationFileCode} @@ -2722,6 +2732,18 @@ restrictions apply for the notation videos: the bar numbers are activated for the line starts only and those bar numbers as well as the bar lines will be black. +%```````````````````````````` +\paragraph{Still Image Video} +%```````````````````````````` + +The \ltbvc\ can also produce a special video file just consisting of +still notation image pages, the subtitle data containing the measure +timings and information at which measure some image is shown. This is +a non-standard video kind: it is simply a tar file with still images, +subtitle file and file with mapping from measures to image names. Its +target must have a \embeddedCode{frameRate} of zero, because this +should not occur for real videos. + %~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ \subsection{Postprocessing Phases} %~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2855,9 +2877,9 @@ prevent distortion in the following steps), an enhancement of the In the configuration file this could look as follows: \begin{configurationFileCode} - _bassPostprocess = \ - " norm -24" \ - " equalizer 150 4o +10" \ + _bassPostprocess = + " norm -24" + " equalizer 150 4o +10" " lowpass -2 600 1.2o" \end{configurationFileCode} @@ -2865,12 +2887,12 @@ Based on that definition above the actual sound style can be defined as follows (referencing the definition by name): \begin{configurationFileCode} - soundStyleBassHard = \ - " highpass -2 40" \ - " lowpass -2 2k" \ - " norm -6 " \ - " tee" \ - " overdrive 12 0 " \ + soundStyleBassHard = + " highpass -2 40" + " lowpass -2 2k" + " norm -6 " + " tee" + " overdrive 12 0 " _bassPostprocess \end{configurationFileCode} @@ -2939,7 +2961,7 @@ reverbLevel}): \begin{commandLine} sox bass.wav bassA.wav highpass -2 40 lowpass -2 2k norm -6 - sox bassA.wav bass-processed.wav overdrive 12 0 norm -24 \ + sox bassA.wav bass-processed.wav overdrive 12 0 norm -24 equalizer 150 4o +10 lowpass -2 600 1.2o reverb 40 \end{commandLine} @@ -2967,9 +2989,10 @@ So finally each audio voice has its processed wav version in Parallel signal paths cannot be handled directly by sox and are emulated by \ltbvc. They can be specified as follows: \begin{itemize} - \item Parallel chains are specified by using chain separators in the - list of effects using the character token ``;'' (which can be - redefined by setting the variable + + \item Parallel \emph{chains} are specified by using chain separators + in the list of effects using the character token ``;'' (which + can be redefined by setting the variable \embeddedCode{audioProcessor.chainSeparator}). \item For each chain its (single) source and target are each given @@ -2983,18 +3006,21 @@ emulated by \ltbvc. They can be specified as follows: Note that, of course, the name of a chain source must occur as a chain target somewhere before. - The first chain has ``->'' (the raw audio file) as its - implicit chain source, the last chain has the refined audio - file as its implicit target. + Each chain has ``->'' (the raw audio file) as its implicit + chain source, the last chain has the refined audio file as its + implicit target. \item A chain may consist of a special ``mix'' effect that does a - weighted mix of several sources into a single target. + decibel-weighted mix of several sources into a single target. E.g. the chain \begin{configurationFileCode} - mix 1.0 -> 0.3 A-> 0.5 B-> ->C + mix 0 -> -3 A-> -6 B-> ->C \end{configurationFileCode} - mixes 100\% of the raw audio file, 30\% of A and 50\% of B - into C. + + mixes the raw audio file with 0dB attenuation (unchanged), the + result of chain A with -3dB attenuation (about 71\% volume) + and the result of chain B with -6dB attenuation (about 50\% + volume) into chain target C. Very often, the last chain is a mix of several sources into the refined audio file as the target. @@ -3007,11 +3033,11 @@ by an octave and having some parallel compression added. We assume that the bass is pre-processed by ``soundStyleBassStd'' and we simple add the postprocessing as follows: \begin{configurationFileCode} - soundStyleBassStd ->A \ - ; A-> pitch -1200 ->B \ - ; mix 1.0 A-> 0.75 B-> ->C \ - ; C-> compand 0.04,0.5 6:-25,-20,-5 -6 -90 0.02 ->D \ - ; mix 1.0 C-> 0.4 D-> + soundStyleBassStd ->A + ; A-> pitch -1200 ->B + ; mix 0 A-> -3 B-> ->C + ; C-> compand 0.04,0.5 6:-25,-20,-5 -6 -90 0.02 ->D + ; mix 0 C-> -8 D-> \end{configurationFileCode} ``A'' contains the preprocessed audio, ``B'' the pitched down version, @@ -3044,8 +3070,8 @@ variable \embeddedCode{voiceNameToOverrideFileNameMap}. As its name tells, it maps voice names to file names. \begin{configurationFileCode} - voiceNameToOverrideFileNameMap = \ - "{ vocals : 'vocals.flac'," \ + voiceNameToOverrideFileNameMap = + "{ vocals : 'vocals.flac'," "bass : 'mybass.wav' }" \end{configurationFileCode} @@ -3122,13 +3148,45 @@ configuration variables described for the ``rawaudio'' and The ``mix'' phase combines the refined audio files into one or more audio file with all voices and in aac audio format. -Audio levels of the individual voices, mastering effects and a final -amplification factor are specified in the configuration. Hence the -audio voices are mixed with those levels, have the mastering audio -processing applied and finally are amplified by the given factor -before the result is compressed into an AAC file. +Audio levels and pan positions of the individual voices, mastering +effects and a final amplification factor are specified in the +configuration. Hence the audio voices are mixed with those levels and +to the given pan positions, have the mastering audio processing +applied and finally are amplified by the given factor before the +result is compressed into an AAC file. + +When the variable \embeddedCode{mixingCommandLine} does not specify a +``pan'' placeholder, panning is done internally. This algorithm does +a traditional balancing of the stereo channels. That means, when the +pan value is less than zero the left channel is unchanged, while the +right channel is linearly attenuated and vice versa for a positive pan +value. So the amplification factors are + +\newcommand{\amplificationFunction}[3]{ + \begin{displaymath} + {\rm amplification~factor}_{\rm #1} = + \left \{ + \begin{array}{cl} + #2 & {{\bf if}~\rm panValue} < 0\\ + #3 & {{\bf if}~\rm panValue} \geq 0\\ + \end{array} + \right .\\ + \end{displaymath} +} -The target file is stored in the +\amplificationFunction{left}{1}{1 - {\rm panValue}} +\amplificationFunction{right}{1 + {\rm panValue}}{1} + +Figure~\ref{figure:panGraph} shows how this default panning function +affects the left and right channel of a stereo signal. + +\begin{centeredFigure} + \centeredExternalPicture{0.85}{panGraph.mps} + \caption{Default Panning Function for Left and Right Channels} + \label{figure:panGraph} +\end{centeredFigure} + +After panning and mixing the target file is stored in the \embeddedCode{audioTargetDirectoryPath} with its name constructed as the concatenation of \embeddedCode{targetFileNamePrefix}, \embeddedCode{fileNamePrefix} and suffix ``-ALL.m4a''. @@ -3146,11 +3204,11 @@ selectable audio group names are mapped onto sets of audio voice names. \begin{configurationFileCode} - audioGroupToVoicesMap = "{" \ - " base : bass/keyboard/keyboardSimple/strings," \ - " voc : vocals/bgVocals," \ - " gtr : guitar," \ - " drm : drums/percussion" \ + audioGroupToVoicesMap = "{" + " base : bass/keyboard/keyboardSimple/strings," + " voc : vocals/bgVocals," + " gtr : guitar," + " drm : drums/percussion" "}" \end{configurationFileCode} @@ -3194,8 +3252,8 @@ mastering effects for this voice and the final amplification level. \varDescriptionShort{languageCode} {\DESCaudioTrackLanguageCode} - \varDescriptionShort{voiceNameTo\-AudioLevelMap} - {\DESCaudioTrackVoiceNameToAudioLevelMap} + \varDescriptionShort{voiceNameTo\-MixSettingMap} + {\DESCaudioTrackVoiceNameToMixSettingMap} \varDescriptionShort{masteringEffectList} {\DESCaudioTrackMasteringEffectList} @@ -3229,7 +3287,7 @@ and some language name. So you must be creative\dots The final stage of audio processing is described by several attributes in the entry for a single audio track within the list of tracks: \embeddedCode{amplificationLevel}, -\embeddedCode{voiceNameToAudioLevelMap} and +\embeddedCode{voiceNameToMixSettingMap} and \embeddedCode{masteringEffectList}. Figure~\ref{figure:trackMixdown} illustrates how the audio voice files from the ``refinedaudio'' phase are combined into the several audio tracks by the mix phase. In @@ -3239,60 +3297,66 @@ there is no adaptation of the voice files done: they are taken unchanged from the previous phase. Nevertheless you can also define global settings for all of those and -reference them in the audio track list variable. Especially the audio -level map may be global, because the track specific mapping will only -use the levels of those voices defined in its associated audio groups. +reference them in the audio track list variable. Especially the mix +settings map may be global, because the track specific mapping will +only use the levels of those voices defined in its associated audio +groups. In the configuration file we can define auxiliary variables for the audio processing: \begin{configurationFileCode} - _voiceNameToAudioLevelMap = "{" \ - " bass : -6, keyboard : -10.5, keyboardSimple : -14," \ - " strings : -2, vocals : 0, bgVocals : -1," \ - " guitar : -4.5, drums : 1.6, percussion : 0" \ + _voiceNameToMixSettingMap = "{" + " bass : -6, keyboard : -10.5, keyboardSimple : -14," + " strings : -2, vocals : 0, bgVocals : -1," + " guitar : -4.5, drums : 1.6, percussion : 0" "}" _masteringEffectList = "" _amplificationLevel = -1.2 \end{configurationFileCode} +Note that an individual mix setting may also contain a pan +specification (separated by a slash). Hence ``bass : -6/0.3R'' would +also be okay and overrides the pan specification given as a list with +variable \embeddedCode{panPositionList}. + For audio tracks we also define an auxiliary variable each to make thing more comprehensible. This is the track with all voices: \begin{configurationFileCode} - _audioTrackWithAllVoices = \ - "all : { audioGroupList : base/voc/gtr/drm," \ - " audioFileTemplate : '$'," \ - " songNameTemplate : '$ [ALL]'," \ - " albumName : 'Best'," \ - " description : 'all voices'," \ - " languageCode : eng," \ - " voiceNameToAudioLevelMap : "_voiceNameToAudioLevelMap"," \ - " masteringEffectList : "_masteringEffectList"," \ + _audioTrackWithAllVoices = + "all : { audioGroupList : base/voc/gtr/drm," + " audioFileTemplate : '$'," + " songNameTemplate : '$ [ALL]'," + " albumName : 'Best'," + " description : 'all voices'," + " languageCode : eng," + " voiceNameToMixSettingMap : "_voiceNameToMixSettingMap"," + " masteringEffectList : "_masteringEffectList"," " amplificationLevel : "_amplificationLevel" }" \end{configurationFileCode} This is the track with all voices except for vocals: \begin{configurationFileCode} - _audioTrackNoVocals = \ - "novoc : { audioGroupList : base/gtr/drm," \ - " audioFileTemplate : '$-novoc'," \ - " songNameTemplate : '$ [-V]'," \ - " albumName : 'Best [no vocals]'," \ - " description : 'no vocals'," \ - " languageCode : deu," \ - " voiceNameToAudioLevelMap : "_voiceNameToAudioLevelMap"," \ - " masteringEffectList : "_masteringEffectList"," \ + _audioTrackNoVocals = + "novoc : { audioGroupList : base/gtr/drm," + " audioFileTemplate : '$-novoc'," + " songNameTemplate : '$ [-V]'," + " albumName : 'Best [no vocals]'," + " description : 'no vocals'," + " languageCode : deu," + " voiceNameToMixSettingMap : "_voiceNameToMixSettingMap"," + " masteringEffectList : "_masteringEffectList"," " amplificationLevel : "_amplificationLevel"}" \end{configurationFileCode} Both of them are used in the audio track list definition. \begin{configurationFileCode} - audioTrackList = "{" \ - _audioTrackWithAllVoices "," \ - _audioTrackWithNoVocals "," \ + audioTrackList = "{" + _audioTrackWithAllVoices "," + _audioTrackWithNoVocals "," ... "}" \end{configurationFileCode} @@ -3595,8 +3659,9 @@ lilypond macro structure is similar for different voices. Finally the drums do some monotonic blues accompaniment. We have to use the \embeddedCode{myDrums} name here, because \embeddedCode{drums} is a predefined name in lilypond. There is no preprocessing of the -lilypond fragment file: it is just included into some boilerplate -code. +lilypond fragment file that could fix this: the fragment is just +included into some boilerplate code, hence it must be conformant to +the lilypond syntax. \begin{lilypondFileCode} drmPhrase = \drummode { 8 hhc hhc } @@ -3660,8 +3725,8 @@ specify the soundfont location (via a temporary variable). \begin{configurationFileCode} _soundFonts = "/usr/local/midi/soundfonts/FluidR3_GM.SF2" - midiToWavRenderingCommandLine = \ - "fluidsynth -n -i -g 1 -R 0" \ + midiToWavRenderingCommandLine = + "fluidsynth -n -i -g 1 -R 0" " -F ${outfile} " _soundFonts " ${infile}" \end{configurationFileCode} @@ -3708,10 +3773,10 @@ timing and velocity. \begin{configurationFileCode} countInMeasureCount = 2 - humanizationStyleRockHard = \ - "{ 0.00: 0.95/A0.2, 0.25: 1.15/0," \ - " 0.50: 0.98/0.3, 0.75: 1.1/0," \ - " OTHER: 0.85/0.25," \ + humanizationStyleRockHard = + "{ 0.00: 0.95/A0.2, 0.25: 1.15/0," + " 0.50: 0.98/0.3, 0.75: 1.1/0," + " OTHER: 0.85/0.25," " SLACK:0.1, RASTER: 0.03125 }" \end{configurationFileCode} @@ -3728,15 +3793,15 @@ parameters can be found in the sox documentation~\cite{reference:soxDocumentation}. \begin{configurationFileCode} - soundStyleBassCrunch = \ - " compand 0.05,0.1 6:-20,0,-15" \ - " highpass -2 60 1o lowpass -2 800 1o equalizer 120 1o +3" \ + soundStyleBassCrunch = + " compand 0.05,0.1 6:-20,0,-15" + " highpass -2 60 1o lowpass -2 800 1o equalizer 120 1o +3" " reverb 60 100 20 100 10" soundStyleDrumsGrit = "overdrive 4 0 reverb 25 50 60 100 40" - soundStyleGuitarCrunch = \ - " compand 0.01,0.1 6:-10,0,-7.5 -6" \ - " overdrive 30 0 gain -10" \ - " highpass -2 300 0.5o lowpass -1 1200" \ + soundStyleGuitarCrunch = + " compand 0.01,0.1 6:-10,0,-7.5 -6" + " overdrive 30 0 gain -10" + " highpass -2 300 0.5o lowpass -1 1200" " reverb 40 50 50 100 30" soundStyleVocalsSimple = " overdrive 5 20" \end{configurationFileCode} @@ -3755,8 +3820,8 @@ voices has path ``./mediaFiles/test-wonderful\_song.m4a''. targetFileNamePrefix = "test-" albumArtFilePath = "./mediaFiles/demo.jpg" - audioGroupToVoicesMap = "{" \ - " base : bass/drums, voc : vocals, gtr : guitar" \ + audioGroupToVoicesMap = "{" + " base : bass/drums, voc : vocals, gtr : guitar" "}" \end{configurationFileCode} @@ -3786,20 +3851,21 @@ channels are at their defaults meaning 10 for drums and arbitrary other values for non-drums. \begin{configurationFileCode} - voiceNameList = "vocals, bass, guitar, drums" - midiInstrumentList = " 18, 35, 26, 13" - midiVolumeList = " 100, 120, 70, 110" - panPositionList = " C, 0.5L, 0.6R, 0.1L" - reverbLevelList = " 0.3, 0.0, 0.0, 0.0" - soundVariantList = "SIMPLE, CRUNCH, CRUNCH, GRIT" + voiceNameList = "vocals, bass, guitar, drums" + midiInstrumentList = " 18, 35, 26, 13" + midiVolumeList = " 100, 120, 70, 110" + panPositionList = " C, 0.5L, 0.6R, 0.1L" + reverbLevelList = " 0.3, 0.0, 0.0, 0.0" + soundVariantList = "SIMPLE, CRUNCH, CRUNCH, GRIT" \end{configurationFileCode} -The audio levels are given in a mapping, which is used in the audio -track list. We use a single mapping for all targets, that means the -relative levels are identical in all mixes. +The audio levels and pan positions are given in a separate mapping, +which is used in the audio track list. We use a single mapping for +all targets, that means the relative levels and pan positions are +identical in all mixes. \begin{configurationFileCode} - _voiceNameToAudioLevelMap = \ + _voiceNameToMixSettingMap = "{ vocals : -4, bass : 0, guitar : -6, drums : -2 }" \end{configurationFileCode} @@ -3819,7 +3885,7 @@ voices except vocals and starts in measure 1. \begin{configurationFileCode} humanizedVoiceNameSet = "bass, guitar, drums" - measureToHumanizationStyleNameMap = \ + measureToHumanizationStyleNameMap = "{ 1 : humanizationStyleRockHard }" \end{configurationFileCode} @@ -3836,7 +3902,7 @@ The overall tempo is 90bpm throughout the song. Because we want to set the \embeddedCode{audioTrackList} variable to non-default (default is one track with all voices), this must come \emph{after} the song parameters, because it relies on the voice name -to audio level mapping. +to mix settings mapping. For a separate global file this means, it has to be included as another fragment \emph{after} the song-specific setting. Since we are @@ -3847,40 +3913,40 @@ for convenience we put them each into an auxiliary variable (but this is not mandatory). \begin{configurationFileCode} - _audioTrackWithAllVoices = \ - "all : { audioGroupList : base/voc/gtr," \ - " audioFileTemplate : '$'," \ - " songNameTemplate : '$ [ALL]'," \ - " albumName : '$'," \ - " description : 'all voices'," \ - " languageCode : deu," \ - " voiceNameToAudioLevelMap : "_voiceNameToAudioLevelMap"}" + _audioTrackWithAllVoices = + "all : { audioGroupList : base/voc/gtr," + " audioFileTemplate : '$'," + " songNameTemplate : '$ [ALL]'," + " albumName : '$'," + " description : 'all voices'," + " languageCode : deu," + " voiceNameToMixSettingMap : "_voiceNameToMixSettingMap"}" \end{configurationFileCode} \begin{configurationFileCode} - _audioTrackWithoutVocals = \ - "novocals : { audioGroupList : base/gtr," \ - " audioFileTemplate : '$-v'," \ - " songNameTemplate : '$ [-V]'," \ - " albumName : '$ [-V]'," \ - " description : 'no vocals'," \ - " languageCode : eng," \ - " voiceNameToAudioLevelMap : "_voiceNameToAudioLevelMap"}" + _audioTrackWithoutVocals = + "novocals : { audioGroupList : base/gtr," + " audioFileTemplate : '$-v'," + " songNameTemplate : '$ [-V]'," + " albumName : '$ [-V]'," + " description : 'no vocals'," + " languageCode : eng," + " voiceNameToMixSettingMap : "_voiceNameToMixSettingMap"}" \end{configurationFileCode} Both are combined into the \embeddedCode{audioTrackList}. \begin{configurationFileCode} - audioTrackList = "{" \ - _audioTrackWithAllVoices "," \ - _audioTrackWithoutVocals \ + audioTrackList = "{" + _audioTrackWithAllVoices "," + _audioTrackWithoutVocals "}" \end{configurationFileCode} -The separate variable \embeddedCode{\_voiceNameToAudioLevelMap} -defined above defines the audio level for all voices; there are no -special mastering effects and all amplification levels are (the -default) 0dB. +The separate variable \embeddedCode{\_voiceNameToMixSettingMap} +defined above defines the audio level (and optionally the pan +positions) for all voices; there are no special mastering effects and +all amplification levels are (the default) 0dB. %-------------------------------- \section{Putting it All Together} @@ -4073,39 +4139,40 @@ planned for future versions: \renewcommand{\chapter}[2]{}% \bibliographystyle{plain} \begin{thebibliography}{SFNT-ORIG} - \bibitem[AAC]{reference:aacDocumentation} - \textit{QAAC - Quicktime AAC}.\\ - \hyperlink{https://sites.google.com/site/qaacpage/} - \bibitem[FFMPEG]{reference:ffmpegDocumentation} - \textit{FFMPEG - Documentation}.\\ - \hyperlink{http://ffmpeg.org/documentation.html} + \bibitem[AAC]{reference:aacDocumentation} + \textit{QAAC - Quicktime AAC}.\\ + \hyperlink{https://sites.google.com/site/qaacpage/} + + \bibitem[FFMPEG]{reference:ffmpegDocumentation} + \textit{FFMPEG - Documentation}.\\ + \hyperlink{http://ffmpeg.org/documentation.html} - \bibitem[FLUID]{reference:fluidsynthDocumentation} - \textit{FluidSynth - Software synthesizer based on the - SoundFont~2 specifications}.\\ - \hyperlink{http://fluidsynth.org} + \bibitem[FLUID]{reference:fluidsynthDocumentation} + \textit{FluidSynth - Software synthesizer based on the + SoundFont~2 specifications}.\\ + \hyperlink{http://fluidsynth.org} - \bibitem[LILY]{reference:lilypondDocumentation} - \textit{Lilypond - Music Notation for Everyone}.\\ - \hyperlink{http://lilypond.org} + \bibitem[LILY]{reference:lilypondDocumentation} + \textit{Lilypond - Music Notation for Everyone}.\\ + \hyperlink{http://lilypond.org} - \bibitem[MP4BOX]{reference:mp4boxDocumentation} - \textit{GPAC - General Documentation MP4Box}.\\ - \hyperlink{https://gpac.wp.imt.fr/mp4box/mp4box-documentation/} + \bibitem[MP4BOX]{reference:mp4boxDocumentation} + \textit{GPAC - General Documentation MP4Box}.\\ + \hyperlink{https://gpac.wp.imt.fr/mp4box/mp4box-documentation/} - \bibitem[SFNT-ORIG]{reference:fluidSoundfontOriginal} - \textit{FluidR3\_GM.sf3 SoundFont at musescore.org}.\\ - \hyperlink{https://github.com/musescore/MuseScore/raw/2.1/share/sound/FluidR3Mono\_GM.sf3} + \bibitem[SFNT-ORIG]{reference:fluidSoundfontOriginal} + \textit{FluidR3\_GM.sf2 SoundFont at archive.org}.\\ + \hyperlink{https://archive.org/compress/fluidr3-gm-gs} - \bibitem[SFNT-MS]{reference:fluidSoundfontMuseScore} - \textit{FluidR3\_GM.sf2 SoundFont at archive.org}.\\ - \hyperlink{https://archive.org/compress/fluidr3-gm-gs} + \bibitem[SFNT-MS]{reference:fluidSoundfontMuseScore} + \textit{FluidR3\_GM.sf3 SoundFont at musescore.org}.\\ + \hyperlink{https://github.com/musescore/MuseScore/raw/2.1/share/sound/FluidR3Mono\_GM.sf3} - \bibitem[SOX]{reference:soxDocumentation} - Chris Bagwell, Lance Norskog et al.: - \textit{SoX - Sound eXchange - Documentation}.\\ - \hyperlink{http://sox.sourceforge.net/Docs/Documentation} + \bibitem[SOX]{reference:soxDocumentation} + Chris Bagwell, Lance Norskog et al.: + \textit{SoX - Sound eXchange - Documentation}.\\ + \hyperlink{http://sox.sourceforge.net/Docs/Documentation} \end{thebibliography} \endgroup @@ -4145,7 +4212,8 @@ been mentioned first in the current document. \varDescriptionFinal{audioGroupToVoicesMap} {\DESCaudioGroupToVoicesMap} {single group "all" mapped to set of all voice - names in \embeddedCode{voiceNameList}} + names in \embeddedCode{voiceNameList} plus + a group for each voice with the same name} {Mix} \varDescriptionFinal{audioProcessor\bdot amplificationEffect} @@ -4326,11 +4394,6 @@ been mentioned first in the current document. {some default assignments from General MIDI} {MidiRelated} - \varDescriptionFinal{midiPanList} - {\DESCmidiPanList} - {all voices have center pan position} - {MidiRelated} - \varDescriptionFinal{midiToWavRendering\-CommandLine} {\DESCmidiToWavRenderingCommandLine} {\mandatory} @@ -4351,6 +4414,11 @@ been mentioned first in the current document. {empty, will be replaced by ffmpeg} {Program} + \varDescriptionFinal{panPositionList} + {\DESCpanPositionList} + {all voices have center pan position} + {MidiRelated} + \varDescriptionFinal{parallelTrack} {\DESCparallelTrack} {empty} @@ -4772,19 +4840,19 @@ been mentioned first in the current document. \item added static typing tags for additional documentation \item professionalized the processing and handling of - configuration file data - - \item renamed \embeddedCode{keepIntermediateFiles} to - \embeddedCode{intermediateFilesAreKept} + configuration file data by a generic data type management \item tried to reduce the mandatory configuration variables as much as possible by providing reasonable default settings - \item added a new logging file command line parameter overriding - the setting in the configuration file + \item added a new logging file command line parameter (overriding + the setting in the configuration file) \item set logging time resolution to 10ms (instead of 1s) + \item renamed \embeddedCode{keepIntermediateFiles} to + \embeddedCode{intermediateFilesAreKept} + \item added several minor corrections in the processing variables (e.g. \embeddedCode{tempLilypondFilePath} now has placeholders for phase and voice name) diff --git a/doc/ltbvcFigure.mp b/doc/ltbvcFigure.mp index 931b1cf..48aa21f 100644 --- a/doc/ltbvcFigure.mp +++ b/doc/ltbvcFigure.mp @@ -2,6 +2,7 @@ outputtemplate := "%j-%c.mps"; %prologues := 3; input boxes; +input graph; input hatching; input TEX; @@ -241,7 +242,7 @@ enddef; %============= % Humanization %============= - + numeric Humanization_fontSizeFactorArrowLabel; numeric Humanization_fontSizeFactorAxisLabel; numeric Humanization_fontSizeFactorTitle; @@ -601,7 +602,7 @@ def PhaseOrFileBox_makeAll = % postprocessing phases Box_make(phaseRawAudio, "C", btex rawaudio etex); Box_make(phaseRefinedAudio, "C", btex refinedaudio etex); - Box_make(phaseMixdown, "C", btex mixdown etex); + Box_make(phaseMix, "C", btex mix etex); Box_make(phaseFinalVideo, "C", btex finalvideo etex); Box_make(fileRawAudio, "FM", btex \stkC{raw}{audio}{files} etex); @@ -616,7 +617,7 @@ def PhaseOrFileBox_setPostprocessingBoxes (expr startPosition) = % sets up the position and size of postprocessing phases and % extracts - Box_setBoxPositions(phaseRawAudio, phaseRefinedAudio, phaseMixdown, + Box_setBoxPositions(phaseRawAudio, phaseRefinedAudio, phaseMix, phaseFinalVideo) (startPosition, PhaseOrFileBox_phaseOffsetVector); @@ -634,8 +635,8 @@ def PhaseOrFileBox_setPostprocessingEdges = addForwardFlow(phaseRawAudio, fileRawAudio); addBackwardFlow(fileRawAudio, phaseRefinedAudio); addForwardFlow(phaseRefinedAudio, fileRefinedAudio); - addBackwardFlow(fileRefinedAudio, phaseMixdown); - addForwardFlow(phaseMixdown, fileAudio); + addBackwardFlow(fileRefinedAudio, phaseMix); + addForwardFlow(phaseMix, fileAudio); addBackwardFlow(fileAudio, phaseFinalVideo); addForwardFlow(fileSilentVideo, phaseFinalVideo); addForwardFlow(fileSubtitleText, phaseFinalVideo); @@ -769,7 +770,7 @@ def MasteringBox_makeAll = Box_make(masteringGroupA, "B", ""); Box_make(mixerBoxA, "B", - btex \stkB{mixdown${}_A$~by}{\tt voiceNameToAudioLevelMap} etex + btex \stkB{mixing${}_A$~by}{\tt voiceNameToMixSettingsMap} etex rotated 90); Box_make(masteringBoxA, "B", btex \stkB{mastering${}_A$~via}{\tt masteringCommandList} etex @@ -784,7 +785,7 @@ def MasteringBox_makeAll = MasteringBoxA_size); Box_make(masteringGroupK, "B", ""); - Box_make(mixerBoxK, "B", btex mixdown${}_K$ etex rotated 90); + Box_make(mixerBoxK, "B", btex mixing${}_K$ etex rotated 90); Box_make(masteringBoxK, "B", btex mastering${}_K$ etex rotated 90); Box_make(amplificationBoxK, "B", btex amplification${}_K$ etex rotated 90); @@ -895,6 +896,7 @@ MasteringGroupK_centerPosition := (xpart MasteringGroupA_centerPosition, TrackBox_xPosition := xpart MasteringGroupA_centerPosition + MasteringGroup_xSize/2 + 20mm; + %==================== %==================== beginfig(1); @@ -924,7 +926,7 @@ beginfig(1); forsuffixes shape = phaseExtract, phaseScore, phaseMidi, phaseSilentVideo, phaseRawAudio, - phaseRefinedAudio, phaseMixdown, + phaseRefinedAudio, phaseMix, phaseFinalVideo: Box_setSizeAndColor(shape, PhaseOrFileBox_phaseBoxSize, PhaseOrFileBox_phaseFillColor); @@ -936,7 +938,7 @@ beginfig(1); forsuffixes shape = phaseExtract, phaseScore, phaseMidi, phaseSilentVideo, phaseRawAudio, - phaseRefinedAudio, phaseMixdown, + phaseRefinedAudio, phaseMix, phaseFinalVideo: addSupport(fileConfig, shape); endfor @@ -945,7 +947,7 @@ beginfig(1); phaseExtract, phaseScore, phaseMidi, phaseSilentVideo, fileExtract, fileScore, fileMidi, fileSilentVideo, fileSubtitleText, - phaseRawAudio, phaseRefinedAudio, phaseMixdown, + phaseRawAudio, phaseRefinedAudio, phaseMix, phaseFinalVideo, fileRawAudio, fileRefinedAudio, fileAudio, fileVideo); endfig; @@ -1020,7 +1022,7 @@ beginfig(3); endfor forsuffixes shape = phaseRawAudio, phaseRefinedAudio, - phaseMixdown, phaseFinalVideo: + phaseMix, phaseFinalVideo: Box_setSizeAndColor(shape, PhaseOrFileBox_phaseBoxSize, PhaseOrFileBox_phaseFillColor); endfor @@ -1029,13 +1031,13 @@ beginfig(3); PhaseOrFileBox_setPostprocessingEdges; forsuffixes shape = phaseRawAudio, phaseRefinedAudio, - phaseMixdown, phaseFinalVideo: + phaseMix, phaseFinalVideo: addSupport(fileConfig, shape); endfor drawboxed(fileConfig, fileMidi, fileSilentVideo, fileSubtitleText, - phaseRawAudio, phaseRefinedAudio, phaseMixdown, + phaseRawAudio, phaseRefinedAudio, phaseMix, phaseFinalVideo, fileRawAudio, fileRefinedAudio, fileAudio, fileVideo); endfig; @@ -1145,4 +1147,44 @@ beginfig(7); {$2\cdot\tau(p_i)\cdot\ell\cdot adj_t$} etex); endfig; +%==================== + +color PanGraph_gridColor; +color PanGraph_firstGraphColor; +color PanGraph_secondGraphColor; +numeric PanGraph_penWidth; +PanGraph_panWidth := 2pt; +PanGraph_gridColor := 0.8 * white; +PanGraph_firstGraphColor = (0, 0, 1); +PanGraph_secondGraphColor = (1, 0, 0); + +%==================== + +beginfig(8); + path leftPanGraph, rightPanGraph; + + leftPanGraph := + (-1, 1) -- (-0.75, 1) -- (-0.5, 1) -- (-0.25, 1) -- (0, 1) + -- (0.25, 0.75) -- (0.5, 0.5) -- (0.75, 0.25) -- (1, 0); + rightPanGraph := + (-1, 0) -- (-0.75, 0.25) -- (-0.5, 0.5) -- (-0.25, 0.75) -- (0, 1) + -- (0.25, 1) -- (0.5, 1) -- (0.75, 1) -- (1, 1); + + draw begingraph(80mm,60mm); + glabel.lft(btex Amplification Factor etex rotated 90, OUT); + glabel.bot(btex Pan Position etex, OUT); + setrange((-1,0), (1,1.2)); + autogrid(grid.bot,grid.lft) withcolor PanGraph_gridColor; + frame.llft; + gdraw leftPanGraph + withcolor PanGraph_firstGraphColor + withpen pencircle scaled PanGraph_panWidth; + glabel.top("left", 2) withcolor PanGraph_firstGraphColor; + gdraw rightPanGraph + withcolor PanGraph_secondGraphColor + withpen pencircle scaled PanGraph_panWidth; + glabel.top("right", 6) withcolor PanGraph_secondGraphColor;; + endgraph +endfig; + end diff --git a/doc/mptextmp.mp b/doc/mptextmp.mp new file mode 100644 index 0000000..266121b --- /dev/null +++ b/doc/mptextmp.mp @@ -0,0 +1 @@ +btex 1 etex diff --git a/lilypondToBandVideoConverter.pdf b/lilypondToBandVideoConverter.pdf index ff58d9c..a75c9eb 100644 Binary files a/lilypondToBandVideoConverter.pdf and b/lilypondToBandVideoConverter.pdf differ diff --git a/lilypondtobvc/src/basemodules/__init__.py b/lilypondtobvc/src/basemodules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lilypondtobvc/src/basemodules/arraytypes.py b/lilypondtobvc/src/basemodules/arraytypes.py new file mode 100644 index 0000000..678810a --- /dev/null +++ b/lilypondtobvc/src/basemodules/arraytypes.py @@ -0,0 +1,364 @@ +# ArrayTypes - simple arrays for bits, integers and reals +# +# author: Dr. Thomas Tensi, 2023 + +#==================== + +from array import array +from basemodules.simpletypes import Bit, BitList, Boolean, Character, \ + Class, Integer, IntegerList, List, \ + Natural, NaturalList, Object, \ + RealList, String +from basemodules.ttbase import iif + +#==================== + +class _MyArray: + + #-------------------- + # PRIVATE FEATURES + #-------------------- + + def _ensureCount (self, + count : Natural): + """Makes allocated size of at least """ + + allocatedCount = len(self._data) + + if allocatedCount < count: + rest = count - allocatedCount + dummyData = array(self._typeCode, [ self._initialValue ]) + extensionCount = 1 + + while rest > 0: + if rest & extensionCount > 0: + self._data.extend(dummyData) + rest -= extensionCount + + dummyData.extend(dummyData) + extensionCount *= 2 + + #-------------------- + + @classmethod + def _fromList (cls, + type : Class, + data : List) -> Object: + """Fills array from """ + + count = len(data) + result = type(count) + + for i in range(count): + result._data[i] = data[i] + + return result + + #-------------------- + + def _slice (self, + first : Natural, + last : Natural = None) -> array: + """Returns array slice from to """ + + count = self._count + + if last is None: last = count + if first < 0: first = count + first + if last < 0: last = count + last + + count = max(last - first, 0) + result = self._data[first:last] + + return result + + #-------------------- + + def _toString (self, + separatorIsUsed : Boolean) -> String: + """Returns string representation""" + + count = self._count + clsName = self.__class__.__name__ + separator = iif(separatorIsUsed, ", ", "") + dataAsString = separator.join(( "%s" % value + for value in self._data )) + return ("%s(%s %r)" % (clsName, self._typeCode, dataAsString)) + + #-------------------- + # EXPORTED FEATURES + #-------------------- + + def __init__ (self, + typeCode : Character, + initialValue : Object, + count : Natural): + """Initializes array from with with + elements""" + + self._data = array(typeCode) + self._count = count + self._typeCode = typeCode + self._initialValue = initialValue + + self._ensureCount(count) + + #-------------------- + + def __iter__ (self): + """Returns an iterator onto self""" + + return iter(self.values()) + + #-------------------- + #-------------------- + + def at (self, + i : Integer) -> Bit: + """Returns value at position """ + + return self._data[i] + + #-------------------- + + def count (self) -> Natural: + """Returns size of array""" + + return self._count + + #-------------------- + + def find (self, + otherArray : Object) -> Integer: + """Returns position where is sub list (or -1 on + failure)""" + + result = -1 + count = self._count + otherCount = otherArray._count + + for i in range(count - otherCount): + isFound = True + k = i + + for j, otherValue in enumerate(otherArray): + if otherValue == self._data[k]: + k += 1 + else: + isFound = False + break + + if isFound: + result = i + + return result + + #-------------------- + + def prepend (self, + otherArray : Object): + """Prepends to current array""" + + originalCount = self._count + otherCount = otherArray._count + self._ensureCount(originalCount + otherCount) + + for i in reversed(range(originalCount)): + self._data[i + otherCount] = self._data[i] + + for i in range(otherCount): + self._data[i] = otherArray._data[i] + + self._count += otherCount + + #-------------------- + + def set (self, + i : Natural, + value : Object): + """Sets data at position to """ + + self._data[i] = value + + #-------------------- + + def trim (self, + count : Natural): + """Sets size of list to """ + + self._count = count + + #-------------------- + + def values (self) -> List: + return self._data[:self._count] + +#==================== + +class BitArray (_MyArray): + """An array of bits 0 and 1""" + + _typeCode = "b" + + #-------------------- + #-------------------- + + def __init__ (self, + count : Natural = 0): + cls = self.__class__ + _MyArray.__init__(self, cls._typeCode, 0, count) + + #-------------------- + + def __repr__ (self): + """Returns string representation""" + + return self._toString(False) + + #-------------------- + #-------------------- + + @classmethod + def fromList (cls, + data : BitList) -> Object: + """Fills array from """ + + return _MyArray._fromList(BitArray, data) + + #-------------------- + + @classmethod + def fromString (cls, + st : String): + """Make bit list from string consisting of zeros and + ones""" + + result = BitArray(len(st)) + + for i, ch in enumerate(st): + value = iif(ch == "0", 0, 1) + result.set(i, value) + + return result + + #-------------------- + + def slice (self, + first : Natural, + last : Natural = None) -> Object: + """Returns slice from to """ + + cls = self.__class__ + return cls.fromList(self._slice(first, last)) + + #-------------------- + + def updateWithValue (self, + position : Natural, + value : Natural, + count : Natural = 4): + """Updates bit array starting at by value in + little-endian manner using bits""" + + ## Logging.trace(">>: position = %d, value = %d, count = %d", + ## position, value, count) + + for i in range(count): + value, positionValue = divmod(int(value), 2) + self.set(position, positionValue) + position = position + 1 + + ## Logging.trace("<<") + +#==================== + +class IntegerArray (_MyArray): + """An array of integer values""" + + _typeCode = "l" + + #-------------------- + + def __init__ (self, + count : Natural = 0): + cls = self.__class__ + _MyArray.__init__(self, cls._typeCode, 0, count) + + #-------------------- + + def __repr__ (self): + """Returns string representation""" + + return self._toString(True) + + #-------------------- + #-------------------- + + @classmethod + def fromList (cls, + data : IntegerList) -> Object: + """Fills array from """ + + return _MyArray._fromList(IntegerArray, data) + +#==================== + +class NaturalArray (_MyArray): + """An array of natural values""" + + _typeCode = "L" + + #-------------------- + + def __init__ (self, + count : Natural = 0): + cls = self.__class__ + super.__init__(self, cls._typeCode, 0, count) + + #-------------------- + + def __repr__ (self): + """Returns string representation""" + + return self._toString(True) + + #-------------------- + #-------------------- + + @classmethod + def fromList (cls, + data : NaturalList) -> Object: + """Fills array from """ + + return _MyArray._fromList(NaturalArray, data) + +#==================== + +class RealArray (_MyArray): + """An array of real values""" + + _typeCode = "d" + + #-------------------- + + def __init__ (self, + count : Natural = 0): + cls = self.__class__ + _MyArray.__init__(self, cls._typeCode, 0.0, count) + + #-------------------- + + def __repr__ (self): + """Returns string representation""" + + return self._toString(True) + + #-------------------- + #-------------------- + + @classmethod + def fromList (cls, + data : RealList) -> Object: + """Fills array from """ + + return _MyArray._fromList(RealArray, data) diff --git a/lilypondtobvc/src/basemodules/attributemanager.py b/lilypondtobvc/src/basemodules/attributemanager.py index d8db99c..05e288b 100644 --- a/lilypondtobvc/src/basemodules/attributemanager.py +++ b/lilypondtobvc/src/basemodules/attributemanager.py @@ -7,11 +7,12 @@ # IMPORTS #==================== -from .simplelogging import Logging -from .simpletypes import Dictionary, Object, String, StringList -from .stringutil import adaptToKind -from .validitychecker import ValidityChecker -from .ttbase import iif, iif3 +from basemodules.simplelogging import Logging +from basemodules.simpletypes import Dictionary, Object, String, \ + StringList +from basemodules.stringutil import adaptToKind +from basemodules.validitychecker import ValidityChecker +from basemodules.ttbase import iif, iif3 #==================== diff --git a/lilypondtobvc/src/basemodules/configurationfile.py b/lilypondtobvc/src/basemodules/configurationfile.py new file mode 100644 index 0000000..ff3bdeb --- /dev/null +++ b/lilypondtobvc/src/basemodules/configurationfile.py @@ -0,0 +1,601 @@ +# -*- coding: utf-8 -*- +# configurationfile - provides reading from a configuration file containing +# comments and assignment to variables +# +# author: Dr. Thomas Tensi, 2014-04 + +#==================== + +import io +import re + +from basemodules.operatingsystem import OperatingSystem +from basemodules.simpleassertion import Assertion +from basemodules.simplelogging import Logging +from basemodules.simpletypes import Dictionary, List, Natural, Object, \ + ObjectSet, Map, String, StringList, \ + StringMap, StringSet, Tuple +from basemodules.typesupport import isString +from basemodules.ttbase import iif, isStdPython, missingValue + +#==================== + +def _reprOfStringToValueMap (stringMap : Map) -> String: + """String representation for a string to value map """ + + entrySeparator = u"§" + entryTemplate = "%s: %s" + keyList = sorted(list(stringMap.keys())) + result = "" + + for key in keyList: + value = stringMap[key] + result += (iif(result == "", "", entrySeparator) + + entryTemplate % (key, value)) + + result = "{" + result + "}"; + return result + +#==================== + +class _Token: + """Simple token within table definition string parser""" + + Kind_number = "number" + Kind_string = "string" + Kind_operator = "operator" + Kind_realNumber = "real" + Kind_integerNumber = "integer" + + #-------------------- + + def __init__ (self, + start : Natural, text : String, + kind : String, value : Object): + """Initializes token with start position, token text, token kind and + value""" + + self.start = start + self.text = text + self.kind = kind + self.value = value + + #-------------------- + + def __repr__ (self) -> String: + """String representation for token """ + + st = ("_Token(%r, '%s', %s, %r)" + % (self.start, self.text, self.kind, self.value)) + return st + +#==================== + +if isStdPython: + _TokenList = List[_Token] +else: + _TokenList = List # List[_Token] + +#==================== + +class ConfigurationFile: + """Provides services for reading a configuration file with key - + value assignments. The parsing process calculates a map from + name to value where the values may be booleans, integers, + reals or strings.""" + + _importCommandName = "INCLUDE" + _trueBooleanValueNames = ["TRUE", "WAHR"] + _validBooleanValueNames = _trueBooleanValueNames + ["FALSE", "FALSCH"] + _commentMarker = "--" + _continuationMarker = "\\" + _realRegExp = re.compile(r"^[+\-]?[0-9]+\.[0-9]*$") + _integerRegExp = re.compile(r"^[+\-]?[0-9]+$") + _hexIntegerRegExp = re.compile(r"^0[xX][0-9A-Fa-f]+$") + _keyValueRegExp = re.compile(r"^(\w+)\s*=\s*(.*)$") + _whiteSpaceCharRegExp = re.compile(r"^\s$") + _identifierCharRegExp = re.compile(r"[A-Za-z0-9_]") + _identifierLineRegExp = re.compile(r"\s*" + + r"[A-Za-z_][A-Za-z0-9_]*" + + r"\s*=") + _escapeCharacter = "\\" + _doubleQuoteCharacter = '"' + + _searchPathList = ["."] + + # error messages + _errorMsg_badOpeningCharacter = "expected either { or [" + + #-------------------- + # LOCAL FEATURES + #-------------------- + + @classmethod + def _adaptConfigurationValue (cls, value : String) -> Object: + """Takes string and constructs either a boolean, a numeric + value or a sanitized string.""" + + Logging.trace(">>: %r", value) + uppercasedValue = value.upper() + + if uppercasedValue in cls._validBooleanValueNames: + result = (uppercasedValue in cls._trueBooleanValueNames) + elif cls._integerRegExp.match(value): + result = int(value) + elif cls._hexIntegerRegExp.match(value): + result = int(value, 16) + elif cls._realRegExp.match(value): + result = float(value) + else: + result = value + + Logging.trace("<<: %r", result) + return result + + #-------------------- + + @classmethod + def _combineFragmentedString (cls, st : String) -> String: + """Combines - possibly fragmented - external representation of a + string given by into a sanitized string.""" + + Logging.trace(">>: %r", st) + + ParseState_inLimbo = 0 + ParseState_inOther = 1 + ParseState_inString = 2 + ParseState_inLiteral = 3 + ParseState_inEscape = 4 + + parseState = ParseState_inLimbo + result = "" + + for ch in st: + # process finite state automaton with three states based + # on next character in string + # Logging.trace("--: (%d) character: %r", parseState, ch) + + if parseState == ParseState_inLimbo: + if ch == cls._doubleQuoteCharacter: + parseState = ParseState_inString + elif not cls._whiteSpaceCharRegExp.search(ch): + parseState = ParseState_inLiteral + result += ch + elif parseState == ParseState_inString: + if ch == cls._doubleQuoteCharacter: + parseState = ParseState_inLimbo + else: + result += ch + parseState = iif(ch == cls._escapeCharacter, + ParseState_inEscape, parseState) + elif parseState == ParseState_inLiteral: + result += ch + if cls._whiteSpaceCharRegExp.search(ch): + parseState = ParseState_inLimbo + elif parseState == ParseState_inEscape: + result += ch + parseState = ParseState_inString + else: + Assertion.check(False, + "bad parse state - %s" % parseState) + + Logging.trace("<<: %r", result) + return result + + #-------------------- + + def _expandVariables (self, st : String) -> String: + """Expands all variables embedded in .""" + + Logging.trace(">>: %r", st) + cls = self.__class__ + + # collect identifiers embedded in value and replace them by + # their value + ParseState_inLimbo = 0 + ParseState_inString = 1 + ParseState_inEscape = 2 + ParseState_inIdentifier = 3 + parseStateToString = { 0 : "-", 1 : "S", + 2 : cls._escapeCharacter, 3 : "I" } + + parseState = ParseState_inLimbo + result = "" + identifier = "" + fsaTrace = "" + + for ch in st: + # process finite state automaton with three states based + # on next character in string + fsaTrace += (iif(fsaTrace == "", "", " ") + + "[%s] %s" % (parseStateToString[parseState], ch)) + + if parseState == ParseState_inLimbo: + if cls._identifierCharRegExp.search(ch): + identifier = ch + parseState = ParseState_inIdentifier + else: + result += ch + if ch == cls._doubleQuoteCharacter: + parseState = ParseState_inString + elif parseState == ParseState_inString: + result += ch + if ch == cls._doubleQuoteCharacter: + parseState = ParseState_inLimbo + elif ch == cls._escapeCharacter: + parseState = ParseState_inEscape + elif parseState == ParseState_inEscape: + result += ch + parseState = ParseState_inString + elif parseState == ParseState_inIdentifier: + if cls._identifierCharRegExp.search(ch): + identifier += ch + else: + identifierValue = self._findIdentifierValue(identifier) + result += identifierValue + result += ch + parseState = iif(ch == cls._doubleQuoteCharacter, + ParseState_inString, ParseState_inLimbo) + + if parseState == ParseState_inIdentifier: + identifierValue = self._findIdentifierValue(identifier) + result += identifierValue + + Logging.trace("--: accumulatedFSATrace = %s", fsaTrace) + Logging.trace("<<: %r", result) + return result + + #-------------------- + + def _findIdentifierValue (self, identifier : String) -> String: + """Returns string representation of associated identifier value + for ; if not found in current key to value map, the + identifier itself is returned""" + + Logging.trace(">>: %s", identifier) + cls = self.__class__ + + if identifier not in self._keyToValueMap: + # leave identifier as is (it might be some value name like + # wahr or false + Logging.traceError("no expansion found") + result = identifier + else: + result = self._keyToValueMap[identifier] + + if not isString(result): + result = repr(result) + else: + result = (cls._doubleQuoteCharacter + result + + cls._doubleQuoteCharacter) + + Logging.trace("<<: expanded %s into %r", identifier, result) + return result + + #-------------------- + + def _lookupFileName (self, + enclosingDirectoryName : String, + originalFileName : String) -> String: + """Returns file name in search paths based on """ + + Logging.trace(">>: directory = %r, file = %r", + enclosingDirectoryName, originalFileName) + + cls = self.__class__ + result = None + separator = OperatingSystem.pathSeparator + simpleFileName = OperatingSystem.basename(originalFileName) + searchPathList = list(cls._searchPathList) + searchPathList.append(enclosingDirectoryName) + + for directoryName in searchPathList: + fileName = iif(directoryName == ".", originalFileName, + directoryName + separator + simpleFileName) + isFound = OperatingSystem.hasFile(fileName) + Logging.trace("--: %r -> found = %r", fileName, isFound) + + if isFound: + result = fileName + break + + Logging.trace("<<: %r", result) + return result + + #-------------------- + + @classmethod + def _mergeContinuationLines (cls, lineList : StringList): + """Merges continuation lines in into single cumulated line + and replaces continuations by empty lines (to preserve line + numbers); the continuation logic is simple: a new + identifier definition line (starting with an identifier and + an equals sign) starts a new logical line unconditionally, + otherwise physical lines are collected and combined with + the previous logical line; embedded comment lines are + skipped and an empty (whitespace only) physical line stops + the collection process (unless followed by a continuation + character)""" + + Logging.trace(">>") + + cumulatedLine = "" + markerLength = len(cls._continuationMarker) + lineListLength = len(lineList) + + for i, originalLine in enumerate(lineList): + currentLine = originalLine.strip() + lineList[i] = "" + + if currentLine.endswith(cls._continuationMarker): + # strip off obsolete continuation marker + currentLine = currentLine[:-markerLength].rstrip() + + if cls._identifierLineRegExp.match(currentLine): + # this is a new definition + + if cumulatedLine > "": + lineList[i - 1] = cumulatedLine + + cumulatedLine = currentLine + loggingFormat = "--: new definition %d (%r)" + elif currentLine.startswith(cls._commentMarker): + # skip comment + loggingFormat = "--: skipped comment %d (%r)" + elif originalLine == "": + if cumulatedLine == "": + loggingFormat = "--: empty line %d (%r)" + else: + lineList[i - 1] = cumulatedLine + loggingFormat = \ + "--: empty line ended previous definition %d (%r)" + + cumulatedLine = "" + else: + # this is not an empty line and it does not start with a + # definition sequence or an import + cumulatedLine += " " + currentLine + loggingFormat = "--: collected continuation %d (%r)" + + Logging.trace(loggingFormat, i+1, currentLine) + + if cumulatedLine > "": + lineList[-1] = cumulatedLine + Logging.trace("--: final line (%s)", currentLine); + + Logging.trace("<<") + + #-------------------- + + @classmethod + def _mustHave (cls, + token : _Token, kindSet : StringSet, + valueSet : ObjectSet = None): + """Ensures that is of a kind in ; if + is not None, token value is also checked""" + + Logging.trace(">>: token = %r, kindSet = %r, valueSet = %r", + token, kindSet, valueSet) + + errorPosition, errorMessage = -1, "" + + if token.kind not in kindSet: + errorPosition = token.start + errorMessage = ("expected kind from %r, found %s" + % (kindSet, token.kind)) + elif valueSet is not None and token.value not in valueSet: + errorPosition = token.start + errorMessage = ("expected value from %r, found %r" + % (valueSet, token.value)) + + result = (errorPosition, errorMessage) + Logging.trace("<<: %r", result) + return result + + #-------------------- + + def _parseConfiguration (self, lineList : StringList): + """Parses configuration file data given by and updates + key to value and key to string value map.""" + + Logging.trace(">>") + + cls = self.__class__ + cls._mergeContinuationLines(lineList) + + for i, currentLine in enumerate(lineList): + lineNumber = i + 1 + + # remove leading and trailing white space from line + currentLine = currentLine.strip() + Logging.trace("--: (%d) %s", i+1, currentLine) + + if (currentLine == "" + or currentLine.startswith(cls._commentMarker)): + # this is an empty line or comment line => skip it + pass + else: + match = cls._keyValueRegExp.search(currentLine) + + if not match: + Logging.traceError("bad line %d without key-value-pair", + lineNumber) + else: + key = match.group(1) + value = match.group(2) + value = self._expandVariables(value) + value = cls._combineFragmentedString(value) + self._keyToStringValueMap[key] = value + Logging.trace("--: string value %r -> %r", key, value) + value = cls._adaptConfigurationValue(value) + self._keyToValueMap[key] = value + Logging.trace("--: adapted value %r -> %r", key, value) + + Logging.trace("<<: %r", self._keyToValueMap) + + #-------------------- + + def _readFile (self, + directoryName : String, + fileName : String, + lineList : StringList, + visitedFileNameSet : StringSet): + """Appends lines of configuration file with to + with leading and trailing whitespace stripped; also handles + embedded imports of files (relative to ; + tells which files have already been + visited""" + + Logging.trace(">>: fileName = %r, directory = %r, visitedFiles = %r", + fileName, directoryName, visitedFileNameSet) + + cls = self.__class__ + errorMessage = "" + isOkay = True + + originalFileName = fileName + fileName = self._lookupFileName(directoryName, originalFileName) + + if fileName is None: + errorMessage = "cannot find %r" % originalFileName + isOkay = False + elif fileName in visitedFileNameSet: + Logging.trace("--: file already included %r", originalFileName) + else: + visitedFileNameSet.add(fileName) + directoryName = OperatingSystem.dirname(fileName) + + with io.open(fileName, "rt", + encoding="utf-8") as configurationFile: + configFileLineList = configurationFile.readlines() + + for currentLine in configFileLineList: + currentLine = currentLine.strip() + isImportLine = \ + currentLine.startswith(cls._importCommandName) + + if isImportLine: + importedFileName = currentLine.split('"')[1] + currentLine = cls._commentMarker + " " + currentLine + + lineList.append(currentLine) + + if isImportLine: + isAbsolutePath = (importedFileName.startswith("/") + or importedFileName.startswith("\\") + or importedFileName[1] == ":") + + # if isAbsolutePath: + # directoryPrefix = "" + # else: + # directoryName = OperatingSystem.dirname(fileName) + # directoryPrefix = iif(directoryName == ".", "", + # directoryName + # + iif(directoryName > "", + # "/", "")) + + #importedFileName = directoryPrefix + importedFileName + Logging.trace("--: IMPORT %r", importedFileName) + + isOkay = self._readFile(directoryName, + importedFileName, lineList, + visitedFileNameSet) + if not isOkay: + Logging.traceError("import failed for %r in %r", + importedFileName, + cls._searchPathList) + isOkay = False + break + + Logging.trace("<<: %r, %r", isOkay, errorMessage) + return isOkay + + #-------------------- + # EXPORTED FEATURES + #-------------------- + + @classmethod + def setSearchPaths (cls, searchPathList : StringList): + """Sets list of search paths to .""" + + Logging.trace(">>: %r", searchPathList) + cls._searchPathList = ["."] + searchPathList + Logging.trace("<<") + + #-------------------- + + def __init__ (self, fileName : String): + """Parses configuration file given by and sets + internal key to value map.""" + + Logging.trace(">>: %r", fileName) + + self._keyToValueMap = {} + self._keyToStringValueMap = {} + visitedFileNameSet = set() + lineList = [] + isOkay = self._readFile("", fileName, lineList, visitedFileNameSet) + self._parseConfiguration(lineList) + + Logging.trace("<<: %s", + _reprOfStringToValueMap(self._keyToValueMap)) + + #-------------------- + + def asStringMap (self) -> StringMap: + """Returns mapping from all keys in configuration file to their + effective values""" + + Logging.trace(">>") + result = dict(self._keyToValueMap) + Logging.trace("<<: %r", result) + return result + + #-------------------- + + def asDictionary (self) -> Dictionary: + """Returns mapping from all keys in configuration file to their + string values as found in the file""" + + Logging.trace(">>") + result = dict(self._keyToStringValueMap) + Logging.trace("<<: %r", result) + return result + + #-------------------- + + def keySet (self) -> StringSet: + """Returns set of all keys in configuration file""" + + Logging.trace(">>") + result = set(self._keyToValueMap.keys()) + Logging.trace("<<: %r", result) + return result + + #-------------------- + + def value (self, + key : String, + defaultValue : Object = missingValue) -> Object: + """Returns value for in configuration file; if + is missing, an error message is logged when + there is no associated value, otherwise is + returned for a missing entry""" + + Logging.trace(">>: key = %s, defaultValue = %r", + key, defaultValue) + + isMandatory = (defaultValue == missingValue) + result = None + + if key in self._keyToValueMap: + result = self._keyToValueMap[key] + if isString(result): + result = result.replace("\\\"", "\"") + elif isMandatory: + Logging.traceError("cannot find value for %s", key) + else: + result = defaultValue + + Logging.trace("<<: %s", result) + return result diff --git a/lilypondtobvc/src/basemodules/datatypesupport.py b/lilypondtobvc/src/basemodules/datatypesupport.py index 8ee121b..0c3c6a1 100644 --- a/lilypondtobvc/src/basemodules/datatypesupport.py +++ b/lilypondtobvc/src/basemodules/datatypesupport.py @@ -9,14 +9,15 @@ from copy import deepcopy import dataclasses -from .regexppattern import RegExpPattern -from .simpleassertion import Assertion -from .simplelogging import Logging -from .simpletypes import Callable, DataType, Dictionary, Object, \ - ObjectList, String, StringList, StringMap -from .stringutil import adaptToKind,convertStringToMap -from .ttbase import iif, iif3 -from .validitychecker import ValidityChecker +from basemodules.regexppattern import RegExpPattern +from basemodules.simpleassertion import Assertion +from basemodules.simplelogging import Logging +from basemodules.simpletypes import Callable, DataType, Dictionary, \ + Object, ObjectList, String, \ + StringList, StringMap +from basemodules.stringutil import adaptToKind, deserializeToMap +from basemodules.ttbase import iif, iif3 +from basemodules.validitychecker import ValidityChecker #==================== # PRIVATE FEATURES @@ -351,7 +352,7 @@ def generateObjectListFromString (cls, Logging.trace(">>: %r", st) result = [] - table = convertStringToMap(st) + table = deserializeToMap(st) for name, attributeNameToValueMap in table.items(): attributeNameToValueMap["name"] = name @@ -377,7 +378,7 @@ def generateObjectMapFromString (cls, Logging.trace(">>: %r", st) result = {} - table = convertStringToMap(st) + table = deserializeToMap(st) for name, attributeNameToValueMap in table.items(): attributeNameToValueMap["name"] = name diff --git a/lilypondtobvc/src/basemodules/jsonfile.py b/lilypondtobvc/src/basemodules/jsonfile.py index 0a046e6..bfb20be 100644 --- a/lilypondtobvc/src/basemodules/jsonfile.py +++ b/lilypondtobvc/src/basemodules/jsonfile.py @@ -12,8 +12,9 @@ from .operatingsystem import OperatingSystem from .simplelogging import Logging -from .simpletypes import Boolean, Dictionary, Natural, Positive, \ - String, StringList, StringSet +from .simpletypes import Boolean, Dictionary, Natural, Object, \ + Positive, String, StringList, StringMap, \ + StringSet from .ttbase import iif #==================== @@ -86,7 +87,7 @@ def __repr__ (self) -> String: @classmethod def adaptKind (cls, - token : _Token, + token : Object, nameToValueMap : Dictionary): """Handles special strings in and adapts it accordingly; if token is an identifier, it might be replaced by value in @@ -405,7 +406,7 @@ class SimpleJsonFile: _commentMarker = "--" _definitionCommandName = "#define" _importCommandName = "#include" - _searchPathList = ["."] + _searchPathList = [] # the map from names in define statements to associated values _nameToValueMap = {} @@ -416,33 +417,40 @@ class SimpleJsonFile: @classmethod def _importFile (cls, + referenceDirectoryName : String, fileName : String, lineList : StringList, visitedFileNameSet : StringSet): - """Imports file named and appends line to , + """Imports file named (possibly relative to + ) and appends line to , updates to break import cycles""" - Logging.trace(">>: file = %r, visitedFiles = %r", - fileName, visitedFileNameSet) + Logging.trace(">>: referenceDirectory = '%s', file = '%s'," + + " visitedFiles = %r", + referenceDirectoryName, fileName, visitedFileNameSet) separator = OperatingSystem.pathSeparator isAbsolutePath = \ - OperatingSystem.isAbsoluteFileName(importedFileName) + OperatingSystem.isAbsoluteFileName(fileName) if isAbsolutePath: - directoryPrefix = "" + effectiveFileName = fileName else: - directoryName = OperatingSystem.dirname(fileName) - directoryPrefix = iif(directoryName == ".", "", - directoryName - + iif(directoryName > "", - separator, "")) - - importedFileName = directoryPrefix + importedFileName - Logging.trace("--: IMPORT %r", importedFileName) - - isOkay = cls._readFile(importedFileName, lineList, - visitedFileNameSet) + # try to find file in search paths + effectiveFileName = cls._lookupFileName(fileName) + + if effectiveFileName is None: + # file is relative to reference directory + effectiveFileName = ("%s%s%s" + % (referenceDirectoryName, + separator, + fileName)) + + effectiveFileName = \ + OperatingSystem.normalizedFileName(effectiveFileName) + Logging.trace("--: IMPORT %r", effectiveFileName) + isOkay = \ + cls._readFile(effectiveFileName, lineList, visitedFileNameSet) Logging.trace("<<: %r", isOkay) return isOkay @@ -459,11 +467,11 @@ def _lookupFileName (cls, result = None separator = OperatingSystem.pathSeparator - simpleFileName = OperatingSystem.basename(originalFileName, True) + simpleFileName = OperatingSystem.basename(originalFileName) for directoryName in cls._searchPathList: fileName = iif(directoryName == ".", originalFileName, - directoryName + separator + simpleFileName) + directoryName + separator + simpleFileName) isFound = OperatingSystem.hasFile(fileName) Logging.trace("--: %r -> found = %r", fileName, isFound) @@ -520,16 +528,14 @@ def _readFile (cls, errorMessage = "" isOkay = True - originalFileName = fileName - fileName = cls._lookupFileName(originalFileName) - - if fileName is None: - errorMessage = "cannot find %r" % originalFileName + if not OperatingSystem.hasFile(fileName): + errorMessage = "cannot find %r" % fileName isOkay = False elif fileName in visitedFileNameSet: - Logging.trace("--: file already included %r", originalFileName) + Logging.trace("--: file already included %r", fileName) else: visitedFileNameSet.update(fileName) + referenceDirectoryName = OperatingSystem.dirname(fileName) with io.open(fileName, "rt", encoding="utf-8") as configurationFile: @@ -554,8 +560,10 @@ def _readFile (cls, if isDefinitionLine: cls._processDefinition(trimmedLine) elif isImportLine: - isOkay = cls._importFile(importedFileName, lineList, - visitedFileNameSet) + isOkay = \ + cls._importFile(referenceDirectoryName, + importedFileName, + lineList, visitedFileNameSet) if not isOkay: Logging.trace("--:import failed for %r in %r", @@ -605,7 +613,7 @@ def setSearchPaths (cls, """Sets list of search paths to .""" Logging.trace(">>: %r", searchPathList) - cls._searchPathList = ["."] + searchPathList + cls._searchPathList = searchPathList Logging.trace("<<") #-------------------- @@ -625,7 +633,9 @@ def read (cls, visitedFileNameSet = set() lineList = [] - isOkay = cls._readFile(fileName, lineList, visitedFileNameSet) + referenceDirectoryName = OperatingSystem.currentDirectoryPath() + isOkay = cls._importFile(referenceDirectoryName, fileName, + lineList, visitedFileNameSet) tokenList = ([] if not isOkay else cls._tokenize(lineList)) diff --git a/lilypondtobvc/src/basemodules/operatingsystem.py b/lilypondtobvc/src/basemodules/operatingsystem.py new file mode 100644 index 0000000..cffad42 --- /dev/null +++ b/lilypondtobvc/src/basemodules/operatingsystem.py @@ -0,0 +1,510 @@ +# operatingsystem -- provides simple facilities for access of operating +# system services +# +# author: Dr. Thomas Tensi, 2014 + +#==================== + +import os + +from basemodules.simplelogging import Logging +from basemodules.simpletypes import Boolean, String, StringList, Tuple +from basemodules.stringutil import newlineReplacedString, splitAt +from basemodules.typesupport import isString, toUnicodeString +from basemodules.ttbase import iif, isMicroPython, isStdPython + +if isStdPython: + import inspect + import os.path + import shutil + import sys + import subprocess + +# MicroPython constants +_MP_S_IFDIR = 0x4000 +_MP_S_IFREG = 0x8000 + +#==================== + +class OperatingSystem: + """Encapsulates access to operating system functions.""" + + pathSeparator = os.sep if isStdPython else "/" + + #-------------------- + # PRIVATE FEATURES + #-------------------- + + @classmethod + def _copyOrMoveFile (cls, + sourceFileName : String, + targetName : String, + isCopyOperation : Boolean, + targetDirectoryCreationIsForced : Boolean = False): + """Depending on either copies or moves file + with to either file or directory target + with ; if , + target directory is created when it does not exist""" + + Logging.trace(">>: %s %r -> %r", + iif(isCopyOperation, "copy", "move"), + sourceFileName, targetName) + + if isStdPython: + isOkay = True + + if cls.hasDirectory(targetName): + directoryName = targetName + else: + directoryName = cls.dirname(targetName) + + if not cls.hasDirectory(directoryName): + if targetDirectoryCreationIsForced: + cls.makeDirectory(directoryName) + else: + errorMessage = "cannot create directory %s" + cls.showMessageOnConsole(errorMessage % directoryName) + Logging.traceError(errorMessage, directoryName) + isOkay = False + + if isOkay: + path = shutil.copy2(sourceFileName, targetName) + isOkay = (path is not None) + + if isOkay and not isCopyOperation: + os.remove(sourceFileName) + + Logging.trace("<<") + + #-------------------- + # EXPORTED METHODS + #-------------------- + + @classmethod + def basename (cls, + fileName : String, + extensionIsShown : Boolean = True) -> String: + """Returns without leading path.""" + + standardSeparator = "/" + fileName = fileName.replace("\\", standardSeparator) + filePartList = fileName.split(standardSeparator) + shortFileName = filePartList[-1] + + if len(shortFileName) > 2 and shortFileName[1] == ":": + # remove windows drive indication + shortFileName = shortFileName[2:] + + if not extensionIsShown: + shortFileName, extension, _ = splitAt(shortFileName, ".") + + result = shortFileName + return result + + #-------------------- + + @classmethod + def copyFile (cls, + sourceFileName : String, + targetName : String, + targetDirectoryCreationIsForced : Boolean = False): + """Copies file with to either file or + directory target with ; if + , target directory is + created when it does not exist""" + + Logging.trace(">>: %r -> %r", sourceFileName, targetName) + cls._copyOrMoveFile(sourceFileName, targetName, True, + targetDirectoryCreationIsForced) + Logging.trace("<<") + + #-------------------- + + @classmethod + def currentDirectoryPath (cls) -> String: + """Returns current directory of program.""" + + Logging.trace(">>") + result = os.getcwd() + Logging.trace("<<: %r", result) + return result + + #-------------------- + + @classmethod + def dirname (cls, + filePath : String) -> String: + """Returns directory of .""" + + standardSeparator = "/" + filePath = filePath.replace("\\", standardSeparator) + filePartList = filePath.split(standardSeparator) + + if len(filePartList) > 1: + result = standardSeparator.join(filePartList[:-1]) + else: + result = filePartList[0] + + if len(result) > 2 and result[1] == ":": + result = result[:2] + else: + result = "" + + return result + + #-------------------- + + @classmethod + def executeCommand (cls, + command : StringList, + abortOnFailure : Boolean = False) -> Tuple: + """Processes (specified as list) in operating + system. When is set, any non-zero return + code aborts the program at once. Returns completion code + and string returned by command in stdout and stderr""" + + Logging.trace(">>: %r", command) + + if not isStdPython: + Logging.traceError("cannot run %s", command) + completionCode = 1 + loggingString = "" + else: + completedProcess = subprocess.run(command, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT) + + loggingString = completedProcess.stdout + loggingString = loggingString.decode() + completionCode = completedProcess.returncode + + if abortOnFailure and completionCode != 0: + message = ("ERROR: return code %d for %s" + % (completionCode, " ".join(command))) + Logging.trace("--: %s", message) + sys.exit("%s\n%s" % (loggingString, message)) + + result = (completionCode, loggingString) + + Logging.trace("<<: (%s, %r)", + completionCode, newlineReplacedString(loggingString)) + return result + + #-------------------- + + @classmethod + def fileNameList (cls, + directoryName : String, + plainFilesOnly : Boolean) -> StringList: + """Returns the list of files in ; if + is set, only plain files are returned, + otherwise only the names of the sub-directories""" + + Logging.trace(">>: directory = %s, plainFilesOnly = %s", + directoryName, plainFilesOnly) + + if isMicroPython: + hasPredicateProc = iif(plainFilesOnly, + lambda f: (f[1] % _MP_S_IFREG) != 0, + lambda f: (f[1] % _MP_S_IFDIR) != 0) + listDirProc = os.ilistdir + nameSelectionProc = lambda x: x[0] + else: + hasPredicateProc = iif(plainFilesOnly, + lambda f: f.is_file, + lambda f: f.is_dir) + listDirProc = os.scandir + nameSelectionProc = lambda x: x.name + + result = [ nameSelectionProc(fileEntry) + for fileEntry in listDirProc(directoryName) + if hasPredicateProc(fileEntry) ] + + Logging.trace("<<: %r", result) + return result + + #-------------------- + + @classmethod + def fullFilePath (cls, + fileName : String) -> String: + """Tells the full file path for """ + + Logging.trace(">>") + + if cls.isAbsoluteFileName(fileName): + result = fileName + else: + result = "%s/%s" % (cls.currentDirectoryPath(), fileName) + result = cls.normalizedFileName(result) + + Logging.trace("<<: %r", result) + return result + + #-------------------- + + @classmethod + def hasFile (cls, + fileName : String) -> Boolean: + """Tells whether signifies a file.""" + + checkProc = (os.path.isfile if isStdPython + else lambda st: os.stat(st)[0] & _MP_S_IFREG != 0) + return isString(fileName) and checkProc(fileName) + + #-------------------- + + @classmethod + def hasDirectory (cls, + directoryName : String) -> Boolean: + """Tells whether signifies a directory.""" + + checkProc = (os.path.isdir if isStdPython + else lambda st: os.stat(st)[0] & _MP_S_IFDIR != 0) + return isString(directoryName) and checkProc(directoryName) + + #-------------------- + + @classmethod + def homeDirectoryPath (cls) -> String: + """Returns home directory path.""" + + Logging.trace(">>") + + if isMicroPython: + result = "???" + else: + result = os.getenv("HOMEPATH") + result = result if result is not None else os.getenv("HOME") + result = result if result is not None else os.path.expanduser("~") + result = toUnicodeString(result) + + Logging.trace("<<: %r", result) + return result + + #-------------------- + + @classmethod + def isAbsoluteFileName (cls, + fileName : String) -> Boolean: + """Tells whether is absolute""" + + Logging.trace(">>") + result = (len(fileName) > 2 + and (fileName[1] == ":" + or fileName.startswith("\\") + or fileName.startswith("/"))) + Logging.trace("<<: %r", result) + return result + + #-------------------- + + @classmethod + def isWritableFile (cls, fileName) -> Boolean: + """Returns whether file named is writable. Opens it + for writing and hence clears its contents""" + + Logging.trace(">>: %r", fileName) + + directoryName = cls.dirname(fileName) + isOkay = cls.hasDirectory(directoryName) + + if isOkay and cls.hasFile(fileName): + try: + file = os.open(fileName, + os.O_APPEND | os.O_EXCL | os.O_RDWR) + except OSError: + isOkay = False + + if isOkay: + if not isPython2: + try: + # try the MSWindows file locking + import msvcrt + msvcrt.locking(file, msvcrt.LK_NBLCK, 1) + msvcrt.locking(file, msvcrt.LK_UNLCK, 1) + except (OSError, IOError): + isOkay = False + + os.close(file) + + Logging.trace("<<: %r", isOkay) + return isOkay + + #-------------------- + + @classmethod + def loggingDirectoryPath (cls) -> String: + """Returns logging directory path.""" + + Logging.trace(">>") + + if isMicroPython: + result = "???" + else: + result = os.getenv("LOGS") + result = (result if result is not None + else cls.tempDirectoryPath()) + result = toUnicodeString(result) + + Logging.trace("<<: %r", result) + return result + + #-------------------- + + @classmethod + def makeDirectory (cls, + directoryName : String): + """Creates directory named .""" + + Logging.trace(">>: %s", directoryName) + + if cls.hasDirectory(directoryName): + Logging.trace("--: directory already exists") + elif isMicroPython: + os.makedir(directoryName) + else: + os.makedirs(directoryName, exist_ok=True) + + Logging.trace("<<") + + #-------------------- + + @classmethod + def moveFile (cls, + sourceFileName : String, + targetName : String, + targetDirectoryCreationIsForced : Boolean = False): + """Moves file with to either file or + directory target with ; if + , target directory is + created when it does not exist""" + + Logging.trace(">>: %r -> %r", sourceFileName, targetName) + cls._copyOrMoveFile(sourceFileName, targetName, False, + targetDirectoryCreationIsForced) + Logging.trace("<<") + + #-------------------- + + @classmethod + def normalizedFileName (cls, + fileName : String) -> String: + """Resolves parent and current directory parts in and + returns normalized version""" + + Logging.trace(">>: %s", fileName) + + partList = fileName.replace("\\", "/").split("/") + result = partList[-1] + skipCount = 0 + + # sanitize path + for part in reversed(partList[:-1]): + if skipCount > 0: + skipCount -= 1 + elif part == ".": + pass + elif part == "..": + skipCount += 1 + else: + result = part + cls.pathSeparator + result + + Logging.trace("<<: %s", result) + return result + + #-------------------- + + @classmethod + def programIsAvailable (cls, + programName : String, + option : String) -> Boolean: + """Checks whether program with can be called.""" + + if not isStdPython: + result = False + else: + nullDevice = open(os.devnull, 'w') + + try: + callResult = subprocess.call([programName, option], + stdout=nullDevice) + except: + callResult = 1 + + result = (callResult == 0) + + return result + + #-------------------- + + @classmethod + def removeFile (cls, + fileName : String, + fileIsKept : Boolean = False): + """Removes file with permanently.""" + + Logging.trace(">>: %r", fileName) + + fileName = cls.normalizedFileName(fileName) + + if fileIsKept: + Logging.trace("--: not removed %r", fileName) + elif not cls.hasFile(fileName): + Logging.trace("--: file already nonexisting %r", fileName) + else: + Logging.trace("--: removing %r", fileName) + os.remove(fileName) + + Logging.trace("<<") + + #-------------------- + + @classmethod + def scriptFilePath (cls) -> String: + """Returns file path of calling script.""" + + Logging.trace(">>") + + if not isStdPython: + result = "???" + else: + result = os.path.abspath(inspect.stack()[1][1]) + + Logging.trace("<<: %r", result) + return result + + #-------------------- + + @classmethod + def showMessageOnConsole (cls, + message : String, + newlineIsAppended : Boolean = True): + """Shows on console (stderr) for giving a trace information + to user; tells whether a newline is added at + the end of the message""" + + Logging.trace("--: %r", message) + + if isStdPython: + st = message + ("\n" if newlineIsAppended else "") + sys.stderr.write(st) + sys.stderr.flush() + + #-------------------- + + @classmethod + def tempDirectoryPath (cls) -> String: + """Returns temporary directory path.""" + + Logging.trace(">>") + + if not isStdPython: + result = "???" + else: + result = os.getenv("TMP") + result = result if result is not None else os.getenv("TEMP") + result = toUnicodeString(result) + + Logging.trace("<<: %r", result) + return result diff --git a/lilypondtobvc/src/basemodules/physicalquantities.py b/lilypondtobvc/src/basemodules/physicalquantities.py new file mode 100644 index 0000000..d02a35f --- /dev/null +++ b/lilypondtobvc/src/basemodules/physicalquantities.py @@ -0,0 +1,26 @@ +# -*- coding:utf-8 -*- +# PhysicalQuantities - provides physical quantities in SI units +# +# author: Dr. Thomas Tensi, 2023-11 + +#==================== +# IMPORTS +#==================== + +from basemodules.simpletypes import Real, String + +#==================== + +Duration = Real # in seconds +Time = Real # in seconds + +#==================== + +def durationToString (d : Duration) -> String: + return "%ss" % d + +#-------------------- + +def timeToString (t : Time) -> String: + return "%ss" % t + diff --git a/lilypondtobvc/src/basemodules/regexppattern.py b/lilypondtobvc/src/basemodules/regexppattern.py new file mode 100644 index 0000000..cf50d75 --- /dev/null +++ b/lilypondtobvc/src/basemodules/regexppattern.py @@ -0,0 +1,229 @@ +# RegExpPattern -- services for construction of patterns for regular +# expressions +# +# author: Dr. Thomas Tensi, 2014 + +#==================== +# IMPORTS +#==================== + +import re + +from .simplelogging import Logging +from .simpletypes import Boolean, Object, Positive, String, Tuple +from .ttbase import iif + +#==================== + +class RegExpPattern: + """This module encapsulates pattern construction for regular + expressions. It helps for lists of elements and maps from one + element type to another. Those patterns may be nested, but + stay efficient by the use of (emulated) atomic groups.""" + + identifierPattern = r"[a-zA-Z]+" + integerPattern = r"\d+" + floatPattern = r"\d+(\.\d+)?" + + #-------------- + # LOCAL METHODS + #-------------- + + @classmethod + def _atomicGroupName (cls, + groupIndex : Positive) -> String: + """returns group name for atomic group with """ + + return "ATO%dMIC" % groupIndex + + #-------------------- + + @classmethod + def _makeAtomicPattern (cls, + pattern : String, + firstGroupIndex : Positive = 1) -> String: + """constructs an atomic group pattern from with groups + starting at """ + + pattern = cls._shiftGroups(pattern, firstGroupIndex + 1) + groupName = cls._atomicGroupName(firstGroupIndex) + result = r"(?=(?P<%s>%s))(?P=%s)" % (groupName, pattern, groupName) + return result + + #-------------------- + + @classmethod + def _makeListPattern (cls, + elementPattern : String, + firstGroupIndex : Positive = 1) -> String: + """Constructs a regexp pattern for list of elements with + with groups starting at ; + assumes that list starts immediately""" + + patternA, patternB = \ + cls._shiftGroupsForPair(elementPattern, elementPattern, + firstGroupIndex) + result = r"(?:%s(?:\s*,\s*%s)*)" % (patternA, patternB) + return result + + #-------------------- + + @classmethod + def _makeOptionalPattern (cls, + pattern : String) -> String: + """Constructs a regexp pattern for making it optional""" + + return "(?:%s|)" % pattern + + #-------------------- + + @classmethod + def _makeMapPattern (cls, + keyPattern : String, + valuePattern : String, + firstGroupIndex : Positive = 1) -> String: + """Constructs a regexp pattern for map of key-value-pairs with + for keys and for values with + atomic groups starting at ; assumes that + map starts immediately""" + + patternA, patternB = cls._shiftGroupsForPair(keyPattern, + valuePattern, + firstGroupIndex + 1) + + elementPattern = r"%s\s*:\s*%s" % (keyPattern, valuePattern) + listPattern = cls._makeListPattern(elementPattern) + pattern = r"\{\s*%s\s*\}" % cls._makeOptionalPattern(listPattern) + result = cls._makeAtomicPattern(pattern, firstGroupIndex) + return result + + #-------------------- + + @classmethod + def _scanForGroups (cls, + pattern : String) -> Tuple: + """looks for atomic groups in and returns their + first and last group index; assumes a consecutive + numbering""" + + firstIndex = 0 + lastIndex = -1 + + for i in range(1, 1000): + if firstIndex > 0 and cls._atomicGroupName(i) not in pattern: + break + else: + lastIndex = i + firstIndex = iif(firstIndex == 0, i, firstIndex) + + return (firstIndex, lastIndex) + + #-------------------- + + @classmethod + def _shiftGroups (cls, + pattern : String, + firstGroupIndex : Positive) -> String: + """looks for atomic groups in and shifts them from + current first position to """ + + patternFirstIndex, patternLastIndex = cls._scanForGroups(pattern) + numberSequence = list(range(patternFirstIndex, patternLastIndex + 1)) + offset = firstGroupIndex - patternFirstIndex + result = pattern + + if offset > 0: + numberSequence.reverse() + + for i in numberSequence: + oldGroupName = cls._atomicGroupName(i) + newGroupName = cls._atomicGroupName(i + offset) + result = result.replace(oldGroupName, newGroupName) + + return result + + #-------------------- + + @classmethod + def _shiftGroupsForPair (cls, + patternA : String, + patternB : String, + firstGroupIndex : Positive) -> String: + """looks for atomic groups in and and + shifts them from their current start index to + for first, and to following indices for + second""" + + resultA = cls._shiftGroups(patternA, firstGroupIndex) + _, lastGroupIndex = cls._scanForGroups(resultA) + resultB = cls._shiftGroups(patternB, lastGroupIndex + 1) + return (resultA, resultB) + + #-------------------- + # EXPORTED METHODS + #-------------------- + + @classmethod + def makeCompactListPattern (cls, + elementPattern : String, + separator : String = "/") -> String: + """Constructs string pattern for a compact list (without + spaces) from and for elements + within list""" + + Logging.trace(">>: elementPattern = %r", elementPattern) + result = (r"%s(%s%s)*" % (elementPattern, separator, elementPattern)) + Logging.trace("<<: %r", result) + return result + + #-------------------- + + @classmethod + def makeListPattern (cls, + elementPattern : String, + mayBeEmpty : Boolean = True) -> String: + """Constructs string pattern for list from + for elements within list; assumes that first list element + starts immediately; if is set, also allows + empty lists""" + + Logging.trace(">>: elementPattern = %r, mayBeEmpty = %r", + elementPattern, mayBeEmpty) + + listPattern = cls._makeListPattern(elementPattern) + result = iif(not mayBeEmpty, listPattern, "(?:%s|)" % listPattern) + + Logging.trace("<<: %r", result) + return result + + #-------------------- + + @classmethod + def makeMapPattern (cls, + keyPattern : String, + valuePattern : String, + mayBeEmpty : Boolean = True) -> String: + """Constructs string pattern for map from for + keys and for values; assumes that map starts + immediately; if is set, also allows + empty maps""" + + Logging.trace(">>: keyPattern = %r, valuePattern = %r," + + " mayBeEmpty = %r", + keyPattern, valuePattern, mayBeEmpty) + + mapPattern = cls._makeMapPattern(keyPattern, valuePattern) + result = iif(not mayBeEmpty, mapPattern, "(?:%s|)" % mapPattern) + + Logging.trace("<<: %r", result) + return result + + #-------------------- + + @classmethod + def makeRegExp (cls, + pattern : String) -> Object: + """Constructs regular expression from with leading + whitespace and trailing end-of-string""" + + return re.compile("^\s*%s\s*$" % pattern) diff --git a/lilypondtobvc/src/basemodules/simpleassertion.py b/lilypondtobvc/src/basemodules/simpleassertion.py new file mode 100644 index 0000000..3bbccaa --- /dev/null +++ b/lilypondtobvc/src/basemodules/simpleassertion.py @@ -0,0 +1,76 @@ +# assertion - provide simple assertion checking +# +# author: Dr. Thomas Tensi, 2014 + +#==================== + +import sys + +from basemodules.simplelogging import Logging +from basemodules.simpletypes import Boolean, String +from basemodules.ttbase import isStdPython + +if isStdPython: + import os.path + +#==================== + +class Assertion: + """Provides some primitive assertion handling of + pre-/postconditions and checking conditions.""" + + isActive = True + + #-------------------- + # LOCAL FEATURES + #-------------------- + + @classmethod + def _internalCheck (cls, + condition : Boolean, + checkKind : String, + errorMessage : String): + """Checks whether holds, otherwise exits with + containing .""" + + if cls.isActive: + if not condition: + Logging.traceError("%s FAILED - %s", checkKind, errorMessage) + programName = ("PROGRAM" if not isStdPython + else os.path.basename(sys.argv[0])) + sys.exit(programName + ": ERROR - " + errorMessage) + + #-------------------- + # EXPORTED FEATURES + #-------------------- + + @classmethod + def check (cls, + condition : Boolean, + errorMessage : String): + """Checks within function with and + exits with on failure.""" + + cls._internalCheck(condition, "CHECK", errorMessage) + + #-------------------- + + @classmethod + def post (cls, + condition : Boolean, + errorMessage : String): + """Checks postcondition within function with + and exits with on failure.""" + + cls._internalCheck(condition, "POSTCONDITION", errorMessage) + + #-------------------- + + @classmethod + def pre (cls, + condition : Boolean, + errorMessage : String): + """Checks precondition and exits with + on failure.""" + + cls._internalCheck(condition, "PRECONDITION", errorMessage) diff --git a/lilypondtobvc/src/basemodules/simplelogging.py b/lilypondtobvc/src/basemodules/simplelogging.py new file mode 100644 index 0000000..ac33507 --- /dev/null +++ b/lilypondtobvc/src/basemodules/simplelogging.py @@ -0,0 +1,426 @@ +# -*- coding: utf-8 -*- +# logging - provides primitive logging with logging levels +# +# author: Dr. Thomas Tensi, 2014-04 + +#==================== + +import sys +import time + +from basemodules.simpletypes import Boolean, Natural, String +from basemodules.ttbase import adaptToRange, iif, isMicroPython +from basemodules.typesupport import toUnicodeString + +if not isMicroPython: + import atexit + import io + +#==================== + +class Logging_Level: + """Defines the levels of logging""" + + # no logging + noLogging = 0 + # only logging of errors and assertion failures + error = 1 + # logging of errors and abbreviated exit-entry traces + standardAbbreviated = 2 + # logging of errors and full exit-entry traces + standard = 3 + # full logging (including internal traces) + verbose = 4 + + #-------------------- + + @classmethod + def fromString (cls, + st : String) -> String: + """Converts string to log level, returns noLogging when + string cannot be converted""" + + result = cls.noLogging + st = st.lower() + + if st == "verbose": + result = cls.verbose + elif st == "standard": + result = cls.standard + elif st == "standardabbreviated": + result = cls.standardAbbreviated + elif st == "error": + result = cls.error + + return result + +#==================== + +class Logging: + """Provides some primitive logging.""" + + + _referenceLevel = Logging_Level.noLogging + _fileName = "" + _fileIsOpen = None + _fileIsKeptOpen = None + _file = None + _isEnabled = True + _timeIsLogged = True + _timeFactor = 1 + _timeFractionalPartTemplate = "" + _timeFractionalDigitCount = 0 + _previousTimestamp = 0 + _timeOfDayString = "" + + # buffer logs data before log file is opened, otherwise a + # write-through will be done + _buffer = [] + + # the list of function names to be ignored when traversing + # run-time stack for relevant function names */ + _ignoredFunctionNameList = \ + ("check", "_internalCheck", "log", "post", "pre", + "trace", "traceError", "_traceWithLevel") + + #-- TRACE PREFICES -- + # length of allowed prefixes for template in a trace call + _tracePrefixLength = 2 + + # trace prefix used for internal traces within a function + _innerTracePrefix = "--" + + # list of allowed prefixes for template in a entry-exit trace + # call + _entryExitPrefixList = (">>", "<<") + + # list of allowed prefixes for template in a trace call + _standardPrefixList = _entryExitPrefixList + (_innerTracePrefix,) + + # -------------------- + # LOCAL FEATURES + # -------------------- + + @classmethod + def _callingFunctionName (cls) -> String: + """Returns function name of calling function. Some functions + are filtered out (like those from UI) and the class name is + prepended.""" + + if isMicroPython: + return "..." + else: + callerDepth = 1 + found = False + + while not found: + currentFrame = sys._getframe(callerDepth) + functionName = currentFrame.f_code.co_name + found = (functionName not in cls._ignoredFunctionNameList) + + if not found: + callerDepth = callerDepth + 1 + else: + # check whether this is a method in a class using + # python conventions + localVariableList = currentFrame.f_locals + hasSelfVariable = ("self" in localVariableList) + hasClsVariable = ("cls" in localVariableList) + + if hasSelfVariable: + variable = localVariableList["self"] + className = variable.__class__.__name__ + elif hasClsVariable: + className = localVariableList["cls"].__name__ + else: + className = "" + + functionName = (className + iif(className > "", ".", "") + + functionName) + + return functionName + + #-------------------- + + @classmethod + def _closeFileConditionally (cls): + if cls._fileIsOpen: + cls._file.close() + + #-------------------- + + @classmethod + def _currentTimeOfDay (cls) -> String: + """Returns current time of day in seconds as string""" + + currentTimestamp = time.time() + + if currentTimestamp != cls._previousTimestamp: + cls._previousTimestamp = currentTimestamp + + if not isMicroPython: + st = time.strftime("%H%M%S") + else: + _, _, _, hours, minutes, seconds, _, _ = time.localtime() + st = "%02d%02d%02d" % (hours, minutes, seconds) + + if cls._timeFractionalDigitCount > 0: + fractionalPart = currentTimestamp - int(currentTimestamp) + fractionalPart *= cls._timeFactor + st += cls._timeFractionalPartTemplate % fractionalPart + + cls._timeOfDayString = st + + return cls._timeOfDayString + + #-------------------- + + @classmethod + def _openOrCreateFile (cls, + isNew : Boolean): + """Creates or reopens logging file depending on value of + """ + + if cls._fileName == "": + cls._file = None + elif cls._fileName.lower() == "stderr": + cls._file = sys.stderr + else: + mode = iif(isNew, "wt", "at") + cls._file = io.open(cls._fileName, mode, + encoding="utf-8", errors='replace') + + cls._fileIsOpen = (cls._file is not None) + + #-------------------- + + @classmethod + def _prefixBefore (cls, st, otherSt): + """Returns part of before ; if is not + in string, the whole string will be returned""" + + splitPosition = st.indexOf(otherSt) + result = st if splitPosition is None else st[0:splitPosition] + return result + + #-------------------- + + @classmethod + def _traceWithLevel (cls, + level : Natural, + template : String, + *argumentList): + """Writes formatted by