added refs RAM cache

subsync/cache.py

@@ -0,0 +1,26 @@
+from subsync.settings import settings
+
+
+class WordsCache(object):
+    def __init__(self):
+        self.init(None)
+
+    def mkId(self, stream):
+        if stream:
+            return (stream.path, stream.no, settings().minWordProb, settings().minWordLen)
+
+    def init(self, id):
+        self.id = self.mkId(id)
+        self.data = []
+        self.progress = []
+
+    def isValid(self, id):
+        return self.mkId(id) == self.id
+
+    def isEmpty(self):
+        return not (self.id or self.data or self.progress)
+
+    def clear(self):
+        self.id = None
+        self.data = []
+        self.progress = None

subsync/gui/layout/settingswin.py

@@ -81,6 +81,20 @@ def __init__( self, parent ):
 		self.m_askForUpdate.SetValue(True) 
 		fgSizer4.Add( self.m_askForUpdate, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL|wx.EXPAND, 5 )
 
+		self.staticText5 = wx.StaticText( self.m_panelGeneral, wx.ID_ANY, _(u"Cache:"), wx.DefaultPosition, wx.DefaultSize, 0 )
+		self.staticText5.Wrap( -1 )
+		fgSizer4.Add( self.staticText5, 0, wx.ALL|wx.ALIGN_RIGHT|wx.ALIGN_CENTER_VERTICAL, 5 )
+
+		self.m_refsCache = wx.CheckBox( self.m_panelGeneral, wx.ID_ANY, _(u"use RAM cache for references"), wx.DefaultPosition, wx.DefaultSize, 0 )
+		self.m_refsCache.SetValue(True) 
+		fgSizer4.Add( self.m_refsCache, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL|wx.EXPAND, 5 )
+
+
+		fgSizer4.Add( ( 0, 0), 1, wx.EXPAND, 5 )
+
+		self.m_buttonClearCache = wx.Button( self.m_panelGeneral, wx.ID_ANY, _(u"Clear cache"), wx.DefaultPosition, wx.DefaultSize, 0 )
+		fgSizer4.Add( self.m_buttonClearCache, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL, 5 )
+
 
 		self.m_panelGeneral.SetSizer( fgSizer4 )
 		self.m_panelGeneral.Layout()
@@ -256,6 +270,7 @@ def __init__( self, parent ):
 		self.Centre( wx.BOTH )
 
 		# Connect Events
+		self.m_buttonClearCache.Bind( wx.EVT_BUTTON, self.onButtonClearCache )
 		self.m_checkAutoJobsNo.Bind( wx.EVT_CHECKBOX, self.onCheckAutoJobsNoCheck )
 		self.m_checkLogToFile.Bind( wx.EVT_CHECKBOX, self.onCheckLogToFileCheck )
 		self.m_buttonLogFileSelect.Bind( wx.EVT_BUTTON, self.onButtonLogFileSelectClick )
@@ -266,6 +281,9 @@ def __del__( self ):
 
 
 	# Virtual event handlers, overide them in your derived class
+	def onButtonClearCache( self, event ):
+		event.Skip()
+
 	def onCheckAutoJobsNoCheck( self, event ):
 		event.Skip()
 

subsync/gui/mainwin.py

@@ -6,6 +6,7 @@
 from subsync.gui.aboutwin import AboutWin
 from subsync.gui.errorwin import error_dlg
 from subsync import assets
+from subsync import cache
 from subsync import img
 from subsync import thread
 from subsync import config
@@ -64,6 +65,8 @@ def __init__(self, parent, subs=None, refs=None):
         self.Fit()
         self.Layout()
 
+        self.refsCache = cache.WordsCache()
+
         self.selfUpdater = None
         assets.init(self.assetsUpdated)
 
@@ -81,13 +84,16 @@ def onButtonMenuClick(self, event):
         self.PopupMenu(self.m_menu)
 
     def onMenuItemSettingsClick(self, event):
-        with SettingsWin(self, settings()) as dlg:
+        with SettingsWin(self, settings(), self.refsCache) as dlg:
             if dlg.ShowModal() == wx.ID_OK:
                 newSettings = dlg.getSettings()
                 if settings() != newSettings:
                     self.changeSettings(newSettings)
 
     def changeSettings(self, newSettings):
+        if not settings().refsCache:
+            self.refsCache.clear()
+
         if settings().logLevel != newSettings.logLevel:
             loggercfg.setLevel(newSettings.logLevel)
 
@@ -130,8 +136,12 @@ def onButtonCloseClick(self, event):
 
     @error_dlg
     def onButtonStartClick(self, event):
-        settings().save()
-        self.start()
+        try:
+            settings().save()
+            self.start()
+        except:
+            self.refsCache.clear()
+            raise
 
     def start(self, listener=None):
         self.validateSelection()
@@ -140,7 +150,9 @@ def start(self, listener=None):
         if self.validateAssets():
             sub = self.m_panelSub.stream
             ref = self.m_panelRef.stream
-            with SyncWin(self, sub, ref, listener) as dlg:
+            cache = self.refsCache if settings().refsCache else None
+
+            with SyncWin(self, sub, ref, cache, listener) as dlg:
                 dlg.ShowModal()
 
             if listener:

subsync/gui/settingswin.py

@@ -7,7 +7,7 @@
 
 
 class SettingsWin(subsync.gui.layout.settingswin.SettingsWin):
-    def __init__(self, parent, settings):
+    def __init__(self, parent, settings, cache=None):
         super().__init__(parent)
         self.m_outputCharEnc.SetString(0, _('same as input subtitles'))
 
@@ -18,6 +18,9 @@ def __init__(self, parent, settings):
 
         self.setSettings(settings)
 
+        self.cache = cache
+        self.m_buttonClearCache.Enable(cache and not cache.isEmpty())
+
     def setSettings(self, settings):
         self.settings = Settings(settings)
 
@@ -100,3 +103,8 @@ def onButtonRestoreDefaultsClick(self, event):
         if dlg.ShowModal() == wx.ID_YES:
             self.setSettings(Settings())
 
+    def onButtonClearCache(self, event):
+        if self.cache:
+            self.cache.clear()
+        self.m_buttonClearCache.Disable()
+

subsync/gui/syncwin.py

@@ -19,7 +19,7 @@
 
 
 class SyncWin(subsync.gui.layout.syncwin.SyncWin):
-    def __init__(self, parent, subs, refs, listener=None):
+    def __init__(self, parent, subs, refs, refsCache=None, listener=None):
         super().__init__(parent)
 
         self.m_buttonDebugMenu.SetLabel(u'\u22ee') # 2630
@@ -49,7 +49,7 @@ def __init__(self, parent, subs, refs, listener=None):
         self.Bind(wx.EVT_TIMER, self.onUpdateTimerTick, self.updateTimer)
 
         with busydlg.BusyDlg(_('Loading, please wait...')):
-            self.sync = synchro.Synchronizer(self, self.subs, self.refs)
+            self.sync = synchro.Synchronizer(self, self.subs, self.refs, refsCache)
             self.sync.start()
 
         self.isRunning = True

subsync/pipeline.py

@@ -17,14 +17,22 @@ def __init__(self, stream):
         self.duration = self.demux.getDuration()
         self.timeWindow = [0, self.duration]
 
+        self.done = False
+
     def destroy(self):
         self.extractor.connectEosCallback(None)
         self.extractor.connectErrorCallback(None)
         self.extractor.stop()
 
-    def selectTimeWindow(self, *window):
-        self.timeWindow = window
-        self.extractor.selectTimeWindow(*window)
+    def selectTimeWindow(self, begin, end, start=None):
+        if end > self.duration:
+            end = self.duration
+        if begin > end:
+            begin = end
+        self.timeWindow = (begin, end)
+        if start == None:
+            start = begin
+        self.extractor.selectTimeWindow(start, end)
 
     def start(self, threadName=None):
         self.extractor.start(threadName=threadName)
@@ -40,14 +48,21 @@ def getProgress(self):
             pos = self.demux.getPosition()
             if math.isclose(pos, 0.0):
                 pos = self.timeWindow[0]
-            return (pos - self.timeWindow[0]) / (self.timeWindow[1] - self.timeWindow[0])
+            if self.timeWindow[1] > self.timeWindow[0]:
+                return (pos - self.timeWindow[0]) / (self.timeWindow[1] - self.timeWindow[0])
 
-    def connectEosCallback(self, cb, dst=None):
-        self.extractor.connectEosCallback(cb)
+    def getPosition(self):
+        return max(self.timeWindow[0], min(self.demux.getPosition(), self.timeWindow[1]))
 
-    def connectErrorCallback(self, cb, dst=None):
-        self.extractor.connectErrorCallback(cb)
+    def connectEosCallback(self, cb):
+        def eos(*args, **kw):
+            self.done = True
+            cb()
 
+        self.extractor.connectEosCallback(eos if cb else None)
+
+    def connectErrorCallback(self, cb):
+        self.extractor.connectErrorCallback(cb)
 
 class SubtitlePipeline(BasePipeline):
     def __init__(self, stream):
@@ -127,18 +142,30 @@ def createProducerPipeline(stream):
         raise Error(_('Not supported stream type'), type=stream.type)
 
 
-def createProducerPipelines(stream, no):
+def createProducerPipelines(stream, no=None, timeWindows=None):
+    if timeWindows != None:
+        no = len(timeWindows)
+
     pipes = []
     for i in range(no):
         p = createProducerPipeline(stream)
         pipes.append(p)
 
         if p.duration:
-            partTime = p.duration / no
-            begin = i * partTime
-            end = begin + partTime
+            if timeWindows != None:
+                start, begin, end = timeWindows[i]
+
+            else:
+                partTime = p.duration / no
+                begin = i * partTime
+                end = begin + partTime
+                start = None
+
             logger.info('job %i/%i time window set to %.2f - %.2f', i+1, no, begin, end)
-            p.selectTimeWindow(begin, end)
+            if start != None:
+                logger.info('job %i/%i starting position set to %.2f', i+1, no, start)
+
+            p.selectTimeWindow(begin, end, start)
 
         else:
             logger.warn('cannot get duration - using single pipeline')

subsync/settings.py

@@ -21,6 +21,7 @@ def __init__(self, settings=None, **kw):
         self.minCorrelation = 0.9999
         self.minWordsSim = 0.6
         self.lastdir = ''
+        self.refsCache = True
         self.autoUpdate = True
         self.askForUpdate = True
         self.debugOptions = False

subsync/synchro.py

@@ -4,6 +4,7 @@
 from subsync.settings import settings
 from subsync import dictionary
 from subsync import encdetect
+from subsync import utils
 import threading
 import multiprocessing
 
@@ -20,10 +21,11 @@ def getJobsNo():
 
 
 class Synchronizer(object):
-    def __init__(self, listener, subs, refs):
+    def __init__(self, listener, subs, refs, refsCache=None):
         self.listener = listener
         self.subs = subs
         self.refs = refs
+        self.refsCache = refsCache
 
         self.fps = refs.stream().frameRate
         if self.fps == None:
@@ -60,14 +62,33 @@ def __init__(self, listener, subs, refs):
         else:
             self.subPipeline.connectWordsCallback(self.correlator.pushSubWord)
 
-        self.refPipelines = pipeline.createProducerPipelines(refs, getJobsNo())
+        if refsCache and refsCache.isValid(self.refs):
+            logger.info('restoring cached reference words (%i)', len(refsCache.data))
+
+            for word, time in refsCache.data:
+                self.correlator.pushRefWord(word, time)
+
+            self.refPipelines = pipeline.createProducerPipelines(refs, timeWindows=refsCache.progress)
+
+        else:
+            if refsCache:
+                refsCache.init(refs)
+
+            self.refPipelines = pipeline.createProducerPipelines(refs, no=getJobsNo())
+
         for p in self.refPipelines:
             p.connectEosCallback(self.onRefEos)
             p.connectErrorCallback(self.onRefError)
-            p.connectWordsCallback(self.correlator.pushRefWord)
+            p.connectWordsCallback(self.onRefWord)
 
         self.pipelines = [ self.subPipeline ] + self.refPipelines
 
+    def onRefWord(self, word, time):
+        self.correlator.pushRefWord(word, time)
+
+        if self.refsCache and self.refsCache.id:
+            self.refsCache.data.append((word, time))
+
     def destroy(self):
         self.stop()
         self.correlator.connectStatsCallback(None)
@@ -90,6 +111,11 @@ def stop(self):
         for p in self.pipelines:
             p.stop()
 
+        if self.refsCache and self.refsCache.id:
+            self.refsCache.progress = [ (p.getPosition(), *p.timeWindow)
+                    for p in self.refPipelines
+                    if not p.done and p.getPosition() < p.timeWindow[1] ]
+
     def isRunning(self):
         if self.correlator.isRunning() and not self.correlator.isDone():
             return True