crabserver3-setup.patch

diff --git a/setup_dependencies.py b/setup_dependencies.py
index a4fb83b..5cf507d 100644
--- a/setup_dependencies.py
+++ b/setup_dependencies.py
@@ -117,7 +117,10 @@ dependencies = {'wmc-rest':{
                         'statics': ['src/couchapps/Agent+'],
                         },
                 'crabcache':{
+                        'packages': ['WMCore.Wrappers+', 'WMCore.Services.UserFileCache+'],
                         'systems': ['wmc-rest'],
+                        'modules': ['WMCore.Services.Requests', 'WMCore.Services.Service', 'WMCore.Services.pycurl_manager',
+                                    'WMCore.Services.EmulatorSwitch'],
                         },
                 'crabserver':{
                         'packages': ['WMCore.Credential', 'WMCore.Services+', 'WMCore.RequestManager+',
@@ -126,7 +129,8 @@ dependencies = {'wmc-rest':{
                         'systems' : ['wmc-rest', 'wmc-database'],
                         },
                 'crabclient':{
-                        'packages': ['WMCore.Wrappers+', 'WMCore.Credential', 'PSetTweaks', 'WMCore.Services.UserFileCache+', 'WMCore.Services.SiteDB+', 'WMCore.Services.PhEDEx+'],
+                        'packages': ['WMCore.Wrappers+', 'WMCore.Credential', 'PSetTweaks', 'WMCore.Services.UserFileCache+',
+                                     'WMCore.Services.SiteDB+', 'WMCore.Services.PhEDEx+'],
                         'systems': ['wmc-base'],
                         'modules': ['WMCore.FwkJobReport.FileInfo', 'WMCore.Services.Requests', 'WMCore.DataStructs.LumiList',
                                     'WMCore.Services.Service', 'WMCore.Services.pycurl_manager', 'WMCore.Services.EmulatorSwitch'],
diff --git a/src/python/WMCore/JobSplitting/EventAwareLumiBased.py b/src/python/WMCore/JobSplitting/EventAwareLumiBased.py
index 097cc94..8170dd6 100644
--- a/src/python/WMCore/JobSplitting/EventAwareLumiBased.py
+++ b/src/python/WMCore/JobSplitting/EventAwareLumiBased.py
@@ -19,7 +19,7 @@ import math
 
 from WMCore.DataStructs.Run         import Run
 from WMCore.JobSplitting.JobFactory import JobFactory
-from WMCore.JobSplitting.LumiBased  import isGoodLumi, isGoodRun
+from WMCore.JobSplitting.LumiBased  import isGoodLumi, isGoodRun, LumiChecker
 from WMCore.WMBS.File               import File
 from WMCore.WMSpec.WMTask           import buildLumiMask
 
@@ -50,6 +50,8 @@ class EventAwareLumiBased(JobFactory):
         runWhitelist    = kwargs.get('runWhitelist', [])
         runs            = kwargs.get('runs', None)
         lumis           = kwargs.get('lumis', None)
+        applyLumiCorrection = bool(kwargs.get('applyLumiCorrection', False))
+
         timePerEvent, sizePerEvent, memoryRequirement = \
                     self.getPerformanceParameters(kwargs.get('performance', {}))
         capJobTime      = kwargs.get('capJobTime', None)
@@ -144,6 +146,7 @@ class EventAwareLumiBased(JobFactory):
         totalAvgEventCount = 0
         currentJobAvgEventCount = 0
         stopTask = False
+        self.lumiChecker = LumiChecker(applyLumiCorrection)
         for location in locationDict:
 
             # For each location, we need a new jobGroup
@@ -204,7 +207,8 @@ class EventAwareLumiBased(JobFactory):
 
                     # Now loop over the lumis
                     for lumi in run:
-                        if not isGoodLumi(goodRunList, run = run.run, lumi = lumi):
+                        if (not isGoodLumi(goodRunList, run = run.run, lumi = lumi) or
+                            self.lumiChecker.isSplitLumi(run.run, lumi, f)):
                             # Kill the chain of good lumis
                             # Skip this lumi
                             if firstLumi != None and firstLumi != lumi:
@@ -256,6 +260,7 @@ class EventAwareLumiBased(JobFactory):
                                 msg = "File %s has too many events (%d) in %d lumi(s)" % (f['lfn'],
                                                                                           f['events'],
                                                                                           f['lumiCount'])
+                            self.lumiChecker.closeJob(self.currentJob)
                             self.newJob(name = self.getJobName(), failedJob = failNextJob,
                                         failedReason = msg)
                             if deterministicPileup:
@@ -321,4 +326,6 @@ class EventAwareLumiBased(JobFactory):
             if stopTask:
                 break
 
+        self.lumiChecker.closeJob(self.currentJob)
+        self.lumiChecker.fixInputFiles()
         return
diff --git a/src/python/WMCore/JobSplitting/FileBased.py b/src/python/WMCore/JobSplitting/FileBased.py
index df7abb8..177546d 100644
--- a/src/python/WMCore/JobSplitting/FileBased.py
+++ b/src/python/WMCore/JobSplitting/FileBased.py
@@ -8,6 +8,8 @@ a set of jobs based on file boundaries
 
 from WMCore.JobSplitting.JobFactory import JobFactory
 from WMCore.WMBS.File import File
+from WMCore.WMSpec.WMTask import buildLumiMask
+from WMCore.JobSplitting.LumiBased import isGoodRun, isGoodLumi
 
 
 class FileBased(JobFactory):
@@ -24,39 +26,79 @@ class FileBased(JobFactory):
         filesPerJob   = int(kwargs.get("files_per_job", 10))
         jobsPerGroup  = int(kwargs.get("jobs_per_group", 0))
         totalFiles    = int(kwargs.get("total_files", 0))
+        runs          = kwargs.get('runs', None)
+        lumis         = kwargs.get('lumis', None)
         runBoundaries = kwargs.get("respect_run_boundaries", False)
         getParents    = kwargs.get("include_parents", False)
         filesInJob    = 0
-        listOfFiles   = []
         timePerEvent, sizePerEvent, memoryRequirement = \
                     self.getPerformanceParameters(kwargs.get('performance', {}))
 
+        goodRunList = {}
+        if runs and lumis:
+            goodRunList = buildLumiMask(runs, lumis)
+
         #Get a dictionary of sites, files
-        locationDict = self.sortByLocation()
+        lDict = self.sortByLocation()
+        locationDict = {}
+
+        for key in lDict:
+            newlist = []
+            for f in lDict[key]:
+                if runs and lumis:
+                    ## Skip this file is it has no runs.
+                    if len(f['runs']) == 0:
+                        continue
+                    f['lumiCount'] = 0
+                    f['runs'] = sorted(f['runs'])
+                    for run in f['runs']:
+                        run.lumis.sort()
+                        f['lumiCount'] += len(run.lumis)
+                    f['lowestRun'] = f['runs'][0]
+                    ## Skip this file is it has no lumis.
+                    if f['lumiCount'] == 0:
+                        continue
+                    ## Do average event per lumi calculation.
+                    f['avgEvtsPerLumi'] = round(float(f['events']) / f['lumiCount'])
+                newlist.append(f)
+                locationDict[key] = sorted(newlist, key = lambda f: f['lfn'])
 
-        ## Make a list with all the files in the locationDict.
+        ## Make a list with all the files, sorting them by LFN. Remove from the list all
+        ## the files filtered out by the lumi-mask (if there is one).
         files = []
         for filesPerLocSet in locationDict.values():
-            for file in filesPerLocSet:
-                files.append(file)
-        ## Here we can apply a lumi-mask and remove files 
-        ## that are left with 0 lumis to process.
-        ## Sort the list of files by LFN.
-        if len(files) != 0:
+            for f in filesPerLocSet:
+                files.append(f)
+        if len(files):
             files = sorted(files, key = lambda f: f['lfn'])
-        ## Keep only the first totalFiles files and remove
-        ## the other files from the locationDict.
+            if runs and lumis:
+                skippedFiles = []
+                for f in files:
+                    skipFile = True
+                    for run in f['runs']:
+                        if not isGoodRun(goodRunList, run.run):
+                            continue
+                        for lumi in run:
+                            if not isGoodLumi(goodRunList, run.run, lumi):
+                                continue
+                            skipFile = False
+                    if skipFile:
+                        skippedFiles.append(f)
+                for f in skippedFiles:
+                    files.remove(f)
+            
+        ## Keep only the first totalFiles files. Remove the other files from the locationDict.
         if totalFiles > 0 and totalFiles < len(files):
             removedFiles = files[totalFiles:]
             files = files[:totalFiles]
-            for file in removedFiles:
+            for f in removedFiles:
                 for locSet in locationDict.keys():
-                    if file in locationDict[locSet]:
-                        locationDict[locSet].remove(file)
+                    if f in locationDict[locSet]:
+                        locationDict[locSet].remove(f)
 
         for locSet in locationDict.keys():
             #Now we have all the files in a certain location set
-            fileList    = locationDict[locSet]
+            fileList = locationDict[locSet]
             filesInJob  = 0
             jobsInGroup = 0
             self.newGroup()
@@ -70,25 +112,76 @@ class FileBased(JobFactory):
                         parent = File(lfn = lfn)
                         f['parents'].add(parent)
                 fileRun = f.get('minrun', None)
+                createNewJob = False
                 if filesInJob == 0 or filesInJob == filesPerJob or (runBoundaries and fileRun != jobRun):
-                    if jobsPerGroup:
-                        if jobsInGroup > jobsPerGroup:
-                            self.newGroup()
-                            jobsInGroup = 0
-
-                    self.newJob(name = self.getJobName())
-                    self.currentJob.addResourceEstimates(memory = memoryRequirement)
-
-                    filesInJob   = 0
-                    jobsInGroup += 1
-                    jobRun       = fileRun
-
-                self.currentJob.addFile(f)
-                filesInJob += 1
-                fileTime = f['events'] * timePerEvent
-                fileSize = f['events'] * sizePerEvent
-                self.currentJob.addResourceEstimates(jobTime = fileTime,
-                                                     disk = fileSize)
-                listOfFiles.append(f)
+                    createNewJob = True
+                if runs and lumis:
+                    for run in f['runs']:
+                        if not isGoodRun(goodRunList, run.run):
+                            continue
+                        firstLumi = None
+                        lastLumi = None
+                        for lumi in run:
+                            if not isGoodLumi(goodRunList, run.run, lumi): 
+                                if firstLumi != None and lastLumi != None:
+                                    self.currentJob['mask'].addRunAndLumis(run = run.run, lumis = [firstLumi, lastLumi])
+                                    addedEvents = ((lastLumi - firstLumi + 1) * f['avgEvtsPerLumi'])
+                                    runAddedTime = addedEvents * timePerEvent
+                                    runAddedSize = addedEvents * sizePerEvent
+                                    self.currentJob.addResourceEstimates(jobTime = runAddedTime, disk = runAddedSize)
+                                    firstLumi = None
+                                    lastLumi = None
+                                continue
+                            if lastLumi != None and lumi != lastLumi + 1:
+                                self.currentJob['mask'].addRunAndLumis(run = run.run, lumis = [firstLumi, lastLumi])
+                                addedEvents = ((lastLumi - firstLumi + 1) * f['avgEvtsPerLumi'])
+                                runAddedTime = addedEvents * timePerEvent
+                                runAddedSize = addedEvents * sizePerEvent
+                                self.currentJob.addResourceEstimates(jobTime = runAddedTime, disk = runAddedSize)
+                                firstLumi = None
+                                lastLumi = None
+                            if createNewJob:
+                                if jobsPerGroup:
+                                    if jobsInGroup > jobsPerGroup:
+                                        self.newGroup()
+                                        jobsInGroup = 0
+                                self.newJob(name = self.getJobName())
+                                self.currentJob.addResourceEstimates(memory = memoryRequirement)
+                                filesInJob = 0
+                                jobsInGroup += 1
+                                jobRun = fileRun
+                                createNewJob = False
+                                self.currentJob.addFile(f)
+                                filesInJob += 1
+                            if firstLumi == None:
+                                firstLumi = lumi
+                            lastLumi = lumi
+                            if self.currentJob and not f in self.currentJob['input_files']:
+                                self.currentJob.addFile(f)
+                                filesInJob += 1
+                        if firstLumi != None and lastLumi != None:
+                            self.currentJob['mask'].addRunAndLumis(run = run.run, lumis = [firstLumi, lastLumi])
+                            addedEvents = ((lastLumi - firstLumi + 1) * f['avgEvtsPerLumi'])
+                            runAddedTime = addedEvents * timePerEvent
+                            runAddedSize = addedEvents * sizePerEvent
+                            self.currentJob.addResourceEstimates(jobTime = runAddedTime, disk = runAddedSize)
+                            firstLumi = None
+                            lastLumi = None
+                else:
+                    if createNewJob:
+                        if jobsPerGroup:
+                            if jobsInGroup > jobsPerGroup:
+                                self.newGroup()
+                                jobsInGroup = 0
+                        self.newJob(name = self.getJobName())
+                        self.currentJob.addResourceEstimates(memory = memoryRequirement)
+                        filesInJob = 0
+                        jobsInGroup += 1
+                        jobRun = fileRun
+                    self.currentJob.addFile(f)
+                    filesInJob += 1
+                    fileTime = f['events'] * timePerEvent
+                    fileSize = f['events'] * sizePerEvent
+                    self.currentJob.addResourceEstimates(jobTime = fileTime, disk = fileSize)
 
         return
diff --git a/src/python/WMCore/JobSplitting/LumiBased.py b/src/python/WMCore/JobSplitting/LumiBased.py
index 2d605b6..bc7150d 100644
--- a/src/python/WMCore/JobSplitting/LumiBased.py
+++ b/src/python/WMCore/JobSplitting/LumiBased.py
@@ -124,6 +124,7 @@ class LumiChecker:
         # Just a cosmetic "if": self.splitLumiFiles is empty when applyLumiCorrection is not enabled
         if not self.applyLumiCorrection:
             return
+
         for (run, lumi), files in self.splitLumiFiles.iteritems():
             for file_ in files:
                 self.lumiJobs[(run, lumi)].addFile(file_)
@@ -375,5 +376,6 @@ class LumiBased(JobFactory):
             if stopTask:
                 break
 
+        self.lumiChecker.closeJob(self.currentJob)
         self.lumiChecker.fixInputFiles()
         return
diff --git a/src/python/WMCore/Services/DBS/DBS3Reader.py b/src/python/WMCore/Services/DBS/DBS3Reader.py
index 4270bd0..b4815c8 100644
--- a/src/python/WMCore/Services/DBS/DBS3Reader.py
+++ b/src/python/WMCore/Services/DBS/DBS3Reader.py
@@ -577,9 +577,9 @@ class DBS3Reader:
                 msg += "%s\n" % str(ex)
                 raise Exception(msg)
 
-            if not blockInfo or len(blockInfo) != len(blockNames): #if we couldnt get data location from PhEDEx, try to look into origin site location from dbs
-                dbsOnly = True
-                blockNames = set(blockNames) - set(blockInfo) #get the blocks we did not find information in phedex
+#            if not blockInfo or len(blockInfo) != len(blockNames): #if we couldnt get data location from PhEDEx, try to look into origin site location from dbs
+                #dbsOnly = True
+#                blockNames = set(blockNames) - set(blockInfo) #get the blocks we did not find information in phedex
 
         if dbsOnly:
             try:
diff --git a/src/python/WMCore/Services/UserFileCache/UserFileCache.py b/src/python/WMCore/Services/UserFileCache/UserFileCache.py
index b2c887d..f0e425b 100644
--- a/src/python/WMCore/Services/UserFileCache/UserFileCache.py
+++ b/src/python/WMCore/Services/UserFileCache/UserFileCache.py
@@ -6,14 +6,75 @@ API for UserFileCache service
 """
 
 import os
-import hashlib
 import json
-import logging
+import shutil
+import hashlib
 import tarfile
+import tempfile
 
 from WMCore.Services.Service import Service
 
 
+def calculateChecksum(tarfile_, exclude=[]):
+    """
+    Calculate the checksum of the tar file in input.
+
+    The tarfile_ input parameter could be a string or a file object (anything compatible
+    with the fileobj parameter of tarfile.open).
+
+    The exclude parameter could be a list of strings, or a callable that takes as input
+    the output of  the list of tarfile.getmembers() and return a list of strings.
+    The exclude param is interpreted as a list of files that will not be taken into consideration
+    when calculating the checksum.
+
+    The output is the checksum of the tar input file.
+
+    The checksum is calculated taking into consideration the names of the objects
+    in the tarfile (files, directories etc) and the content of each file.
+
+    Each file is exctracted, read, and then deleted right after the input is passed
+    to the hasher object. The file is read in chuncks of 4096 bytes to avoid memory
+    issues.
+    """
+
+    hasher = hashlib.sha256()
+
+    ## "massage" out the input parameters
+    if isinstance(tarfile_, basestring):
+        tar = tarfile.open(tarfile_, mode='r')
+    else:
+        tar = tarfile.open(fileobj=tarfile_, mode='r')
+
+    if exclude and hasattr(exclude, '__call__'):
+        excludeList = exclude(tar.getmembers())
+    else:
+        excludeList = exclude
+
+
+    tmpDir = tempfile.mkdtemp()
+    try:
+        for tarmember in tar:
+            if tarmember.name in excludeList:
+                continue
+            hasher.update(tarmember.name)
+            if tarmember.isfile() and tarmember.name.split('.')[-1]!='pkl':
+                tar.extractall(path=tmpDir, members=[tarmember])
+                fn = os.path.join(tmpDir, tarmember.name)
+                with open(fn, 'rb') as fd:
+                    while True:
+                        buf = fd.read(4096)
+                        if not buf:
+                            break
+                        hasher.update(buf)
+                os.remove(fn)
+    finally:
+        #never leave tmddir around
+        shutil.rmtree(tmpDir)
+    checksum = hasher.hexdigest()
+
+    return checksum
+
+
 class UserFileCache(Service):
     """
     API for UserFileCache service
@@ -69,12 +130,15 @@ class UserFileCache(Service):
         self['logger'].debug('Wrote %s' % fileName)
         return fileName
 
-    def upload(self, fileName):
+    def upload(self, fileName, excludeList = []):
         """
         Upload the tarfile fileName to the user file cache. Returns the hash of the content of the file
         which can be used to retrieve the file later on.
         """
-        params = [('hashkey', self.checksum(fileName))]
+
+        #The parameter newchecksum tells the crabcace to use the new algorithm. It's there
+        #for guarantee backward compatibility
+        params = [('hashkey', calculateChecksum(fileName, excludeList)), ('newchecksum', '1')]
 
         resString = self["requests"].uploadFile(fileName=fileName, fieldName='inputfile',
                                                 url=self['endpoint'] + 'file',
@@ -82,16 +146,3 @@ class UserFileCache(Service):
 
         return json.loads(resString)['result'][0]
 
-    def checksum(self, fileName):
-        """
-        Calculate the checksum of the file. We don't just hash the contents because
-        that includes the timestamp of when the tar was made, not just the timestamps
-        of the constituent files
-        """
-
-        tar = tarfile.open(fileName, mode='r')
-        lsl = [(x.name, int(x.size), int(x.mtime), x.uname) for x in tar.getmembers()]
-        hasher = hashlib.sha256(str(lsl))
-        checksum = hasher.hexdigest()
-
-        return checksum
diff --git a/src/python/WMCore/Services/pycurl_manager.py b/src/python/WMCore/Services/pycurl_manager.py
index 385ca1f..be95e7e 100644
--- a/src/python/WMCore/Services/pycurl_manager.py
+++ b/src/python/WMCore/Services/pycurl_manager.py
@@ -138,7 +138,7 @@ class RequestHandler(object):
                         % (str(exc), type(data))
                 logging.warning(msg)
                 return data
-            return data
+            return res
         else:
             return data
 
diff --git a/src/python/WMCore/Storage/SiteLocalConfig.py b/src/python/WMCore/Storage/SiteLocalConfig.py
index 7e43d3b..8d7e3a2 100644
--- a/src/python/WMCore/Storage/SiteLocalConfig.py
+++ b/src/python/WMCore/Storage/SiteLocalConfig.py
@@ -274,6 +274,8 @@ def processLocalStageOut():
                 localReport['option'] = subnode.attrs.get('value', None)
             elif subnode.name == 'catalog':
                 localReport['catalog'] = subnode.attrs.get('url', None)
+            elif subnode.name == 'phedex-node':
+                localReport['phedex-node'] = subnode.attrs.get('value', None)
         report['localStageOut'] = localReport
 
 @coroutine
diff --git a/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py b/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py
index e0cfdb2..64bc385 100755
--- a/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py
+++ b/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py
@@ -598,6 +598,11 @@ class SetupCMSSWPset(ScriptInterface):
                 print traceback.format_exc()
                 raise ex
 
+            self.process.add_(
+                cms.Service("CondorStatusService"
+                            )
+                )
+
             if funcName == "repack":
                 self.handleRepackSettings()
 
diff --git a/test/python/WMCore_t/JobSplitting_t/EventAwareLumiBased_t.py b/test/python/WMCore_t/JobSplitting_t/EventAwareLumiBased_t.py
index 192a147..beed60a 100644
--- a/test/python/WMCore_t/JobSplitting_t/EventAwareLumiBased_t.py
+++ b/test/python/WMCore_t/JobSplitting_t/EventAwareLumiBased_t.py
@@ -626,6 +626,104 @@ class EventAwareLumiBasedTest(unittest.TestCase):
         self.assertEqual(jobs[0]['mask'].getRunAndLumis(), {1: [[10, 14]], 2: [[20, 21]], 4: [[40, 40]]})
         self.assertEqual(jobs[1]['mask'].getRunAndLumis(), {4: [[41, 41]]})
 
+    def testH_LumiCorrections(self):
+        """
+        _LumiCorrections_
+
+        Test the splitting algorithm can handle lumis which
+        cross multiple files.
+        """
+        splitter = SplitterFactory()
+        testSubscription = self.createSubscription(nFiles = 2, lumisPerFile = 2,
+                                                   twoSites = False, nEventsPerFile = 150)
+        files = testSubscription.getFileset().getFiles()
+        self.assertEqual(len(files), 2)
+        # at the moment we have two files with two lumis each:
+        #  file0 has run0 and lumis 0,1. 150 events
+        #  file1 has run1 and lumis 2,3. 150 evens
+        jobFactory = splitter(package = "WMCore.DataStructs",
+                              subscription = testSubscription)
+
+        jobGroups = jobFactory(events_per_job = 50,
+                               halt_job_on_file_boundaries = False,
+                               splitOnRun = False,
+                               performance = self.performanceParams,
+                               applyLumiCorrection = False
+                              )
+
+        # The splitting algorithm will assume 75 events per lumi.
+        # We will have one job per lumi
+        self.assertEqual(len(jobGroups), 1)
+        jobs = jobGroups[0].jobs
+        self.assertEqual(len(jobs), 4)
+
+        testSubscription = self.createSubscription(nFiles = 2, lumisPerFile = 2,
+                                           twoSites = False, nEventsPerFile = 150)
+        files = testSubscription.getFileset().getFiles()
+        # Now modifyng and adding duplicated lumis.
+        for runObj in files[0]['runs']:
+            if runObj.run != 0:
+                continue
+            runObj.lumis.append(42)
+        for runObj in files[1]['runs']:
+            if runObj.run != 1:
+                continue
+            runObj.run = 0
+            runObj.lumis.append(42)
+        files[1]['locations'] = set(['blenheim'])
+        jobFactory = splitter(package = "WMCore.DataStructs",
+                              subscription = testSubscription)
+        jobGroups = jobFactory(events_per_job = 50,
+                               halt_job_on_file_boundaries = True,
+                               performance = self.performanceParams,
+                               applyLumiCorrection = True)
+
+        # Now we will have:
+        #   file0: Run0 and lumis [0, 1, 42]
+        #   file1: Run0 and lumis [2, 3, 42]
+        # Splitting algorithm is assuming 50 events per lumi
+        # Three jobs (one per lumu) for the first file
+        # Two jobs for the second file (42 is duplicated)
+        self.assertEqual(len(jobGroups), 1)
+        jobs = jobGroups[0].jobs
+        self.assertEqual(len(jobs), 5)
+        self.assertEqual(len(jobs[0]['input_files']), 1)
+        self.assertEqual(len(jobs[1]['input_files']), 1)
+        self.assertEqual(len(jobs[2]['input_files']), 2)
+        self.assertEqual(len(jobs[3]['input_files']), 1)
+        self.assertEqual(len(jobs[4]['input_files']), 1)
+        self.assertEqual(jobs[0]['mask'].getRunAndLumis(), {0: [[0, 0]]})
+        self.assertEqual(jobs[1]['mask'].getRunAndLumis(), {0: [[1, 1]]})
+        self.assertEqual(jobs[2]['mask'].getRunAndLumis(), {0: [[42, 42]]})
+        self.assertEqual(jobs[3]['mask'].getRunAndLumis(), {0: [[2, 2]]})
+        self.assertEqual(jobs[4]['mask'].getRunAndLumis(), {0: [[3, 3]]})
+
+
+        #Check that if the last two jobs have the same duplicated lumi you do not get an error
+        testSubscription = self.createSubscription(nFiles = 2, lumisPerFile = 2,
+                                           twoSites = False, nEventsPerFile = 150)
+        files = testSubscription.getFileset().getFiles()
+        # Now modifying and adding the same duplicated lumis in the Nth and Nth-1 jobs
+        for runObj in files[0]['runs']:
+            if runObj.run != 0:
+                continue
+            runObj.lumis.append(42)
+        for runObj in files[1]['runs']:
+            runObj.run = 0
+            runObj.lumis = [42]
+        files[1]['locations'] = set(['blenheim'])
+        jobFactory = splitter(package = "WMCore.DataStructs",
+                              subscription = testSubscription)
+        jobGroups = jobFactory(events_per_job = 50,
+                               halt_job_on_file_boundaries = True,
+                               performance = self.performanceParams,
+                               applyLumiCorrection = True)
+
+        self.assertEqual(len(jobGroups), 1)
+        jobs = jobGroups[0].jobs
+        self.assertEqual(len(jobs), 3)
+
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/python/WMCore_t/JobSplitting_t/FileBased_t.py b/test/python/WMCore_t/JobSplitting_t/FileBased_t.py
index 9d5451e..a8fec4e 100644
--- a/test/python/WMCore_t/JobSplitting_t/FileBased_t.py
+++ b/test/python/WMCore_t/JobSplitting_t/FileBased_t.py
@@ -15,6 +15,7 @@ from WMCore.DataStructs.Fileset import Fileset
 from WMCore.DataStructs.Job import Job
 from WMCore.DataStructs.Subscription import Subscription
 from WMCore.DataStructs.Workflow import Workflow
+from WMCore.DataStructs.Run import Run
 
 from WMCore.JobSplitting.SplitterFactory import SplitterFactory
 from WMCore.Services.UUID import makeUUID
@@ -39,11 +40,17 @@ class FileBasedTest(unittest.TestCase):
             newFile = File(makeUUID(), size = 1000, events = 100)
             newFile.setLocation('blenheim')
             newFile.setLocation('malpaquet')
+            lumis = []
+            for lumi in range(20):
+                lumis.append((i * 100) + lumi)
+                newFile.addRun(Run(i, *lumis))
             self.multipleFileFileset.addFile(newFile)
 
         self.singleFileFileset = Fileset(name = "TestFileset2")
         newFile = File("/some/file/name", size = 1000, events = 100)
         newFile.setLocation('blenheim')
+        lumis = range(50,60) + range(70,80)
+        newFile.addRun(Run(13, *lumis))
         self.singleFileFileset.addFile(newFile)
 
         testWorkflow = Workflow()
@@ -188,5 +195,38 @@ class FileBasedTest(unittest.TestCase):
 
         return
 
+    def test4WithLumiMask(self):
+        """
+        _test4WithLumiMask_
+
+        Test file based job splitting when
+        """
+        splitter = SplitterFactory()
+        jobFactory = splitter(self.multipleFileSubscription)
+
+        jobGroups = jobFactory(files_per_job = 2,
+                               total_files = 3,
+                               runs = ['1', '2', '4', '5'],
+                               lumis = ['100,130', '203,204,207,221', '401,405', '500, 520'],
+                               performance = self.performanceParams)
+
+        self.assertEqual(len(jobGroups), 1)
+
+        self.assertEqual(len(jobGroups[0].jobs), 2)
+
+        fileList = []
+        for job in jobGroups[0].jobs:
+            assert len(job.getFiles(type = "list")) in [2, 1], \
+                   "ERROR: Job contains incorrect number of files."
+
+            for file in job.getFiles(type = "lfn"):
+                assert file not in fileList, \
+                       "ERROR: File duplicated!"
+                fileList.append(file)
+
+        self.assertEqual(len(fileList), 3)
+
+        return
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/python/WMCore_t/JobSplitting_t/LumiBased_t.py b/test/python/WMCore_t/JobSplitting_t/LumiBased_t.py
index 3266803..3b22569 100644
--- a/test/python/WMCore_t/JobSplitting_t/LumiBased_t.py
+++ b/test/python/WMCore_t/JobSplitting_t/LumiBased_t.py
@@ -277,5 +277,30 @@ class LumiBasedTest(unittest.TestCase):
         self.assertEqual(jobs[1]['mask'].getRunAndLumis(), {2: [[200, 200]]})
         self.assertEqual(jobs[2]['mask'].getRunAndLumis(), {3: [[300, 300]]})
 
+
+        #Check that if the last two jobs have the same duplicated lumi you do not get an error
+        testSubscription = self.createSubscription(nFiles = 2, lumisPerFile = 2,
+                                           twoSites = False)
+        files = testSubscription.getFileset().getFiles()
+        # Now modifying and adding the same duplicated lumis in the Nth and Nth-1 jobs
+        for runObj in files[0]['runs']:
+            if runObj.run != 0:
+                continue
+            runObj.lumis.append(42)
+        for runObj in files[1]['runs']:
+            runObj.run = 0
+            runObj.lumis = [42]
+        files[1]['locations'] = set(['blenheim'])
+        jobFactory = splitter(package = "WMCore.DataStructs",
+                              subscription = testSubscription)
+        jobGroups = jobFactory(events_per_job = 50,
+                               halt_job_on_file_boundaries = True,
+                               performance = self.performanceParams,
+                               applyLumiCorrection = True)
+
+        self.assertEqual(len(jobGroups), 1)
+        jobs = jobGroups[0].jobs
+        self.assertEqual(len(jobs), 3)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/python/WMCore_t/Services_t/UserFileCache_t/UserFileCache_t.py b/test/python/WMCore_t/Services_t/UserFileCache_t/UserFileCache_t.py
index 466c4ee..4b3f86d 100644
--- a/test/python/WMCore_t/Services_t/UserFileCache_t/UserFileCache_t.py
+++ b/test/python/WMCore_t/Services_t/UserFileCache_t/UserFileCache_t.py
@@ -8,7 +8,7 @@ import filecmp
 import os
 from os import path
 
-from WMCore.Services.UserFileCache.UserFileCache import UserFileCache
+from WMCore.Services.UserFileCache.UserFileCache import UserFileCache, calculateChecksum
 from WMCore.WMBase import getTestBase
 
 class UserFileCacheTest(unittest.TestCase):
@@ -21,14 +21,13 @@ class UserFileCacheTest(unittest.TestCase):
         """
         Tests checksum method
         """
-        self.ufc = UserFileCache()
-        checksum1 = self.ufc.checksum(fileName=path.join(getTestBase(), 'WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz'))
-        checksum2 = self.ufc.checksum(fileName=path.join(getTestBase(), 'WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz'))
+        checksum1 = calculateChecksum(tarfile_=path.join(getTestBase(), 'WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz'))
+        checksum2 = calculateChecksum(tarfile_=path.join(getTestBase(), 'WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz'))
         self.assertTrue(checksum1)
         self.assertTrue(checksum2)
         self.assertFalse(checksum1 == checksum2)
 
-        self.assertRaises(IOError, self.ufc.checksum, **{'fileName': 'does_not_exist'})
+        self.assertRaises(IOError, calculateChecksum, **{'tarfile_': 'does_not_exist'})
         return
 
     def testUploadDownload(self):
diff --git a/test/python/WMCore_t/Storage_t/SiteLocalConfig_t.py b/test/python/WMCore_t/Storage_t/SiteLocalConfig_t.py
index 9c907f5..65aa6bd 100644
--- a/test/python/WMCore_t/Storage_t/SiteLocalConfig_t.py
+++ b/test/python/WMCore_t/Storage_t/SiteLocalConfig_t.py
@@ -11,8 +11,11 @@ import unittest
 from WMQuality.TestInit import TestInit
 from WMCore.WMBase import getTestBase
 
-from WMCore.Storage.SiteLocalConfig import SiteLocalConfig
+from WMCore.Storage.SiteLocalConfig import SiteLocalConfig, SiteConfigError
 from WMCore.Storage.SiteLocalConfig import loadSiteLocalConfig
+from WMCore.Services.PhEDEx.PhEDEx import PhEDEx
+
+from nose.plugins.attrib import attr
 
 class SiteLocalConfigTest(unittest.TestCase):
     def setUp(self):
@@ -156,6 +159,38 @@ class SiteLocalConfigTest(unittest.TestCase):
                          "Error: Wrong site name.")
 
 
+    @attr("integration")
+    def testSlcPhedexNodesEqualPhedexApiNodes(self):
+        """
+        For each site, verify that the stageout node specified in
+        site-local-config.xml is the same as the one returned by the PhEDEx api.
+        """
+        os.environ["CMS_PATH"] = "/cvmfs/cms.cern.ch"
+
+        phedex = PhEDEx()
+        nodes = phedex.getNodeMap()["phedex"]["node"]
+
+        # Make a dict for translating the se names into regular site names.
+        node_map = {}
+        for node in nodes:
+            node_map[str(node[u"se"])] = str(node[str(u"name")])
+        
+        for d in os.listdir("/cvmfs/cms.cern.ch/SITECONF/"):
+            # Only T0_, T1_... folders are needed
+            if d[0] == "T":
+                os.environ['WMAGENT_SITE_CONFIG_OVERRIDE'] ='/cvmfs/cms.cern.ch/SITECONF/%s/JobConfig/site-local-config.xml' % (d)
+                try:
+                    slc = loadSiteLocalConfig()
+                except SiteConfigError as e:
+                    print e.args[0]
+                phedexNode = slc.localStageOut.get("phedex-node")
+                # If slc is correct, perform check
+                if "se-name" in slc.localStageOut and slc.localStageOut["se-name"] in node_map and phedexNode != None:
+                    self.assertEqual(phedexNode, node_map[slc.localStageOut["se-name"]], \
+                            "Error: Node specified in SLC (%s) doesn't match node returned by PhEDEx api (%s)." \
+                            % (phedexNode, node_map[slc.localStageOut["se-name"]]))
+                    
+        return 
 
 if __name__ == "__main__":
     unittest.main()