From 80a53e88510cb488c6d95d2889458be9325dd44a Mon Sep 17 00:00:00 2001 From: Michael Heinold Date: Mon, 26 Jun 2017 13:34:40 +0200 Subject: [PATCH] More package changes and fixes for that. --- .../de/dkfz/roddy/BEIntegrationTestStarter.groovy | 4 ++-- .../groovy/de/dkfz/roddy/execution/jobs/BEJob.groovy | 10 +++++----- .../jobs/{JobResult.groovy => BEJobResult.groovy} | 10 +++++----- .../execution/jobs/BatchEuphoriaJobManager.groovy | 12 ++++++------ .../de/dkfz/roddy/execution/jobs/FakeBEJob.groovy | 4 ++++ .../lsf/rest/BatchEuphoriaJobManagerAdapter.groovy | 7 ++++--- .../jobs/cluster/lsf/rest/LSFRestJobManager.groovy | 6 +++--- .../execution/jobs/cluster/pbs/PBSCommand.groovy | 5 +++-- .../execution/jobs/cluster/pbs/PBSJobManager.groovy | 7 ++++--- .../DirectSynchronousExecutionJobManager.groovy | 8 ++++---- 10 files changed, 40 insertions(+), 33 deletions(-) rename src/main/groovy/de/dkfz/roddy/execution/jobs/{JobResult.groovy => BEJobResult.groovy} (78%) diff --git a/src/main/groovy/de/dkfz/roddy/BEIntegrationTestStarter.groovy b/src/main/groovy/de/dkfz/roddy/BEIntegrationTestStarter.groovy index 6aa3d094..35f49c9e 100644 --- a/src/main/groovy/de/dkfz/roddy/BEIntegrationTestStarter.groovy +++ b/src/main/groovy/de/dkfz/roddy/BEIntegrationTestStarter.groovy @@ -17,7 +17,7 @@ import de.dkfz.roddy.execution.jobs.JobManagerCreationParameters import de.dkfz.roddy.execution.jobs.JobManagerCreationParametersBuilder import de.dkfz.roddy.execution.jobs.JobState import de.dkfz.roddy.execution.io.ExecutionResult -import de.dkfz.roddy.execution.jobs.JobResult +import de.dkfz.roddy.execution.jobs.BEJobResult import de.dkfz.roddy.tools.BufferUnit import de.dkfz.roddy.tools.BufferValue import de.dkfz.roddy.tools.LoggerWrapper @@ -108,7 +108,7 @@ class BEIntegrationTestStarter { def jobList = [testJob] // run single job and check status - JobResult jr = jobManager.runJob(testJob) + BEJobResult jr = jobManager.runJob(testJob) if (jobManager.isHoldJobsEnabled()) { log.postAlwaysInfo("Started ${jr.jobID.id}") ensureProperJobStates(maxSleep, jobList, [JobState.HOLD], jobManager) diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJob.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJob.groovy index a0a0805b..35a5b07d 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJob.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJob.groovy @@ -13,11 +13,11 @@ import java.util.concurrent.atomic.AtomicLong /** * The job class represents a generic and abstract form of cluster job which can be run using a job manager. - * When a job is executed with the JM, the used Command and a JobResult object will be created and added to this + * When a job is executed with the JM, the used Command and a BEJobResult object will be created and added to this * object. */ @groovy.transform.CompileStatic -class BEJob { +class BEJob { private static final de.dkfz.roddy.tools.LoggerWrapper logger = de.dkfz.roddy.tools.LoggerWrapper.getLogger(BEJob.class.getSimpleName()) @@ -91,7 +91,7 @@ class BEJob { /** * Stores the result when the job was executed. */ - de.dkfz.roddy.execution.jobs.JobResult runResult + JR runResult ////////////////////////////////////////////////////////////// // Now come some job / command specific settings @@ -139,7 +139,7 @@ class BEJob { } //TODO Create a runArray method which returns several job results with proper array ids. - de.dkfz.roddy.execution.jobs.JobResult run() { + JR run() { } @@ -154,7 +154,7 @@ class BEJob { return BEFakeJobID.isFakeJobID(jobID) } - protected void postProcessArrayJob(JobResult runResult) { + protected void postProcessArrayJob(JR runResult) { throw new NotImplementedException() Map prmsAsStringMap = new LinkedHashMap<>() for (String k : parameters.keySet()) { diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/JobResult.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobResult.groovy similarity index 78% rename from src/main/groovy/de/dkfz/roddy/execution/jobs/JobResult.groovy rename to src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobResult.groovy index 5a9b8a4c..6d8bfb31 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/JobResult.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/BEJobResult.groovy @@ -16,7 +16,7 @@ import de.dkfz.roddy.core.InfoObject * * @author michael */ -public class JobResult implements Serializable { +public class BEJobResult implements Serializable { /** * The command which was used to create this result. @@ -49,19 +49,19 @@ public class JobResult implements Serializable { public transient final List parentJobs; // Compatibility constructor. Does nothing, leaves responsibility in sub class. - protected JobResult() { + protected BEJobResult() { } - public JobResult(InfoObject object, Command command, BEJobDependencyID jobID, boolean wasExecuted, File toolID, Map jobParameters, List parentJobs) { + public BEJobResult(Object object, Command command, BEJobDependencyID jobID, boolean wasExecuted, File toolID, Map jobParameters, List parentJobs) { this(command, jobID, wasExecuted, false, toolID, jobParameters, parentJobs) } - public JobResult(Command command, BEJobDependencyID jobID, boolean wasExecuted, File toolID, Map jobParameters, List parentJobs) { + public BEJobResult(Command command, BEJobDependencyID jobID, boolean wasExecuted, File toolID, Map jobParameters, List parentJobs) { this(command, jobID, wasExecuted, false, toolID, jobParameters, parentJobs) } - public JobResult(Command command, BEJobDependencyID jobID, boolean wasExecuted, boolean wasArray, File toolID, Map jobParameters, List parentJobs) { + public BEJobResult(Command command, BEJobDependencyID jobID, boolean wasExecuted, boolean wasArray, File toolID, Map jobParameters, List parentJobs) { this.command = command; this.jobID = jobID; this.wasExecuted = wasExecuted; diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy index a5491ee2..1352f525 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy @@ -7,7 +7,7 @@ package de.dkfz.roddy.execution.jobs import de.dkfz.roddy.execution.io.ExecutionResult -import de.dkfz.roddy.execution.jobs.JobResult as JobResult +import de.dkfz.roddy.execution.jobs.BEJobResult import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.BEExecutionService import de.dkfz.roddy.tools.LoggerWrapper @@ -152,19 +152,19 @@ abstract class BatchEuphoriaJobManager { * @param job * @return */ - abstract JobResult runJob(BEJob job) + abstract BEJobResult runJob(BEJob job) /** * Called by the execution service after a command was executed. */ - JobResult extractAndSetJobResultFromExecutionResult(Command command, ExecutionResult res) { - JobResult jobResult + BEJobResult extractAndSetJobResultFromExecutionResult(Command command, ExecutionResult res) { + BEJobResult jobResult if (res.successful) { String exID = parseJobID(res.resultLines[0]) def job = command.getJob() def jobDependencyID = createJobDependencyID(job, exID) command.setExecutionID(jobDependencyID) - jobResult = new JobResult(command, jobDependencyID, res.successful, false, job.tool, job.parameters, job.parentJobs as List) + jobResult = new BEJobResult(command, jobDependencyID, res.successful, false, job.tool, job.parameters, job.parentJobs as List) job.setRunResult(jobResult) } return jobResult @@ -209,7 +209,7 @@ abstract class BatchEuphoriaJobManager { abstract GenericJobInfo parseGenericJobInfo(String command) - abstract JobResult convertToArrayResult(BEJob arrayChildJob, JobResult parentJobsResult, int arrayIndex) + abstract BEJobResult convertToArrayResult(BEJob arrayChildJob, BEJobResult parentJobsResult, int arrayIndex) abstract void updateJobStatus() diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/FakeBEJob.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/FakeBEJob.groovy index bed60f9b..08dc9604 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/FakeBEJob.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/FakeBEJob.groovy @@ -18,6 +18,10 @@ class FakeBEJob extends BEJob { super("Fakejob", null,null, "", null, [], [:], [], [], null) } + public FakeBEJob(Object context) { + this() + } + public FakeBEJob(InfoObject context) { this() } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/BatchEuphoriaJobManagerAdapter.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/BatchEuphoriaJobManagerAdapter.groovy index ed969b3d..99340176 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/BatchEuphoriaJobManagerAdapter.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/BatchEuphoriaJobManagerAdapter.groovy @@ -8,6 +8,7 @@ package de.dkfz.roddy.execution.jobs.cluster.lsf.rest import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.BEExecutionService +import de.dkfz.roddy.execution.jobs.BEJobResult import de.dkfz.roddy.execution.jobs.cluster.ClusterJobManager import de.dkfz.roddy.execution.jobs.Command import de.dkfz.roddy.execution.jobs.GenericJobInfo @@ -16,7 +17,7 @@ import de.dkfz.roddy.execution.jobs.JobManagerCreationParameters import de.dkfz.roddy.execution.jobs.JobState import de.dkfz.roddy.execution.jobs.ProcessingCommands import de.dkfz.roddy.execution.jobs.BEJobDependencyID -import de.dkfz.roddy.execution.jobs.JobResult +import de.dkfz.roddy.execution.jobs.BEJobResult import sun.reflect.generics.reflectiveObjects.NotImplementedException /** @@ -33,7 +34,7 @@ class BatchEuphoriaJobManagerAdapter extends ClusterJobManager { } @Override - JobResult runJob(BEJob job) { + BEJobResult runJob(BEJob job) { return null } @@ -69,7 +70,7 @@ class BatchEuphoriaJobManagerAdapter extends ClusterJobManager { } @Override - JobResult convertToArrayResult(BEJob arrayChildJob, JobResult parentJobsResult, int arrayIndex) { + BEJobResult convertToArrayResult(BEJob arrayChildJob, BEJobResult parentJobsResult, int arrayIndex) { throw new NotImplementedException() } // needed diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy index cd7e498f..2543c5b1 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy @@ -14,7 +14,7 @@ import de.dkfz.roddy.execution.jobs.cluster.pbs.PBSResourceProcessingCommand import de.dkfz.roddy.execution.jobs.GenericJobInfo import de.dkfz.roddy.execution.jobs.BEJob import de.dkfz.roddy.execution.jobs.JobManagerCreationParameters -import de.dkfz.roddy.execution.jobs.JobResult +import de.dkfz.roddy.execution.jobs.BEJobResult import de.dkfz.roddy.execution.jobs.JobState import de.dkfz.roddy.execution.jobs.ProcessingCommands import de.dkfz.roddy.tools.BufferUnit @@ -55,7 +55,7 @@ class LSFRestJobManager extends BatchEuphoriaJobManagerAdapter { @Override - de.dkfz.roddy.execution.jobs.JobResult runJob(BEJob job) { + BEJobResult runJob(BEJob job) { submitJob(job) return job.runResult } @@ -175,7 +175,7 @@ class LSFRestJobManager extends BatchEuphoriaJobManagerAdapter { RestResult result = restExecutionService.execute(new RestCommand(URI_JOB_SUBMIT, requestBody.toString(), headers, RestCommand.HttpMethod.HTTPPOST)) as RestResult if (result.statusCode == 200) { logger.postAlwaysInfo("status code: " + result.statusCode + " result:" + new XmlSlurper().parseText(result.body)) - job.setRunResult(new JobResult(job.lastCommand, new PBSJobDependencyID(job, new XmlSlurper().parseText(result.body).text()), true, job.tool, job.parameters, job.parentJobs as List)) + job.setRunResult(new BEJobResult(job.lastCommand, new PBSJobDependencyID(job, new XmlSlurper().parseText(result.body).text()), true, job.tool, job.parameters, job.parentJobs as List)) } else { logger.postAlwaysInfo("status code: " + result.statusCode + " result: " + result.body) } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommand.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommand.groovy index b7b34c5b..f65040bf 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommand.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommand.groovy @@ -217,10 +217,11 @@ class PBSCommand extends Command { String assembleVariableExportString() { StringBuilder qsubCall = new StringBuilder() + if (job.getParameterFile()) { - qsubCall << getVariablesParameter() << "PARAMETER_FILE=" << job.getParameterFile() + qsubCall << getVariablesParameter() << "CONFIG_FILE=" << job.parameters["CONFIG_FILE"] << ",PARAMETER_FILE=" << job.getParameterFile() } else { - def finalParameters = job.finalParameters() + List finalParameters = job.finalParameters() if (finalParameters) qsubCall << getVariablesParameter() << finalParameters.join(",") } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy index ea623f82..6348b3a8 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy @@ -8,6 +8,7 @@ package de.dkfz.roddy.execution.jobs.cluster.pbs import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.BEExecutionService +import de.dkfz.roddy.execution.jobs.BEJobResult import de.dkfz.roddy.execution.jobs.cluster.ClusterJobManager import de.dkfz.roddy.StringConstants import de.dkfz.roddy.execution.io.ExecutionResult @@ -15,7 +16,7 @@ import de.dkfz.roddy.execution.jobs.GenericJobInfo import de.dkfz.roddy.execution.jobs.BEJob import de.dkfz.roddy.execution.jobs.BEJobDependencyID import de.dkfz.roddy.execution.jobs.JobManagerCreationParameters -import de.dkfz.roddy.execution.jobs.JobResult +import de.dkfz.roddy.execution.jobs.BEJobResult import de.dkfz.roddy.execution.jobs.JobState import de.dkfz.roddy.execution.jobs.ProcessingCommands import de.dkfz.roddy.tools.BufferUnit @@ -93,7 +94,7 @@ class PBSJobManager extends ClusterJobManager { } @Override - JobResult runJob(BEJob job) { + BEJobResult runJob(BEJob job) { def command = createCommand(job) def executionResult = executionService.execute(command) extractAndSetJobResultFromExecutionResult(command, executionResult) @@ -284,7 +285,7 @@ class PBSJobManager extends ClusterJobManager { } @Override - JobResult convertToArrayResult(BEJob arrayChildJob, JobResult parentJobsResult, int arrayIndex) { + BEJobResult convertToArrayResult(BEJob arrayChildJob, BEJobResult parentJobsResult, int arrayIndex) { return null } diff --git a/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy b/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy index a3bbac4d..45213510 100644 --- a/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy +++ b/src/main/groovy/de/dkfz/roddy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy @@ -11,7 +11,7 @@ import de.dkfz.roddy.config.ResourceSet import de.dkfz.roddy.execution.BEExecutionService import de.dkfz.roddy.execution.jobs.*; import de.dkfz.roddy.execution.jobs.BEJobDependencyID; -import de.dkfz.roddy.execution.jobs.JobResult; +import de.dkfz.roddy.execution.jobs.BEJobResult; import de.dkfz.roddy.tools.LoggerWrapper /** @@ -75,7 +75,7 @@ public class DirectSynchronousExecutionJobManager extends BatchEuphoriaJobManage } @Override - public JobResult convertToArrayResult(BEJob arrayChildJob, JobResult parentJobsResult, int arrayIndex) { + public BEJobResult convertToArrayResult(BEJob arrayChildJob, BEJobResult parentJobsResult, int arrayIndex) { throw new RuntimeException("Not implemented yet! " + this.getClass().getName() + ".convertToArrayResult()"); } @@ -90,7 +90,7 @@ public class DirectSynchronousExecutionJobManager extends BatchEuphoriaJobManage // } // @Override -// public JobResult convertToArrayResult(BEJob arrayChildJob, JobResult parentJobsResult, int arrayIndex) { +// public BEJobResult convertToArrayResult(BEJob arrayChildJob, BEJobResult parentJobsResult, int arrayIndex) { // throw new RuntimeException("Not implemented yet! " + this.getClass().getName() + ".convertToArrayResult()"); // } @@ -179,7 +179,7 @@ public class DirectSynchronousExecutionJobManager extends BatchEuphoriaJobManage } @Override - public JobResult runJob(BEJob job) { + public BEJobResult runJob(BEJob job) { return null; }