Skip to content

Commit

Permalink
Merge branch 'FixPackages' into develop
Browse files Browse the repository at this point in the history
  • Loading branch information
Michael Heinold committed Jun 26, 2017
2 parents 83f6462 + 80a53e8 commit 55cfdc5
Show file tree
Hide file tree
Showing 10 changed files with 40 additions and 33 deletions.
4 changes: 2 additions & 2 deletions src/main/groovy/de/dkfz/roddy/BEIntegrationTestStarter.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import de.dkfz.roddy.execution.jobs.JobManagerCreationParameters
import de.dkfz.roddy.execution.jobs.JobManagerCreationParametersBuilder
import de.dkfz.roddy.execution.jobs.JobState
import de.dkfz.roddy.execution.io.ExecutionResult
import de.dkfz.roddy.execution.jobs.JobResult
import de.dkfz.roddy.execution.jobs.BEJobResult
import de.dkfz.roddy.tools.BufferUnit
import de.dkfz.roddy.tools.BufferValue
import de.dkfz.roddy.tools.LoggerWrapper
Expand Down Expand Up @@ -108,7 +108,7 @@ class BEIntegrationTestStarter {
def jobList = [testJob]

// run single job and check status
JobResult jr = jobManager.runJob(testJob)
BEJobResult jr = jobManager.runJob(testJob)
if (jobManager.isHoldJobsEnabled()) {
log.postAlwaysInfo("Started ${jr.jobID.id}")
ensureProperJobStates(maxSleep, jobList, [JobState.HOLD], jobManager)
Expand Down
10 changes: 5 additions & 5 deletions src/main/groovy/de/dkfz/roddy/execution/jobs/BEJob.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ import java.util.concurrent.atomic.AtomicLong

/**
* The job class represents a generic and abstract form of cluster job which can be run using a job manager.
* When a job is executed with the JM, the used Command and a JobResult object will be created and added to this
* When a job is executed with the JM, the used Command and a BEJobResult object will be created and added to this
* object.
*/
@groovy.transform.CompileStatic
class BEJob<J extends BEJob> {
class BEJob<J extends BEJob, JR extends BEJobResult> {

private static final de.dkfz.roddy.tools.LoggerWrapper logger = de.dkfz.roddy.tools.LoggerWrapper.getLogger(BEJob.class.getSimpleName())

Expand Down Expand Up @@ -91,7 +91,7 @@ class BEJob<J extends BEJob> {
/**
* Stores the result when the job was executed.
*/
de.dkfz.roddy.execution.jobs.JobResult runResult
JR runResult

//////////////////////////////////////////////////////////////
// Now come some job / command specific settings
Expand Down Expand Up @@ -139,7 +139,7 @@ class BEJob<J extends BEJob> {
}

//TODO Create a runArray method which returns several job results with proper array ids.
de.dkfz.roddy.execution.jobs.JobResult run() {
JR run() {

}

Expand All @@ -154,7 +154,7 @@ class BEJob<J extends BEJob> {
return BEFakeJobID.isFakeJobID(jobID)
}

protected void postProcessArrayJob(JobResult runResult) {
protected void postProcessArrayJob(JR runResult) {
throw new NotImplementedException()
Map<String, Object> prmsAsStringMap = new LinkedHashMap<>()
for (String k : parameters.keySet()) {
Expand Down
10 changes: 5 additions & 5 deletions ...kfz/roddy/execution/jobs/JobResult.groovy → ...z/roddy/execution/jobs/BEJobResult.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import de.dkfz.roddy.core.InfoObject
*
* @author michael
*/
public class JobResult implements Serializable {
public class BEJobResult implements Serializable {

/**
* The command which was used to create this result.
Expand Down Expand Up @@ -49,19 +49,19 @@ public class JobResult implements Serializable {
public transient final List<BEJob> parentJobs;

// Compatibility constructor. Does nothing, leaves responsibility in sub class.
protected JobResult() {
protected BEJobResult() {

}

public JobResult(InfoObject object, Command command, BEJobDependencyID jobID, boolean wasExecuted, File toolID, Map<String, String> jobParameters, List<BEJob> parentJobs) {
public BEJobResult(Object object, Command command, BEJobDependencyID jobID, boolean wasExecuted, File toolID, Map<String, String> jobParameters, List<BEJob> parentJobs) {
this(command, jobID, wasExecuted, false, toolID, jobParameters, parentJobs)
}

public JobResult(Command command, BEJobDependencyID jobID, boolean wasExecuted, File toolID, Map<String, String> jobParameters, List<BEJob> parentJobs) {
public BEJobResult(Command command, BEJobDependencyID jobID, boolean wasExecuted, File toolID, Map<String, String> jobParameters, List<BEJob> parentJobs) {
this(command, jobID, wasExecuted, false, toolID, jobParameters, parentJobs)
}

public JobResult(Command command, BEJobDependencyID jobID, boolean wasExecuted, boolean wasArray, File toolID, Map<String, String> jobParameters, List<BEJob> parentJobs) {
public BEJobResult(Command command, BEJobDependencyID jobID, boolean wasExecuted, boolean wasArray, File toolID, Map<String, String> jobParameters, List<BEJob> parentJobs) {
this.command = command;
this.jobID = jobID;
this.wasExecuted = wasExecuted;
Expand Down
12 changes: 6 additions & 6 deletions src/main/groovy/de/dkfz/roddy/execution/jobs/BatchEuphoriaJobManager.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
package de.dkfz.roddy.execution.jobs

import de.dkfz.roddy.execution.io.ExecutionResult
import de.dkfz.roddy.execution.jobs.JobResult as JobResult
import de.dkfz.roddy.execution.jobs.BEJobResult
import de.dkfz.roddy.config.ResourceSet
import de.dkfz.roddy.execution.BEExecutionService
import de.dkfz.roddy.tools.LoggerWrapper
Expand Down Expand Up @@ -152,19 +152,19 @@ abstract class BatchEuphoriaJobManager<C extends Command> {
* @param job
* @return
*/
abstract JobResult runJob(BEJob job)
abstract BEJobResult runJob(BEJob job)

/**
* Called by the execution service after a command was executed.
*/
JobResult extractAndSetJobResultFromExecutionResult(Command command, ExecutionResult res) {
JobResult jobResult
BEJobResult extractAndSetJobResultFromExecutionResult(Command command, ExecutionResult res) {
BEJobResult jobResult
if (res.successful) {
String exID = parseJobID(res.resultLines[0])
def job = command.getJob()
def jobDependencyID = createJobDependencyID(job, exID)
command.setExecutionID(jobDependencyID)
jobResult = new JobResult(command, jobDependencyID, res.successful, false, job.tool, job.parameters, job.parentJobs as List<BEJob>)
jobResult = new BEJobResult(command, jobDependencyID, res.successful, false, job.tool, job.parameters, job.parentJobs as List<BEJob>)
job.setRunResult(jobResult)
}
return jobResult
Expand Down Expand Up @@ -209,7 +209,7 @@ abstract class BatchEuphoriaJobManager<C extends Command> {

abstract GenericJobInfo parseGenericJobInfo(String command)

abstract JobResult convertToArrayResult(BEJob arrayChildJob, JobResult parentJobsResult, int arrayIndex)
abstract BEJobResult convertToArrayResult(BEJob arrayChildJob, BEJobResult parentJobsResult, int arrayIndex)

abstract void updateJobStatus()

Expand Down
4 changes: 4 additions & 0 deletions src/main/groovy/de/dkfz/roddy/execution/jobs/FakeBEJob.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ class FakeBEJob extends BEJob {
super("Fakejob", null,null, "", null, [], [:], [], [], null)
}

public FakeBEJob(Object context) {
this()
}

public FakeBEJob(InfoObject context) {
this()
}
Expand Down
7 changes: 4 additions & 3 deletions ...roovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/BatchEuphoriaJobManagerAdapter.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ package de.dkfz.roddy.execution.jobs.cluster.lsf.rest

import de.dkfz.roddy.config.ResourceSet
import de.dkfz.roddy.execution.BEExecutionService
import de.dkfz.roddy.execution.jobs.BEJobResult
import de.dkfz.roddy.execution.jobs.cluster.ClusterJobManager
import de.dkfz.roddy.execution.jobs.Command
import de.dkfz.roddy.execution.jobs.GenericJobInfo
Expand All @@ -16,7 +17,7 @@ import de.dkfz.roddy.execution.jobs.JobManagerCreationParameters
import de.dkfz.roddy.execution.jobs.JobState
import de.dkfz.roddy.execution.jobs.ProcessingCommands
import de.dkfz.roddy.execution.jobs.BEJobDependencyID
import de.dkfz.roddy.execution.jobs.JobResult
import de.dkfz.roddy.execution.jobs.BEJobResult
import sun.reflect.generics.reflectiveObjects.NotImplementedException

/**
Expand All @@ -33,7 +34,7 @@ class BatchEuphoriaJobManagerAdapter extends ClusterJobManager {
}

@Override
JobResult runJob(BEJob job) {
BEJobResult runJob(BEJob job) {
return null
}

Expand Down Expand Up @@ -69,7 +70,7 @@ class BatchEuphoriaJobManagerAdapter extends ClusterJobManager {
}

@Override
JobResult convertToArrayResult(BEJob arrayChildJob, JobResult parentJobsResult, int arrayIndex) {
BEJobResult convertToArrayResult(BEJob arrayChildJob, BEJobResult parentJobsResult, int arrayIndex) {
throw new NotImplementedException()
}
// needed
Expand Down
6 changes: 3 additions & 3 deletions src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/lsf/rest/LSFRestJobManager.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import de.dkfz.roddy.execution.jobs.cluster.pbs.PBSResourceProcessingCommand
import de.dkfz.roddy.execution.jobs.GenericJobInfo
import de.dkfz.roddy.execution.jobs.BEJob
import de.dkfz.roddy.execution.jobs.JobManagerCreationParameters
import de.dkfz.roddy.execution.jobs.JobResult
import de.dkfz.roddy.execution.jobs.BEJobResult
import de.dkfz.roddy.execution.jobs.JobState
import de.dkfz.roddy.execution.jobs.ProcessingCommands
import de.dkfz.roddy.tools.BufferUnit
Expand Down Expand Up @@ -55,7 +55,7 @@ class LSFRestJobManager extends BatchEuphoriaJobManagerAdapter {


@Override
de.dkfz.roddy.execution.jobs.JobResult runJob(BEJob job) {
BEJobResult runJob(BEJob job) {
submitJob(job)
return job.runResult
}
Expand Down Expand Up @@ -175,7 +175,7 @@ class LSFRestJobManager extends BatchEuphoriaJobManagerAdapter {
RestResult result = restExecutionService.execute(new RestCommand(URI_JOB_SUBMIT, requestBody.toString(), headers, RestCommand.HttpMethod.HTTPPOST)) as RestResult
if (result.statusCode == 200) {
logger.postAlwaysInfo("status code: " + result.statusCode + " result:" + new XmlSlurper().parseText(result.body))
job.setRunResult(new JobResult(job.lastCommand, new PBSJobDependencyID(job, new XmlSlurper().parseText(result.body).text()), true, job.tool, job.parameters, job.parentJobs as List<BEJob>))
job.setRunResult(new BEJobResult(job.lastCommand, new PBSJobDependencyID(job, new XmlSlurper().parseText(result.body).text()), true, job.tool, job.parameters, job.parentJobs as List<BEJob>))
} else {
logger.postAlwaysInfo("status code: " + result.statusCode + " result: " + result.body)
}
Expand Down
5 changes: 3 additions & 2 deletions src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSCommand.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,10 +217,11 @@ class PBSCommand extends Command {
String assembleVariableExportString() {
StringBuilder qsubCall = new StringBuilder()


if (job.getParameterFile()) {
qsubCall << getVariablesParameter() << "PARAMETER_FILE=" << job.getParameterFile()
qsubCall << getVariablesParameter() << "CONFIG_FILE=" << job.parameters["CONFIG_FILE"] << ",PARAMETER_FILE=" << job.getParameterFile()
} else {
def finalParameters = job.finalParameters()
List<String> finalParameters = job.finalParameters()
if (finalParameters)
qsubCall << getVariablesParameter() << finalParameters.join(",")
}
Expand Down
7 changes: 4 additions & 3 deletions src/main/groovy/de/dkfz/roddy/execution/jobs/cluster/pbs/PBSJobManager.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,15 @@ package de.dkfz.roddy.execution.jobs.cluster.pbs

import de.dkfz.roddy.config.ResourceSet
import de.dkfz.roddy.execution.BEExecutionService
import de.dkfz.roddy.execution.jobs.BEJobResult
import de.dkfz.roddy.execution.jobs.cluster.ClusterJobManager
import de.dkfz.roddy.StringConstants
import de.dkfz.roddy.execution.io.ExecutionResult
import de.dkfz.roddy.execution.jobs.GenericJobInfo
import de.dkfz.roddy.execution.jobs.BEJob
import de.dkfz.roddy.execution.jobs.BEJobDependencyID
import de.dkfz.roddy.execution.jobs.JobManagerCreationParameters
import de.dkfz.roddy.execution.jobs.JobResult
import de.dkfz.roddy.execution.jobs.BEJobResult
import de.dkfz.roddy.execution.jobs.JobState
import de.dkfz.roddy.execution.jobs.ProcessingCommands
import de.dkfz.roddy.tools.BufferUnit
Expand Down Expand Up @@ -93,7 +94,7 @@ class PBSJobManager extends ClusterJobManager<PBSCommand> {
}

@Override
JobResult runJob(BEJob job) {
BEJobResult runJob(BEJob job) {
def command = createCommand(job)
def executionResult = executionService.execute(command)
extractAndSetJobResultFromExecutionResult(command, executionResult)
Expand Down Expand Up @@ -284,7 +285,7 @@ class PBSJobManager extends ClusterJobManager<PBSCommand> {
}

@Override
JobResult convertToArrayResult(BEJob arrayChildJob, JobResult parentJobsResult, int arrayIndex) {
BEJobResult convertToArrayResult(BEJob arrayChildJob, BEJobResult parentJobsResult, int arrayIndex) {
return null
}

Expand Down
8 changes: 4 additions & 4 deletions ...dy/execution/jobs/direct/synchronousexecution/DirectSynchronousExecutionJobManager.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import de.dkfz.roddy.config.ResourceSet
import de.dkfz.roddy.execution.BEExecutionService
import de.dkfz.roddy.execution.jobs.*;
import de.dkfz.roddy.execution.jobs.BEJobDependencyID;
import de.dkfz.roddy.execution.jobs.JobResult;
import de.dkfz.roddy.execution.jobs.BEJobResult;
import de.dkfz.roddy.tools.LoggerWrapper

/**
Expand Down Expand Up @@ -75,7 +75,7 @@ public class DirectSynchronousExecutionJobManager extends BatchEuphoriaJobManage
}

@Override
public JobResult convertToArrayResult(BEJob arrayChildJob, JobResult parentJobsResult, int arrayIndex) {
public BEJobResult convertToArrayResult(BEJob arrayChildJob, BEJobResult parentJobsResult, int arrayIndex) {
throw new RuntimeException("Not implemented yet! " + this.getClass().getName() + ".convertToArrayResult()");
}

Expand All @@ -90,7 +90,7 @@ public class DirectSynchronousExecutionJobManager extends BatchEuphoriaJobManage
// }

// @Override
// public JobResult convertToArrayResult(BEJob arrayChildJob, JobResult parentJobsResult, int arrayIndex) {
// public BEJobResult convertToArrayResult(BEJob arrayChildJob, BEJobResult parentJobsResult, int arrayIndex) {
// throw new RuntimeException("Not implemented yet! " + this.getClass().getName() + ".convertToArrayResult()");
// }

Expand Down Expand Up @@ -179,7 +179,7 @@ public class DirectSynchronousExecutionJobManager extends BatchEuphoriaJobManage
}

@Override
public JobResult runJob(BEJob job) {
public BEJobResult runJob(BEJob job) {
return null;
}

Expand Down

0 comments on commit 55cfdc5

Please sign in to comment.