ShanoirUploader: auto test import + perf optimization QueryPACSService

shanoir-ng-back/pom.xml

@@ -49,7 +49,7 @@ along with this program. If not, see https://www.gnu.org/licenses/gpl-3.0.html
 		<java.version>17</java.version>
 		<mapstruct.version>1.5.3.Final</mapstruct.version>
 		<keycloak.version>22.0.1</keycloak.version>
-		<dcm4che.version>5.30.0</dcm4che.version>
+		<dcm4che.version>5.31.0</dcm4che.version>
 	</properties>
 
 	<dependencyManagement>

shanoir-ng-datasets/src/main/java/org/shanoir/ng/datasetacquisition/model/mr/ContrastAgentUsed.java

@@ -41,7 +41,10 @@ public enum ContrastAgentUsed {
 	GADOVIST(7),
 
 	// Clariscan
-	CLARISCAN(8);
+	CLARISCAN(8),
+
+	// Dotarem
+	DOTAREM(9);
 
 	private int id;
 

shanoir-ng-import/src/main/java/org/shanoir/ng/importer/dicom/ImagesCreatorAndDicomFileAnalyzerService.java

@@ -228,7 +228,6 @@ private void processOneDicomFileForAllInstances(File dicomFile, List<Image> imag
 			if (DicomSerieAndInstanceAnalyzer.checkInstanceIsIgnored(attributes)) {
 				// do nothing here as instances list will be emptied after split between images and non-images
 			} else {
-				// divide here between non-images and images, non-images at first
 				Image image = new Image();
 				/**
 				 * Attention: the path of each image is always relative: either to the temporary folder created
@@ -242,7 +241,7 @@ private void processOneDicomFileForAllInstances(File dicomFile, List<Image> imag
 		} catch (IOException iOE) {
 			throw iOE;
 		} catch (Exception e) {
-			LOG.error("Error while processing DICOM file: " + dicomFile.getAbsolutePath());
+			LOG.error("Error while processing DICOM file, one for entire serie: " + dicomFile.getAbsolutePath());
 			throw e;
 		}
 	}

shanoir-ng-studies/src/main/java/org/shanoir/ng/study/controler/StudyApi.java

@@ -30,7 +30,6 @@
 import org.shanoir.ng.study.dua.DataUserAgreement;
 import org.shanoir.ng.study.model.Study;
 import org.shanoir.ng.study.model.StudyUser;
-import org.springframework.core.io.Resource;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.prepost.PostAuthorize;
 import org.springframework.security.access.prepost.PreAuthorize;

shanoir-ng-studies/src/main/java/org/shanoir/ng/study/controler/StudyApiController.java

@@ -21,16 +21,12 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.text.SimpleDateFormat;
-import java.time.LocalDate;
-import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
-import java.util.Date;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.io.FileUtils;
-import org.joda.time.format.DateTimeFormat;
 import org.shanoir.ng.shared.core.model.IdName;
 import org.shanoir.ng.shared.error.FieldErrorMap;
 import org.shanoir.ng.shared.event.ShanoirEvent;
@@ -210,6 +206,7 @@ public ResponseEntity<StudyDTO> saveNewStudy(@RequestBody final Study study, fin
 
 		Study createdStudy;
 		try {
+			addCurrentUserAsStudyUserIfEmptyStudyUsers(study);
 			createdStudy = studyService.create(study);
 			eventService.publishEvent(new ShanoirEvent(ShanoirEventType.CREATE_STUDY_EVENT,
 					createdStudy.getId().toString(), KeycloakUtil.getTokenUserId(), "", ShanoirEvent.SUCCESS));
@@ -220,6 +217,19 @@ public ResponseEntity<StudyDTO> saveNewStudy(@RequestBody final Study study, fin
 		return new ResponseEntity<>(studyMapper.studyToStudyDTO(createdStudy), HttpStatus.OK);
 	}
 
+	private void addCurrentUserAsStudyUserIfEmptyStudyUsers(final Study study) {
+		if (study.getStudyUserList() == null) {
+			List<StudyUser> studyUserList = new ArrayList<StudyUser>();
+			StudyUser studyUser = new StudyUser();
+			studyUser.setStudy(study);
+			studyUser.setUserId(KeycloakUtil.getTokenUserId());
+			studyUser.setUserName(KeycloakUtil.getTokenUserName());
+			studyUser.setStudyUserRights(Arrays.asList(StudyUserRight.CAN_SEE_ALL, StudyUserRight.CAN_DOWNLOAD, StudyUserRight.CAN_IMPORT, StudyUserRight.CAN_ADMINISTRATE));
+			studyUserList.add(studyUser);
+			study.setStudyUserList(studyUserList);
+		}
+	}
+
 	@Override
 	public ResponseEntity<StudyStorageVolumeDTO> getDetailedStorageVolume(@PathVariable("studyId") final Long studyId) throws RestServiceException {
 		StudyStorageVolumeDTO dto = studyService.getDetailedStorageVolume(studyId);

shanoir-ng-studies/src/main/java/org/shanoir/ng/study/service/StudyServiceImpl.java

@@ -118,7 +118,6 @@ public class StudyServiceImpl implements StudyService {
 	@Autowired
 	private SubjectStudyRepository subjectStudyRepository;
 
-
 	@Override
 	public void deleteById(final Long id) throws EntityNotFoundException {
 		final Study study = studyRepository.findById(id).orElse(null);
@@ -154,8 +153,10 @@ public Study create(final Study study) throws MicroServiceCommunicationException
 			}
 		}
 
-		for (SubjectStudy subjectStudy : study.getSubjectStudyList()) {
-			subjectStudy.setStudy(study);
+		if (study.getSubjectStudyList() != null) {
+			for (SubjectStudy subjectStudy : study.getSubjectStudyList()) {
+				subjectStudy.setStudy(study);
+			}
 		}
 
 		if (study.getTags() != null) {
@@ -185,15 +186,16 @@ public Study create(final Study study) throws MicroServiceCommunicationException
 			}
 		}
 
-		List<SubjectStudy> subjectStudyListSave = new ArrayList<SubjectStudy>(study.getSubjectStudyList());
+		List<SubjectStudy> subjectStudyListSave = null;
+		if (study.getSubjectStudyList() != null) {
+			subjectStudyListSave = new ArrayList<SubjectStudy>(study.getSubjectStudyList());
+		}
 		Map<Long, List<SubjectStudyTag>> subjectStudyTagSave = new HashMap<>();
 		study.setSubjectStudyList(null);
 		Study studyDb = studyRepository.save(study);
-		//studyDb.setSubjectStudyList(new ArrayList<SubjectStudy>());
 
 		if (subjectStudyListSave != null) {
 			updateTags(subjectStudyListSave, studyDb.getTags());
-			//ListDependencyUpdate.updateWith(studyDb.getSubjectStudyList(), subjectStudyListSave);
 			studyDb.setSubjectStudyList(new ArrayList<>());
 			for (SubjectStudy subjectStudy : subjectStudyListSave) {
 				SubjectStudy newSubjectStudy = new SubjectStudy();
@@ -203,7 +205,6 @@ public Study create(final Study study) throws MicroServiceCommunicationException
 				newSubjectStudy.setSubjectType(subjectStudy.getSubjectType());
 				newSubjectStudy.setStudy(studyDb);
 				subjectStudyTagSave.put(subjectStudy.getSubject().getId(), subjectStudy.getSubjectStudyTags());
-				//newSubjectStudy.setSubjectStudyTags(subjectStudy.getSubjectStudyTags());
 				studyDb.getSubjectStudyList().add(newSubjectStudy);
 			}
 			studyDb = studyRepository.save(studyDb);

shanoir-uploader/src/main/java/org/shanoir/uploader/ShUpConfig.java

@@ -70,6 +70,8 @@ public class ShUpConfig {
 
 	public static final String RANDOM_SEED = "random.seed";
 
+	public static final String PROFILE = "profile";
+
 	/**
 	 * Static variables
 	 */

shanoir-uploader/src/main/java/org/shanoir/uploader/action/DownloadOrCopyRunnable.java

@@ -66,12 +66,12 @@ public void run() {
 
 			/**
 			 * 2. Fill MRI information into serie from first DICOM file of each serie
-			 * This has already been done for CD/DVD import, but not yet here for PACS
+			 * This has already been done for CD/DVD import, but not yet here for PACS.
 			 */
 			if (this.isFromPACS) {
 				for (Iterator iterator = selectedSeries.iterator(); iterator.hasNext();) {
 					SerieTreeNode serieTreeNode = (SerieTreeNode) iterator.next();
-					dicomFileAnalyzer.getAdditionalMetaDataFromFirstInstanceOfSerie(filePathDicomDir, serieTreeNode.getSerie(), null, isFromPACS);
+					dicomFileAnalyzer.getAdditionalMetaDataFromFirstInstanceOfSerie(uploadFolder.getAbsolutePath(), serieTreeNode.getSerie(), null, isFromPACS);
 				}
 			}
 		} catch (FileNotFoundException e) {

shanoir-uploader/src/main/java/org/shanoir/uploader/action/FindDicomActionListener.java

@@ -246,7 +246,7 @@ private void fillMediaWithPatients(Media media, final List<Patient> patients) {
 		for (Iterator iterator = patients.iterator(); iterator.hasNext();) {
 			Patient patient = (Patient) iterator.next();
 			final PatientTreeNode patientTreeNode = media.initChildTreeNode(patient);
-			logger.info("Patient info read from DICOMDIR: " + patient.toString());
+			logger.info("Patient info read: " + patient.toString());
 			// add patients
 			media.addTreeNode(patient.getPatientID(), patientTreeNode);
 			List<Study> studies = patient.getStudies();

shanoir-uploader/src/main/java/org/shanoir/uploader/action/ImportFinishRunnable.java

@@ -60,13 +60,12 @@ public void run() {
 		boolean anonymizationSuccess = false;
 		try {
 			String anonymizationProfile = ShUpConfig.profileProperties.getProperty(ANONYMIZATION_PROFILE);
-			anonymizationSuccess = anonymizer.anonymize(uploadFolder, anonymizationProfile, subjectName);
+			anonymizationSuccess = anonymizer.pseudonymize(uploadFolder, anonymizationProfile, subjectName);
 		} catch (IOException e) {
 			logger.error(uploadFolder.getName() + ": " + e.getMessage(), e);
 		}
 
 		if (anonymizationSuccess) {
-			logger.info(uploadFolder.getName() + ": DICOM files successfully anonymized.");
 			/**
 			 * Write import-job.json to disk
 			 */

shanoir-uploader/src/main/java/org/shanoir/uploader/action/init/InitialStartupState.java

@@ -49,7 +49,9 @@ public class InitialStartupState implements State {
 	private static final String SU_V6_0_3 = ".su_v6.0.3";
 
 	private static final String SU_V6_0_4 = ".su_v6.0.4";
-
+
+	private static final String SU_V7_0_1 = ".su_v7.0.1";
+
 	public void load(StartupStateContext context) throws Exception {
 		initShanoirUploaderFolder();
 		initLogging();
@@ -60,8 +62,8 @@ public void load(StartupStateContext context) throws Exception {
 		logger.info(System.getProperty("java.vendor.url"));
 		logger.info(System.getProperty("java.version"));
         InetAddress inetAddress = InetAddress.getLocalHost();
-        logger.info("IP Address:- " + inetAddress.getHostAddress());
-        logger.info("Host Name:- " + inetAddress.getHostName());
+        logger.info("IP Address: " + inetAddress.getHostAddress());
+        logger.info("Host Name: " + inetAddress.getHostName());
 		 // Disable http request to check for quartz upload
 		System.setProperty("org.quartz.scheduler.skipUpdateCheck", "true");
 		System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");
@@ -70,6 +72,7 @@ public void load(StartupStateContext context) throws Exception {
 		initLanguage();
 		copyPseudonymus();
 		initProfiles();
+		initProfile();
 		initStartupDialog(context);
 		context.setState(new ProxyConfigurationState());
 		context.nextState();
@@ -78,7 +81,8 @@ public void load(StartupStateContext context) throws Exception {
 	private void doMigration() throws IOException {
 		// as properties, that exist already are not replaced/changed, start with the last version before,
 		// as considered as more important
-		// overwrite with properties from ShanoirUploader v6.0.4 or v6.0.3, if existing
+		// overwrite with properties from ShanoirUploader v7.0.1, v6.0.4 or v6.0.3, if existing
+		migrateFromVersion(SU_V7_0_1);
 		migrateFromVersion(SU_V6_0_4);
 		migrateFromVersion(SU_V6_0_3);
 		// migrate properties from ShanoirUploader v5.2
@@ -91,11 +95,11 @@ private void migrateFromVersion(String version) throws IOException {
 		final File shanoirUploaderFolderForVersion = new File(shanoirUploaderFolderPathForVersion);
 		boolean shanoirUploaderFolderExistsForVersion = shanoirUploaderFolderForVersion.exists();
 		if (shanoirUploaderFolderExistsForVersion) {
-			logger.info("Start migrating properties from version " + version + " (.su == v5.2) of ShUp.");
+			logger.info("Start migrating properties from version " + version + " of ShUp.");
 			copyPropertiesFile(shanoirUploaderFolderForVersion, ShUpConfig.shanoirUploaderFolder, ShUpConfig.LANGUAGE_PROPERTIES);
 			copyPropertiesFile(shanoirUploaderFolderForVersion, ShUpConfig.shanoirUploaderFolder, ShUpConfig.PROXY_PROPERTIES);
 			copyPropertiesFile(shanoirUploaderFolderForVersion, ShUpConfig.shanoirUploaderFolder, ShUpConfig.DICOM_SERVER_PROPERTIES);
-			logger.info("Finished migrating properties from version " + version + " (.su == v5.2) of ShUp: language, proxy, dicom_server.");
+			logger.info("Finished migrating properties from version " + version + " of ShUp: language, proxy, dicom_server.");
 		}
 	}
 
@@ -297,4 +301,10 @@ private void initLanguage() {
 		}
 	}
 
+	private void initProfile() throws FileNotFoundException, IOException {
+		String profile = ShUpConfig.basicProperties.getProperty(ShUpConfig.PROFILE);
+		if (profile != null && !profile.isEmpty()) {
+			ShUpConfig.profileSelected = profile;
+		}
+	}
 }

shanoir-uploader/src/main/java/org/shanoir/uploader/action/init/ProxyConfigurationState.java

@@ -37,7 +37,7 @@ public void load(StartupStateContext context) {
 		switch (httpResponseCode){
 			case 200 :
 				context.getShUpStartupDialog().updateStartupText("\n" + ShUpConfig.resourceBundle.getString("shanoir.uploader.startup.test.proxy.success"));
-				context.setState(new SelectProfileManualConfigurationState());
+				context.setState(new SelectProfileConfigurationState());
 				context.nextState();
 				break;
 			default:

shanoir-uploader/src/main/java/org/shanoir/uploader/action/init/SelectProfileConfigurationState.java

@@ -0,0 +1,25 @@
+package org.shanoir.uploader.action.init;
+
+
+import org.apache.log4j.Logger;
+import org.shanoir.uploader.ShUpConfig;
+
+public class SelectProfileConfigurationState implements State {
+
+	private static Logger logger = Logger.getLogger(SelectProfileConfigurationState.class);
+
+	public void load(StartupStateContext context) {
+		if(ShUpConfig.profileSelected == null) {
+			context.setState(new SelectProfileManualConfigurationState());
+			context.nextState();
+		} else {
+			logger.info("Profile found in basic.properties. Used as default: " + ShUpConfig.profileSelected);
+			SelectProfilePanelActionListener actionListener = new SelectProfilePanelActionListener(null, null);
+			actionListener.configureSelectedProfile(ShUpConfig.profileSelected);
+			context.getShUpStartupDialog().updateStartupText("\nProfile: " + ShUpConfig.profileSelected);
+			context.setState(new AuthenticationConfigurationState());
+			context.nextState();
+		}
+	}
+
+}

shanoir-uploader/src/main/java/org/shanoir/uploader/action/init/SelectProfilePanelActionListener.java

@@ -30,14 +30,16 @@ public SelectProfilePanelActionListener(SelectProfileConfigurationPanel selectPr
 	public void actionPerformed(ActionEvent e) {
 		String selectedProfile = (String) selectProfilePanel.selectProfileCB.getSelectedItem();
 		ShUpConfig.profileSelected = selectedProfile;
+		configureSelectedProfile(selectedProfile);
+		sSC.nextState();
+	}
+
+	public void configureSelectedProfile(String selectedProfile) {
 		String filePath = File.separator + ShUpConfig.PROFILE_DIR + selectedProfile;
 		ShUpConfig.profileDirectory = new File(ShUpConfig.shanoirUploaderFolder, filePath);
 		logger.info("Profile directory set to: " + ShUpConfig.profileDirectory.getAbsolutePath());
 		File profilePropertiesFile = new File(ShUpConfig.profileDirectory, ShUpConfig.PROFILE_PROPERTIES);
 		loadPropertiesFromFile(profilePropertiesFile, ShUpConfig.profileProperties);
-
-		ShUpConfig.encryption.decryptIfEncryptedString(profilePropertiesFile,
-				ShUpConfig.profileProperties, "shanoir.server.user.password");
 		logger.info("Profile " + selectedProfile + " successfully initialized.");
 
 		File keycloakJson = new File(ShUpConfig.profileDirectory, ShUpConfig.KEYCLOAK_JSON);
@@ -47,7 +49,7 @@ public void actionPerformed(ActionEvent e) {
 		} else {
 			logger.error("Error: missing keycloak.json! Connection with sh-ng will not work.");
 			return;
-	}
+		}
 
 		// check if pseudonymus has been copied in case of true
 		if (Boolean.parseBoolean(ShUpConfig.profileProperties.getProperty(ShUpConfig.MODE_PSEUDONYMUS))) {
@@ -72,7 +74,6 @@ public void actionPerformed(ActionEvent e) {
 				return;
 			}
 		}
-		sSC.nextState();
 	}
 
 	private void loadPropertiesFromFile(final File propertiesFile, final Properties properties) {

shanoir-uploader/src/main/java/org/shanoir/uploader/dicom/DicomServerClient.java

@@ -42,6 +42,7 @@ public class DicomServerClient implements IDicomServerClient {
 	private File workFolder;
 
 	public DicomServerClient(final Properties dicomServerProperties, final File workFolder) {
+		logger.info("New DicomServerClient created with properties: " + dicomServerProperties.toString());
 		config.initWithPropertiesFile(dicomServerProperties);
 		this.workFolder = workFolder;
 		// Initialize connection configuration parameters here: to be used for all queries
@@ -126,17 +127,25 @@ public boolean accept(File dir, String name) {
 							}
 						}
 					};
-					File[] newFileNames = uploadFolder.listFiles(oldFileNamesAndDICOMFilter);
-					logger.debug("newFileNames: " + newFileNames.length);
-					for (int i = 0; i < newFileNames.length; i++) {
-						fileNamesForSerie.add(newFileNames[i].getName());
+					File serieFolder = new File (uploadFolder.getAbsolutePath() + File.separator + seriesInstanceUID);
+					if (serieFolder.exists()) {
+						File[] newFileNames = serieFolder.listFiles(oldFileNamesAndDICOMFilter);
+						logger.debug("newFileNames: " + newFileNames.length);
+						for (int i = 0; i < newFileNames.length; i++) {
+							fileNamesForSerie.add(newFileNames[i].getName());
+						}
+						serieTreeNode.setFileNames(fileNamesForSerie);
+						retrievedDicomFiles.addAll(fileNamesForSerie);
+						oldFileNames.addAll(fileNamesForSerie);
+						logger.info(uploadFolder.getName() + ":\n\n Download of " + fileNamesForSerie.size()
+								+ " DICOM files for serie " + seriesInstanceUID + ": " + serieTreeNode.getDisplayString()
+								+ " was successful.\n\n");
+					} else {
+						logger.error(uploadFolder.getName() + ":\n\n Download of " + fileNamesForSerie.size()
+						+ " DICOM files for serie " + seriesInstanceUID + ": " + serieTreeNode.getDisplayString()
+						+ " has failed.\n\n");
+						return null;
 					}
-					serieTreeNode.setFileNames(fileNamesForSerie);
-					retrievedDicomFiles.addAll(fileNamesForSerie);
-					oldFileNames.addAll(fileNamesForSerie);
-					logger.info(uploadFolder.getName() + ":\n\n Download of " + fileNamesForSerie.size()
-							+ " DICOM files for serie " + seriesInstanceUID + ": " + serieTreeNode.getDisplayString()
-							+ " was successful.\n\n");
 				} else {
 					logger.error(uploadFolder.getName() + ":\n\n Download of " + fileNamesForSerie.size()
 							+ " DICOM files for serie " + seriesInstanceUID + ": " + serieTreeNode.getDisplayString()