Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SYNC-350 : Add support to synchronize complex obs data #39

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

SYNC-350 : Add support to synchronize complex obs data #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a636a49
SYNC-350 : Add support to synchronize complex obs data
mseaton Nov 17, 2020
c083e91
Complex obs test with larger binary file (fails with OOM error)
mseaton Nov 18, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions api/src/main/java/org/openmrs/module/sync/SyncComplexObsUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
/**
* The contents of this file are subject to the OpenMRS Public License
* Version 1.0 (the "License"); you may not use this file except in
* compliance with the License. You may obtain a copy of the License at
* http://license.openmrs.org
*
* Software distributed under the License is distributed on an "AS IS"
* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
* License for the specific language governing rights and limitations
* under the License.
*
* Copyright (C) OpenMRS, LLC. All Rights Reserved.
*/
package org.openmrs.module.sync;

import java.io.File;

import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.openmrs.GlobalProperty;
import org.openmrs.api.GlobalPropertyListener;
import org.openmrs.api.context.Context;
import org.openmrs.util.OpenmrsConstants;
import org.openmrs.util.OpenmrsUtil;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;

/**
* Utility class for working with complex obs
* This is implemented as a GlobalPropertyListener and ApplicationContextAware in order to ensure that global property
* values can be accessed reliably without the need to hit the database, as these are accessed by the Interceptor and
* can cause issues by forcing a flush upon query.
*/
public class SyncComplexObsUtil implements GlobalPropertyListener, ApplicationContextAware {

private static final Log log = LogFactory.getLog(SyncComplexObsUtil.class);

public static final String GP_NAME = OpenmrsConstants.GLOBAL_PROPERTY_COMPLEX_OBS_DIR;
public static final String DEFAULT_VAL = "complex_obs";
public static String COMPLEX_OBS_DIR = DEFAULT_VAL;

public static final String VALUE_COMPLEX = "valueComplex";
public static final String COMPLEX_DATA = "complexData";

@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
COMPLEX_OBS_DIR = Context.getAdministrationService().getGlobalProperty(GP_NAME, DEFAULT_VAL);
}

@Override
public boolean supportsPropertyName(String s) {
return GP_NAME.equals(s);
}

@Override
public void globalPropertyChanged(GlobalProperty globalProperty) {
log.debug("Global property <" + globalProperty.getProperty() + "> changed");
COMPLEX_OBS_DIR = globalProperty.getPropertyValue();
}

@Override
public void globalPropertyDeleted(String s) {
log.debug("Global property <" + s + "> deleted");
COMPLEX_OBS_DIR = DEFAULT_VAL;
}

/**
* Adapted from AbstractHandler
*/
public static File getComplexDataFile(String valueComplex) {
File ret = null;
try {
if (StringUtils.isNotBlank(valueComplex)) {
String[] names = valueComplex.split("\\|");
String filename = names.length < 2 ? names[0] : names[names.length - 1];
File dir = OpenmrsUtil.getDirectoryInApplicationDataDirectory(COMPLEX_OBS_DIR);
if (!dir.exists()) {
dir.mkdirs();
}
ret = new File(dir, filename);
}
}
catch (Exception e) {
log.warn("Error trying to retrieve complex data file for obs: " + valueComplex, e);
}
return ret;
}

public static String getComplexDataEncoded(String valueComplex) {
String ret = null;
try {
File complexObsFile = getComplexDataFile(valueComplex);
if (complexObsFile != null && complexObsFile.exists()) {
log.debug("Found a complex obs file at: " + complexObsFile);
byte[] fileBytes = FileUtils.readFileToByteArray(complexObsFile);
ret = Base64.encodeBase64String(fileBytes);
}
}
catch (Exception e) {
log.warn("Error trying to retrieve complex data for obs: " + valueComplex, e);
}
return ret;
}

public static void setComplexDataForObs(String valueComplex, String encodedData) {
try {
File complexObsFile = getComplexDataFile(valueComplex);
if (complexObsFile != null && StringUtils.isNotBlank(encodedData)) {
byte[] bytes = Base64.decodeBase64(encodedData);
FileUtils.writeByteArrayToFile(complexObsFile, bytes);
}
}
catch (Exception e) {
log.warn("Error writing complex data for obs: " + valueComplex, e);
}
}
}
14 changes: 11 additions & 3 deletions api/src/main/java/org/openmrs/module/sync/api/db/hibernate/HibernateSyncInterceptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import org.openmrs.PersonAttributeType;
import org.openmrs.User;
import org.openmrs.api.context.Context;
import org.openmrs.module.sync.SyncComplexObsUtil;
import org.openmrs.module.sync.SyncException;
import org.openmrs.module.sync.SyncItem;
import org.openmrs.module.sync.SyncItemKey;
Expand Down Expand Up @@ -618,9 +619,16 @@ private void addProperty(HashMap<String, PropertyClassValue> values, OpenmrsObje
Normalizer n;
String propertyTypeName = propertyType.getName();
if ((n = SyncUtil.getNormalizer(propertyTypeName)) != null) {
// Handle safe types like
// boolean/String/integer/timestamp via Normalizers
values.put(propertyName, new PropertyClassValue(propertyTypeName, n.toString(propertyValue)));
// Handle safe types like boolean/String/integer/timestamp via Normalizers
String strValue = n.toString(propertyValue);
values.put(propertyName, new PropertyClassValue(propertyTypeName, strValue));
// If there is a valueComplex defined, ensure complexData is also added to the sync item
if (entity instanceof Obs && SyncComplexObsUtil.VALUE_COMPLEX.equalsIgnoreCase(propertyName)) {
String encoded = SyncComplexObsUtil.getComplexDataEncoded(strValue);
if (encoded != null) {
values.put(SyncComplexObsUtil.COMPLEX_DATA, new PropertyClassValue("byte[]", encoded));
}
}
} else if ((n = SyncUtil.getNormalizer(propertyValue.getClass())) != null) {
values.put(propertyName, new PropertyClassValue(propertyValue.getClass().getName(), n.toString(propertyValue)));
} else if (propertyType.isCollectionType() && (n = isCollectionOfSafeTypes(entity, propertyName)) != null) {
Expand Down
36 changes: 28 additions & 8 deletions api/src/main/java/org/openmrs/module/sync/api/impl/SyncIngestServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,18 @@
*/
package org.openmrs.module.sync.api.impl;

import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hibernate.collection.internal.PersistentSet;
import org.openmrs.Concept;
import org.openmrs.Obs;
import org.openmrs.OpenmrsObject;
import org.openmrs.PatientIdentifier;
import org.openmrs.Person;
Expand All @@ -28,6 +35,7 @@
import org.openmrs.api.context.Context;
import org.openmrs.api.db.SerializedObject;
import org.openmrs.module.ModuleUtil;
import org.openmrs.module.sync.SyncComplexObsUtil;
import org.openmrs.module.sync.SyncConstants;
import org.openmrs.module.sync.SyncItem;
import org.openmrs.module.sync.SyncItemState;
Expand All @@ -49,14 +57,9 @@
import org.openmrs.util.OpenmrsConstants;
import org.openmrs.util.OpenmrsUtil;
import org.springframework.transaction.annotation.Transactional;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

@Transactional
public class SyncIngestServiceImpl implements SyncIngestService {

Expand Down Expand Up @@ -535,15 +538,32 @@ private OpenmrsObject processOpenmrsObject(OpenmrsObject o, SyncItem item, Strin
//if we are doing insert/update:
//1. set serialized props state
//2. force it down the hibernate's throat with help of openmrs api

// If this node contains complexData, keep track of this separately to save rather than set on obs
String complexData = null;

for ( int i = 0; i < nodes.getLength(); i++ ) {
try {
log.debug("trying to set property: " + nodes.item(i).getNodeName() + " in className " + className);
SyncUtil.setProperty(o, nodes.item(i), allFields);
Node node = nodes.item(i);
String nodeName = node.getNodeName();
log.debug("trying to set property: " + nodeName + " in className " + className);
if (o instanceof Obs && "complexData".equalsIgnoreCase(nodeName)) {
complexData = node.getTextContent();
}
else {
SyncUtil.setProperty(o, node, allFields);
}
} catch ( Exception e ) {
log.error("Error when trying to set " + nodes.item(i).getNodeName() + ", which is a " + className, e);
throw new SyncIngestException(e, SyncConstants.ERROR_ITEM_UNSET_PROPERTY, nodes.item(i).getNodeName() + "," + className + "," + e.getMessage(), itemContent,null);
}
}

// If this is an Obs and complex data was found in the sync record, save this complex data
if (o instanceof Obs && complexData != null) {
Obs obs = (Obs) o;
SyncComplexObsUtil.setComplexDataForObs(obs.getValueComplex(), complexData);
}

// now try to commit this fully inflated object
try {
Expand Down
Loading