From 129cf9c9b224596a65fc275be354107712b1927e Mon Sep 17 00:00:00 2001 From: Drew Erickson Date: Wed, 16 Oct 2013 10:15:36 -0700 Subject: [PATCH] Improved Flatten and Embed FlatJSON and EmbedJSON now handle lists of json objects better. Also, minor changes to GetENCODE error printing. --- ENCODETools.py | 42 +++++++++++++++++++++++++++++++----------- update2.py | 13 +++++++------ 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/ENCODETools.py b/ENCODETools.py index 4d06c7d..9e2061c 100644 --- a/ENCODETools.py +++ b/ENCODETools.py @@ -26,23 +26,26 @@ def GetENCODE(object_id,keys): '''GET an ENCODE object as JSON and return as dict''' if type(object_id) is str: url = keys['server']+object_id+'?limit=all' - print(url) + #print(url) try: response = requests.get(url, auth=(keys['authid'],keys['authpw']), headers=HEADERS) - # nope + if not response.status_code == 200: + print >> sys.stderr, response.text + # no except Exception as e: print("Get request failed:") #print(e) + # yes else: return response.json() # patch object to server -def patch_ENCODE(obj_id, patch_json): +def patch_ENCODE(obj_id,patch_json,keys): '''PATCH an existing ENCODE object and return the response JSON''' url = keys['server']+obj_id json_payload = json.dumps(patch_json) - response = requests.patch(url, auth=(keys['authid'],keys['pw']), data=json_payload) + response = requests.patch(url, auth=(keys['authid'],keys['authpw']), data=json_payload) print "Patch:" print response.status_code if not response.status_code == 200: @@ -107,8 +110,8 @@ def ValidJSON(object_type,object_id,new_object): return True # intended to fix invalid JSON. DOES NOT DO ANYTHING YET. -def CleanJSON(object_type,object_id,new_object): - for key,value in new_object.list(): +def CleanJSON(object_type,object_id,new_object,keys): + for key,value in new_object.items(): new_object.pop(key) if not ValidJSON(object_type,object_id,new_object): new_object[key] = value @@ -121,15 +124,32 @@ def FlatJSON(json_object,keys): for key,value in json_object.items(): if type(value) is dict: json_object[key] = json_object[key][u'@id'] + if type(value) is list: + #print("Found List: " + key) + value_new = [] + for value_check in value: + #print("Checking...") + if type(value_check) is dict: + #print("Found Object") + value_check = value_check[u'@id'] + #print(value_check) + value_new.append(value_check) + json_object[key] = value_new return json_object # expand json object def EmbedJSON(json_object,keys): for key,value in json_object.items(): + value_list = [] if type(value) is unicode: - if str(value[0]) == '/': - json_sub_object = GetENCODE(str(value),keys) - if type(json_sub_object) is dict: - #json_sub_object = EmbedJSON(json_sub_object,keys) - json_object[key] = json_sub_object + value_list.append(value) + elif type(value) is list: + value_list = value + for value_check in value_list: + if type(value_check) is unicode: + if str(value_check[0]) == '/': + json_sub_object = GetENCODE(str(value_check),keys) + if type(json_sub_object) is dict: + #json_sub_object = EmbedJSON(json_sub_object,keys) + json_object[key] = json_sub_object return json_object diff --git a/update2.py b/update2.py index c8be859..03bcf31 100644 --- a/update2.py +++ b/update2.py @@ -68,7 +68,7 @@ for new_object in object_list: - new_object = FlatJSON(new_object) + new_object = FlatJSON(new_object,keys) # define object parameters. NEEDS TO RUN A CHECK TO CONFIRM THESE EXIST FIRST. object_type = str(new_object[u'@type'][0]) @@ -79,7 +79,7 @@ # check to see if object already exists # PROBLEM: SHOULD CHECK UUID AND NOT USE ANY SHORTCUT METADATA THAT MIGHT NEED TO CHANGE # BUT CAN'T USE UUID IF NEW... HENCE PROBLEM - old_object = FlatJSON(get_ENCODE(object_id)) + old_object = FlatJSON(get_ENCODE(object_id,keys),keys) # # test the validity of new object # if not ValidJSON(object_type,object_id,new_object): @@ -123,13 +123,13 @@ print('Validation of ' + object_id + ' succeeded.') # post the new object(s). SHOULD HANDLE ERRORS GRACEFULLY - response = new_ENCODE(object_collection,new_object) + response = new_ENCODE(object_collection,new_object,keys) # if object is found, check for differences and patch it if needed. else: - # compare new object to old one, remove identical fields. + # compare new object to old one, remove identical fields. Also, remove fields not present in schema. SHOULD INFORM OF THIS OPERATION, BUT NOT NEEDED WHEN SINGLE PATCH CODE EXISTS. for key in new_object.keys(): if new_object.get(key) == old_object.get(key): new_object.pop(key) @@ -141,13 +141,14 @@ # inform user of the updates print(object_id + ' has updates.') - print(new_object) + #print(new_object) # patch each field to object individually for key,value in new_object.items(): patch_single = {} patch_single[key] = value - response = patch_ENCODE(object_id, patch_single) + print(patch_single) + response = patch_ENCODE(object_id,patch_single,keys) # inform user there are no updates else: