Merge pull request #60 from cligu/dev

Dev

css/gitdox.css

@@ -16,7 +16,7 @@ h1, h2{font-family: asul, "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans",
 	}
 
 .corpusbutton{
-	background-color: #23d83c !important;
+	background-color: #941c1fc4 !important;
 
 }
 
@@ -93,7 +93,7 @@ select select:focus {
 
 #meta_table, #meta_table_corpus{width:800px;  table-layout: fixed;}
 
-#meta_table_corpus{background-color: #cefbce}
+#meta_table_corpus{background-color: #941c1f8c}
 
 table.sortable th:not(.sorttable_sorted):not(.sorttable_sorted_reverse):not(.sorttable_nosort):after { 
     content: "\00a0\25B4\25BE" 

editor.py

@@ -68,6 +68,7 @@ def serialize_file(text_content,file_name):
 
 
 def load_page(user,admin,theform):
+	print("Content-type:text/html\r\n\r\n")
 	global ether_url
 	global code_2fa
 	if theform.getvalue("2fa"):
@@ -265,7 +266,12 @@ def load_page(user,admin,theform):
 		# Therefore, a file may be associated with the target repo subdir zangsir/coptic-xml-tool/uploaded_commits,
 		# and that is fine, but we will need to make this uploaded_commits subdir first to create our file.
 		if not os.path.isdir(prefix + subdir) and subdir != "":
-			os.mkdir(prefix + subdir, 0755)
+			dirs = subdir.split(os.sep)[:-1]
+			path_so_far = ""
+			for dir in dirs:
+				if not os.path.isdir(prefix + path_so_far + dir + os.sep):
+					os.mkdir(prefix + path_so_far + dir + os.sep, 0755)
+				path_so_far += dir + os.sep
 
 		if mode == "xml":
 			text_content = generic_query("SELECT content FROM docs WHERE id=?", (doc_id,))[0][0]
@@ -363,7 +369,7 @@ def load_page(user,admin,theform):
 	# Metadata
 	if theform.getvalue('metakey'):
 		metakey = theform.getvalue('metakey')
-		metavalue = theform.getvalue('metavalue')
+		metavalue = theform.getvalue('metavalue').replace("\t","").replace("\n","").replace("\r","")
 		if user != "demo":
 			save_meta(int(doc_id),metakey.decode("utf8"),metavalue.decode("utf8"))
 	if theform.getvalue('metaid'):
@@ -372,7 +378,7 @@ def load_page(user,admin,theform):
 			delete_meta(metaid, doc_id)
 	if theform.getvalue('corpus_metakey'):
 		metakey = theform.getvalue('corpus_metakey')
-		metavalue = theform.getvalue('corpus_metavalue')
+		metavalue = theform.getvalue('corpus_metavalue').replace("\t","").replace("\n","").replace("\r","")
 		if user != "demo":
 			save_meta(int(doc_id),metakey.decode("utf8"),metavalue.decode("utf8"),corpus=True)
 	if theform.getvalue('corpus_metaid'):
@@ -392,7 +398,7 @@ def load_page(user,admin,theform):
 	if user == "demo":
 		nlp_service = disabled_nlp_service
 
-	page= "Content-type:text/html\r\n\r\n"
+	page= ""#"Content-type:text/html\r\n\r\n"
 	if mode == "ether":
 		embedded_editor = urllib.urlopen(prefix + "templates" + os.sep + "ether.html").read()
 		ether_url += "gd_" + corpus + "_" + docname

export.py

@@ -16,7 +16,11 @@ def create_zip(content_name_pairs):
 	zf = zipfile.ZipFile(io_file, mode='w', compression=zipfile.ZIP_DEFLATED)
 
 	for content, name in content_name_pairs:
-		zf.writestr(name, content)#.encode("utf8"))
+		try:
+			zf.writestr(name, content)#.encode("utf8"))
+		except Exception as e:
+			print("Content-type:text/html\r\n\r\n")
+			raise e
 
 	return io_file
 

modules/ether.py

@@ -95,6 +95,13 @@ def read_config(self,config_file):
 		else:
 			self.template = "<meta %%all%%>\n%%body%%\n</meta>\n"
 
+def unescape_xml(text):
+	# Fix various common compounded XML escapes
+	text = text.replace("&amp;lt;","<").replace("&amp;gt;",">")
+	text = text.replace("&lt;","<").replace("&gt;",">")
+	text = text.replace("&amp;","&")
+	return text
+
 def build_meta_tag(doc_id):
 	meta = "<meta"
 	meta_items = []
@@ -103,14 +110,16 @@ def build_meta_tag(doc_id):
 	for item in meta_rows:
 		key, value = item[2], item[3]
 		if not key.startswith("ignore:"):
-			key = key.replace("=", "&equals;")
-			value = value.replace('"', "&quot;")
+			key = key.replace("=", "&equals;")  # Key may not contain equals sign
+			value = value.replace('"', "&quot;")  # Value may not contain double quotes
+			value = unescape_xml(value)
 			meta_items.append(key + '="' + value + '"')
 
 	meta_props = " ".join(meta_items)
 	if meta_props != "":
 		meta_props = " " + meta_props
 	output = meta + meta_props + ">\n"
+	output = output.replace("<meta >","<meta>")
 	return output
 
 
@@ -126,6 +135,7 @@ def fill_meta_template(doc_id,template):
 		if not key.startswith("ignore:"):
 			key = key.replace("=", "&equals;")
 			value = value.replace('"', "&quot;")
+			value = unescape_xml(value)
 			meta_items.append(escape(key) + '="' + escape(value) + '"')
 			meta_dict[escape(key)] = escape(value)
 
@@ -148,6 +158,7 @@ def fill_meta_template(doc_id,template):
 		if key != "body": # Never overwrite body template position
 			template = template.replace("%%" + key + "%%",meta_dict[key])
 
+	template = template.replace("<meta >","<meta>")
 	return template
 
 
@@ -202,7 +213,7 @@ def flush_open(annos, row_num, colmap):
 	flushed = ""
 	for anno in annos:
 		element, name, value = anno
-		flushed += "cell:"+colmap[name] + str(row_num) + ":t:" + value + "\n"
+		flushed += "cell:"+colmap[name] + str(row_num) + ":t:" + value + "\n"  # NO t >TVF
 	return flushed
 
 
@@ -213,7 +224,7 @@ def flush_close(closing_element, last_value, last_start, row_num, colmap, aliase
 			span_string = ":rowspan:" + str(row_num - last_start[alias])
 		else:
 			span_string = ""
-		flushed += "cell:" + colmap[alias] + str(last_start[alias]) + ":t:" + last_value[alias]+span_string + ":f:1\n"
+		flushed += "cell:" + colmap[alias] + str(last_start[alias]) + ":t:" + last_value[alias]+span_string + "\n"  # Use t for tvf to leave links on
 	return flushed
 
 
@@ -295,15 +306,15 @@ def sgml_to_ether(sgml, ignore_elements=False):
 
 		elif len(line) > 0:  # Token
 			token = line.strip()
-			output += "cell:A"+str(current_row)+":t:"+token+":f:1\n"
+			output += "cell:A"+str(current_row)+":t:"+token+":f:1:tvf:1\n"  # NO f <> tvf for links
 			current_row +=1
 		else:  # Empty line
 			current_row +=1
 
-	preamble += "cell:A1:t:tok:f:2\n"
+	preamble += "cell:A1:t:tok:f:2\n" # f <> tvf for links
 	output = preamble + output
 	for header in colmap:
-		output += "cell:"+colmap[header]+"1:t:"+header+":f:2\n"
+		output += "cell:"+colmap[header]+"1:t:"+header+":f:2\n" # NO f <> tvf for links
 
 	output += "\nsheet:c:" + str(maxcol) + ":r:" + str(current_row-1) + ":tvf:1\n"
 
@@ -312,7 +323,7 @@ def sgml_to_ether(sgml, ignore_elements=False):
 	output += """
 font:1:* * Antinoou
 font:2:normal bold * *
-valueformat:1:text-wiki
+valueformat:1:text-plain
 --SocialCalcSpreadsheetControlSave
 Content-type: text/plain; charset=UTF-8
 
@@ -374,14 +385,14 @@ def ether_to_sgml(ether, doc_id,config=None):
 	close_tags = defaultdict(list)
 	for cell in cells:
 		if cell[1] == 1:  # Header row
-			colname = cell[2]['t']
+			colname = cell[2]['t'].replace("\\c",":")
 			if colname in config.aliases:
 				colmap[cell[0]] = config.aliases[colname]
 			else:
 				colmap[cell[0]] = colname
 			# Make sure that everything that should be exported has some priority
 			if colname not in config.priorities and config.export_all:
-				if not colname.lower().startswith("ignore\\c"):  # Never export columns prefixed with "ignore:"
+				if not colname.lower().startswith("ignore:"):  # Never export columns prefixed with "ignore:"
 					if "@" in colname:
 						elem = colname.split("@",1)[0]
 					else:
@@ -410,7 +421,7 @@ def ether_to_sgml(ether, doc_id,config=None):
 			else:
 				sec_element = ""
 
-			if element not in config.priorities or (element.startswith("ignore\\c") and config.no_ignore):  # Guaranteed to be in priorities if it should be included
+			if element not in config.priorities or (element.startswith("ignore:") and config.no_ignore):  # Guaranteed to be in priorities if it should be included
 				continue  # Move on to next cell if this is not a desired column
 			if row != last_row:  # New row starting, sort previous lists for opening and closing orders
 				#close_tags[row].sort(key=lambda x: (-last_open_index[x],x))
@@ -453,8 +464,6 @@ def ether_to_sgml(ether, doc_id,config=None):
 				if element in config.no_content:
 					if element == attrib:
 						attrib = ""
-				else:
-					attrib = col_name
 
 				if attrib in config.tok_annos:
 					# TT SGML token annotation, append to token with tab separator and move on
@@ -484,7 +493,7 @@ def ether_to_sgml(ether, doc_id,config=None):
 				open_tag_length[element] = int(close_row) - int(last_open_index[element])
 
 	# Sort last row tags
-	close_tags[row].sort(key=lambda x: (last_open_index[x],config.priorities.index(x)), reverse=True)
+	#close_tags[row].sort(key=lambda x: (last_open_index[x],config.priorities.index(x)), reverse=True)
 	if row + 1 in close_tags:
 		close_tags[row+1].sort(key=lambda x: (last_open_index[x],config.priorities.index(x)), reverse=True)
 	for element in open_tags[last_row]:
@@ -520,7 +529,7 @@ def ether_to_sgml(ether, doc_id,config=None):
 			toks[r] = ""  # Caution - empty token!
 		output += toks[r] + '\n'
 
-	output = output.replace('\c', ':')
+	output = output.replace('\\c', ':')
 	#output += "</meta>\n"
 	if "%%body%%" in template:
 		output = template.replace("%%body%%",output.strip())

modules/gitdox_sql.py

@@ -118,6 +118,8 @@ def cell(text):
 
 def print_meta(doc_id, corpus=False):
 	meta = get_doc_meta(doc_id, corpus=corpus)
+	if meta is None:
+		meta = []
 	# docid,metaid,key,value - four cols
 	metaid_id = "metaid" if not corpus else "corpus_metaid"
 	table_id = "meta_table" if not corpus else "meta_table_corpus"
@@ -159,9 +161,9 @@ def print_meta(doc_id, corpus=False):
 def save_meta(doc_id,key,value,corpus=False):
 	if corpus:
 		_, corpus_name, _, _, _, _, _ = get_doc_info(doc_id)
-		generic_query("INSERT OR REPLACE INTO metadata(docid,key,value,corpus_meta) VALUES(?,?,?,?)", ('NULL',key, value,corpus_name))
+		generic_query("INSERT OR REPLACE INTO metadata(docid,key,value,corpus_meta) VALUES(?,?,?,?)", (None,key, value,corpus_name))
 	else:
-		generic_query("INSERT OR REPLACE INTO metadata(docid,key,value,corpus_meta) VALUES(?,?,?,?)",(doc_id,key,value,'NULL'))
+		generic_query("INSERT OR REPLACE INTO metadata(docid,key,value,corpus_meta) VALUES(?,?,?,?)",(doc_id,key,value,None))
 		invalidate_doc_by_id(doc_id)
 
 def delete_meta(metaid, doc_id, corpus=False):
@@ -170,7 +172,11 @@ def delete_meta(metaid, doc_id, corpus=False):
 		invalidate_doc_by_id(doc_id)
 
 def get_doc_info(doc_id):
-	return generic_query("SELECT name,corpus,filename,status,assignee_username,mode,schema FROM docs WHERE id=?", (int(doc_id),))[0]
+	res = generic_query("SELECT name,corpus,filename,status,assignee_username,mode,schema FROM docs WHERE id=?", (int(doc_id),))
+	if len(res) > 0:
+		return res[0]
+	else:
+		return res
 
 def get_all_docs(corpus=None, status=None):
 	if corpus is None:
@@ -186,8 +192,12 @@ def get_all_docs(corpus=None, status=None):
 
 def get_doc_meta(doc_id, corpus=False):
 	if corpus:
-		_, corpus_name, _, _, _, _, _ = get_doc_info(doc_id)
-		return generic_query("SELECT * FROM metadata WHERE corpus_meta=? ORDER BY key COLLATE NOCASE", (corpus_name,))
+		fields = get_doc_info(doc_id)
+		if len(fields) > 0:
+			_, corpus_name, _, _, _, _, _ = fields
+			return generic_query("SELECT * FROM metadata WHERE corpus_meta=? ORDER BY key COLLATE NOCASE",(corpus_name,))
+		else:
+			return None
 	else:
 		return generic_query("SELECT * FROM metadata WHERE docid=? ORDER BY key COLLATE NOCASE", (int(doc_id),))
 

modules/validate_spreadsheet.py

@@ -198,7 +198,7 @@ def parse_ether(ether, doc, corpus):
 				#		cell_row += c
 				#	else:
 				#		cell_col += c
-				cell_content = parts[3]
+				cell_content = parts[3].replace("\\c",":")
 				if "rowspan:" in line:
 					cell_span = parts[-1]
 				else:
@@ -282,14 +282,20 @@ def apply_rule(rule, parsed_ether, meta):
 			arg_boundaries = []
 			name_content = {}
 			arg_content = {}
+			name_filled = []
+			arg_filled = []
 
 			# find boundary rows
 			for cell in parsed_ether[name]:
 				name_boundaries.append(cell.row)
 				name_content[cell.row] = cell.content
+				for i in range(int(cell.row), int(cell.row) + int(cell.span)):
+					name_filled.append(str(i))
 			for cell in parsed_ether[argument]:
 				arg_boundaries.append(cell.row)
 				arg_content[cell.row] = cell.content
+				for i in range(int(cell.row), int(cell.row) + int(cell.span)):
+					arg_filled.append(str(i))
 
 			if operator == "==":
 				for row in name_content:
@@ -302,20 +308,22 @@ def apply_rule(rule, parsed_ether, meta):
 			else:
 				for boundary in name_boundaries:
 					if boundary not in arg_boundaries:
-						report += "Span break on line " + boundary + " in column " + name + " but not " \
-								  + argument + "<br/>"
-						cells.append(name_letter + boundary)
+						if boundary in arg_filled:
+							report += "Span break on line " + boundary + " in column " + name + " but not " \
+									  + argument + "<br/>"
+							cells.append(name_letter + boundary)
 				if operator == "=":
 					for boundary in arg_boundaries:
 						if boundary not in name_boundaries:
-							cells.append(arg_letter + boundary)
+							if boundary in name_filled:
+								cells.append(arg_letter + boundary)
 
-	elif domain == "meta":
-		meta_report, meta_extra = apply_meta_rule(rule, meta)
-		report += meta_report
-		extra += meta_extra
+		elif domain == "meta":
+			meta_report, meta_extra = apply_meta_rule(rule, meta)
+			report += meta_report
+			extra += meta_extra
 
-	return report, extra, cells
+		return report, extra, cells
 
 
 def apply_meta_rule(rule, meta):

templates/editor.html

@@ -113,7 +113,7 @@ <h2>Editor | <a href="index.py">back to document list</a> </h2>
 <h3>meta data</h3>
 **metadata**
 
-<div class="button" onClick="openPopup('popupPage.html')"><i class="fa fa-info-circle"> </i> Add meta</div>
+<div class="button" onClick="openPopup('popupPage.html')"><i class="fa fa-info-circle"> </i> Add document meta</div>
 
 <input TYPE="hidden" NAME="metakey">
 <input type="hidden" name="metavalue">