- refactoring: modular parsers introduced

TODO

@@ -38,6 +38,8 @@ available is currently monolithically located in snafu.py. Parsers
 should become a new subsystem class to ease the addition of new ones and
 maintenance of existing ones.
 
+ -> parser code is now available as subsystem, with per-type defaults
+
 There is currently a global default executor which makes it impossible
 to execute functions written in different languages within the same
 session. Rather, there should be a per-function default executor which
@@ -70,6 +72,9 @@ for at least the Python language. This executor should output statistics
 about local and remote function calls and generally useful information
 from the profile module.
 
+ -> a tracing executor is now available in a forked version
+ https://github.com/Nopx/snafu
+
 A snafu-deploy tool could be developed to quickly deploy functions to
 any cloud provider without going through the hassle of learning the
 provider-dependent deployment tools. Execution is then possible through

snafulib/parsers/c.py

@@ -0,0 +1,33 @@
+# Snafu: Snake Functions - C Parser
+
+import os
+
+def activatefile(self, source, convention, SnafuFunctionSource):
+	if not os.path.isfile("snafulib/executors/java/cexec"):
+		pwd = os.getcwd()
+		os.chdir("snafulib/executors/c")
+		os.system("gcc -Wall -O2 -o cexec cexec.c -ldl")
+		os.chdir(pwd)
+
+	if source.endswith(".c"):
+		binfile = source.replace(".c", ".so")
+		if not os.path.isfile(binfile):
+			if not self.quiet:
+				print("» c source:", source)
+			pwd = os.getcwd()
+			os.chdir(os.path.dirname(source))
+			os.system("gcc -Wall -O2 -fPIC -shared -o {} {}".format(os.path.basename(binfile), os.path.basename(source)))
+			os.chdir(pwd)
+			source = binfile
+		else:
+			return
+
+	if not self.quiet:
+		print("» c module:", source)
+
+	funcname = os.path.basename(source).replace(".", "_") + ".handler"
+	funcparams = ["input"]
+	if not self.quiet:
+		print("  function: {} (unchecked)".format(funcname))
+	sourceinfos = SnafuFunctionSource(source, scan=False)
+	self.functions[funcname] = ([funcname] + funcparams, None, sourceinfos)

snafulib/parsers/java.py

@@ -0,0 +1,65 @@
+# Snafu: Snake Functions - Java Parser
+
+import os
+
+def activatefile(self, source, convention, SnafuFunctionSource):
+	if not os.path.isfile("snafulib/executors/java/JavaExec.class"):
+		pwd = os.getcwd()
+		os.chdir("snafulib/executors/java")
+		os.system("javac JavaExec.java")
+		os.chdir(pwd)
+
+	funcname = None
+	configname = source.split(".")[0] + ".config"
+	if os.path.isfile(configname):
+		if not self.quiet:
+			print("  config:", configname)
+		config = json.load(open(configname))
+		if config:
+			funcname = config["FunctionName"]
+	else:
+		if convention == "lambda":
+			if not self.quiet:
+				print("  skip source {}".format(source))
+			return
+
+	if source.endswith(".java"):
+		binfile = source.replace(".java", ".class")
+		if not os.path.isfile(binfile):
+			if not self.quiet:
+				print("» java source:", source)
+			pwd = os.getcwd()
+			os.chdir(os.path.dirname(source))
+			os.system("javac {}".format(os.path.basename(source)))
+			os.chdir(pwd)
+			source = binfile
+		else:
+			return
+
+	if not self.quiet:
+		print("» java module:", source)
+
+	if funcname:
+		# FIXME: shortcut leading to non-executable code for Lambda-imported Java
+		if not self.quiet:
+			print("  function: {}".format(funcname))
+		sourceinfos = SnafuFunctionSource(source, scan=False)
+		self.functions[funcname] = ([funcname], None, sourceinfos)
+		return
+
+	#javacmd = "java -cp executors/java/:{} JavaExec {} fib 3".format(os.path.dirname(source), os.path.basename(source).split(".")[0])
+	#javacmd = "java JavaExec Hello myHandler 5 null"
+	#print("JAVA", javacmd)
+	javacmd = "java -cp snafulib/executors/java/:{} JavaExec {} SCAN".format(os.path.dirname(source), os.path.basename(source).split(".")[0])
+	#os.system(javacmd)
+	out, err = subprocess.Popen(javacmd, shell=True, stdout=subprocess.PIPE).communicate()
+	for funcname in out.decode("utf-8").split("\n"):
+		if not funcname:
+			continue
+		funcname, *funcparams = funcname.split(" ")
+		funcname = os.path.basename(source).split(".")[0] + "." + funcname
+		if not self.quiet:
+			print("  function: {}".format(funcname))
+		sourceinfos = SnafuFunctionSource(source, scan=False)
+		self.functions[funcname] = ([funcname] + funcparams, None, sourceinfos)
+

snafulib/parsers/javascript.py

@@ -0,0 +1,45 @@
+# Snafu: Snake Functions - JavaScript Parser
+
+def activatefile(self, source, convention, SnafuFunctionSource):
+	if not self.quiet:
+		print("» javascript module:", source)
+
+	try:
+		import pyesprima.pyesprima3
+	except:
+		if not self.quiet:
+			print("Warning: javascript parser not ready for {}, skipping.".format(source), file=sys.stderr)
+		return
+
+	#pyesprima.pyesprima3.unichr = chr
+
+	try:
+		ast = pyesprima.pyesprima3.parse(open(source).read())
+	except Exception as e:
+		if not self.quiet:
+			print("Warning: {} is not parseable, skipping. [{}]".format(source, e), file=sys.stderr)
+		return
+
+	for body in ast.body:
+		if body.type == "FunctionDeclaration":
+			funcname = body["id"]["name"]
+			if funcname == "main":
+				funcname = os.path.basename(source).split(".")[0] + "." + funcname
+				if not self.quiet:
+					print("  function: {}".format(funcname))
+				sourceinfos = SnafuFunctionSource(source, scan=False)
+				funcparams = ["input"]
+				self.functions[funcname] = ([funcname] + funcparams, None, sourceinfos)
+			else:
+				if not self.quiet:
+					print("  skip function {}".format(funcname))
+		elif body.type == "ExpressionStatement":
+			if body.expression.left.type == "MemberExpression":
+				if body.expression.left.object.name == "exports":
+					funcname = body.expression.left.property.name
+					funcname = os.path.basename(source).split(".")[0] + "." + funcname
+					if not self.quiet:
+						print("  function: {}".format(funcname))
+					sourceinfos = SnafuFunctionSource(source, scan=False)
+					funcparams = ["req", "res"]
+					self.functions[funcname] = ([funcname] + funcparams, None, sourceinfos)

snafulib/parsers/python.py

@@ -0,0 +1,79 @@
+# Snafu: Snake Functions - Python Parser
+
+import os
+import ast
+import importlib
+import types
+import sys
+import json
+
+def activatefile(self, source, convention, SnafuFunctionSource):
+	sourceinfos = None
+	try:
+		sourceinfos = SnafuFunctionSource(source)
+		sourcecode = sourceinfos.content
+	except:
+		print("Warning: {} is not parseable, skipping.".format(source), file=sys.stderr)
+		return
+	if not self.quiet:
+		print("» module:", source)
+
+	handler = None
+	config = None
+	configname = source.split(".")[0] + ".config"
+	if os.path.isfile(configname):
+		if not self.quiet:
+			print("  config:", configname)
+		config = json.load(open(configname))
+		if config:
+			if "Handler" in config:
+				handler = config["Handler"]
+
+	connectorconfig = None
+	connectorconfigname = source.split(".")[0] + ".ini"
+	if os.path.isfile(connectorconfigname):
+		if not self.quiet:
+			print("  connectors:", connectorconfigname)
+		connectorconfig = connectorconfigname
+
+	sourcetree = ast.parse(sourcecode)
+	loader = importlib.machinery.SourceFileLoader(os.path.basename(source), source)
+	mod = types.ModuleType(loader.name)
+	sourceinfos.module = mod
+	if not os.path.dirname(source) in sys.path:
+		sys.path.append(os.path.dirname(source))
+	try:
+		loader.exec_module(mod)
+	except Exception as e:
+		if not self.quiet:
+			print("  Warning: skipping due to import error: {}".format(e))
+		return
+	sourcename = os.path.basename(source).split(".")[0]
+	for node in ast.walk(sourcetree):
+		if type(node) == ast.FunctionDef:
+			if not handler:
+				handlername = "lambda_handler"
+				handlerbase = sourcename
+			else:
+				handlerbase, handlername = handler.split(".")
+			if convention not in ("lambda", "openwhisk") or (node.name == handlername and sourcename == handlerbase):
+				funcname = sourcename + "." + node.name
+				if config and "FunctionName" in config and convention in ("lambda", "openwhisk"):
+					funcname = config["FunctionName"]
+				try:
+					func = getattr(mod, node.name)
+				except:
+					print("  skip method {}.{}".format(sourcename, node.name))
+				else:
+					if not self.quiet:
+						print("  function: {}".format(funcname))
+					#if not node.name in self.functions:
+					#	self.functions[node.name] = {}
+					#self.functions[node.name][sourcename] = (func, config, sourceinfos)
+					self.functions[funcname] = (func, config, sourceinfos)
+					if connectorconfig:
+						self.functionconnectors[funcname] = connectorconfig
+			else:
+				if not self.quiet:
+					print("  skip function {}.{}".format(sourcename, node.name))
+