Skip to content

Commit

Permalink
Prettier Printing. (#1)
Browse files Browse the repository at this point in the history
* Replaced the print stagement using logging (error, warning, info and debug levels used).
Added progress bar using tqdm package in performSearch function for the 4 stages of exploration.
Changes Antlr Check version to 4.9.1.
Added requirements.txt

* Added more information to the print statment that prints the stage of exploration.

* Added log level as a command line argument.
Added  descriptive error messages.

* Reverted parameters to old state

* Slight changes to the strings printed by logging

* Resolved formatting comments

* Updated README.md with comand to change logging level

* Converted the log level input arguments to lowercase strings
krantikiran68 authored Jan 15, 2021
1 parent 39668ef commit 30c7eeb
Showing 13 changed files with 145 additions and 84 deletions.
2 changes: 1 addition & 1 deletion tools/SeeDot/.vscode/launch.json
Original file line number Diff line number Diff line change
@@ -24,7 +24,7 @@
"-t",
"x86",
"-n",
"1"
"1",
]
}
]
5 changes: 3 additions & 2 deletions tools/SeeDot/README.md
Original file line number Diff line number Diff line change
@@ -26,7 +26,7 @@ This document describes the tool usage with an example.
SeeDot can be invoked using **`SeeDot-dev.py`** file. The arguments for the script are supplied as follows:

```
usage: SeeDot-dev.py [-h] [-a] [-v] [-d] [-m] [-n] [-dt] [-t] [-s] [-sf]
usage: SeeDot-dev.py [-h] [-a] [-v] [-d] [-m] [-n] [-dt] [-t] [-s] [-sf] [-l]
[--load-sf] [--convert] [--tempdir] [-o]
optional arguments:
@@ -40,11 +40,12 @@ optional arguments:
procedure (1 for a single-class classification problem)
-t , --target Target device ['x86', 'arduino', 'm3']
-s , --source model source type ['seedot', 'onnx', 'tf']
-l , --log Logging level (in increasing order) ['error', 'critical', 'warning', 'info', 'debug']
```

An example invocation is as follows:
```
python SeeDot-dev.py -a rnn -v fixed -d usps10 -n 1 -t arduino -m red_disagree
python SeeDot-dev.py -a rnn -v fixed -d usps10 -n 1 -t arduino -m red_disagree -l info
```

SeeDot expects the `train` and the `test` data files in a specific format. Each data file should be of the shape `[numberOfDataPoints, numberOfFeatures + n]`, where the ground truth/output is in the first `n` columns. The tool currently supports numpy arrays (.npy) for inputting model parameters.
9 changes: 8 additions & 1 deletion tools/SeeDot/SeeDot-dev.py
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@
import seedot.main as main
import seedot.predictor as predictor
import seedot.util as util

import logging
import seedot.compiler.converter.converter as converter

# This is the file which is invoked to run the compiler (Refer to README.md).
@@ -82,6 +82,8 @@ def parseArgs(self):
help="Scratch directory for intermediate files")
parser.add_argument("-o", "--outdir", metavar='',
help="Directory to output the generated Arduino sketch")
parser.add_argument("-l", "--log", choices=config.Log.all,
default=config.Log.default, metavar='', help="Log Level to use")

self.args = parser.parse_args()

@@ -135,7 +137,12 @@ def checkMSBuildPath(self):
def setGlobalFlags(self):
np.seterr(all='warn')

def setLogLevel(self):
logging.basicConfig(level=os.environ.get("LOGLEVEL", self.args.log.upper()))

def run(self):
self.setLogLevel()

if util.windows():
self.checkMSBuildPath()

21 changes: 21 additions & 0 deletions tools/SeeDot/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
antlr4-python3-runtime==4.7
bokeh==2.2.3
certifi==2020.12.5
Jinja2==2.11.2
joblib==1.0.0
MarkupSafe==1.1.1
numpy==1.19.5
onnx==1.8.0
packaging==20.8
Pillow==8.1.0
protobuf==3.14.0
pyparsing==2.4.7
python-dateutil==2.8.1
PyYAML==5.3.1
scikit-learn==0.24.0
scipy==1.6.0
six==1.15.0
threadpoolctl==2.1.0
tornado==6.1
tqdm==4.56.0
typing-extensions==3.7.4.3
10 changes: 5 additions & 5 deletions tools/SeeDot/seedot/compiler/codegen/codegenBase.py
Original file line number Diff line number Diff line change
@@ -864,7 +864,7 @@ def getBestAlignment(mA, align, maxMem, memUsage, f):
p.terminate()
p.join()
optimalInputGenSuccess = False
print("Timeout while generating DLX input files for optimal memory usage, attempting to fit variables within %d bytes" % maxAllowedMemUsage)
Util.getLogger().error("Timeout while generating DLX input files for optimal memory usage, attempting to fit variables within %d bytes. Returning to exploration..." % maxAllowedMemUsage)
alignment = getBestAlignment(memAlloc, alignment, 0, maxAllowedMemUsage, operator.le)
p = mp.Process(target=DLXInputGen.generateDLXInput, args=(memAlloc, alignment, maxAllowedMemUsage, False, dlxInputDumpDirectory))
p.start()
@@ -881,10 +881,10 @@ def getBestAlignment(mA, align, maxMem, memUsage, f):
try:
process = subprocess.call([exeFile], stdin=fin, stdout=fout, stderr=ferr, timeout=timeout)
except subprocess.TimeoutExpired:
print("Memory Allocator Program Timed out.")
Util.getLogger().error("DLX program for memory management timed out. Retrying with maximum allowed memory...")
if not self.checkDlxSuccess(dlxErrorDumpDirectory):
if not optimalInputGenSuccess:
assert False, "Unable to allocate variables within %d bytes. ABORT" % maxAllowedMemUsage
assert False, "DLX unable to allocate variables within %d bytes. ABORT" % maxAllowedMemUsage
else:
alignment = getBestAlignment(memAlloc, alignment, 0, maxAllowedMemUsage, operator.le)
p = mp.Process(target=DLXInputGen.generateDLXInput, args=(memAlloc, alignment, maxAllowedMemUsage, False, dlxInputDumpDirectory))
@@ -898,9 +898,9 @@ def getBestAlignment(mA, align, maxMem, memUsage, f):
try:
process = subprocess.call([exeFile], stdin=fin, stdout=fout, stderr=ferr, timeout=timeout)
except subprocess.TimeoutExpired:
print("Memory Allocator Program Timed out.")
Util.getLogger().error("DLX program for memory management timed out.")
if not self.checkDlxSuccess(dlxErrorDumpDirectory):
assert False, "Unable to allovate variables within %d bytes. ABORT" % maxAllowedMemUsage
assert False, "DLX unable to allovate variables within %d bytes. ABORT" % maxAllowedMemUsage
totalScratchSize = self.readDlxAllocation(dlxOutputDumpDirectory, alignment, varOrderAndSize)
if not forM3():
self.out.printf("char scratch[%d];\n"%(totalScratchSize), indent=True)
10 changes: 5 additions & 5 deletions tools/SeeDot/seedot/compiler/codegen/x86.py
Original file line number Diff line number Diff line change
@@ -26,16 +26,16 @@ def __init__(self, outputDir, generateAllFiles, printSwitch, idStr, paramInNativ
if generateAllFiles:
self.out = Writer(cppFile)
else:
print("Opening file to output cpp code: ID" + idStr)
getLogger().info("Opening file to output cpp code: ID" + idStr)
for i in range(3):
print("Try %d" % (i+1))
getLogger().debug("Try %d" % (i+1))
try:
self.out = Writer(cppFile, "a")
except:
print("OS prevented file from opening. Sleeping for %d seconds" % (i+1))
getLogger().exception("OS prevented file from opening. Sleeping for %d seconds" % (i+1))
time.sleep(i+1)
else:
print("Opened")
getLogger().debug("Opened")
break

self.generateAllFiles = generateAllFiles
@@ -323,7 +323,7 @@ def isInt(a):
self.out.decreaseIndent()
self.out.printf('}\n', indent=True)

print("Closing File after outputting cpp code: ID " + self.idStr)
getLogger().debug("Closing file after outputting cpp code: ID " + self.idStr)
self.out.close()

def printFor(self, ir):
3 changes: 2 additions & 1 deletion tools/SeeDot/seedot/compiler/compiler.py
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@

import numpy as np


# The Compiler class reads in the input code, converts it first into an AST, and subsequently into an IR which
# contains a sequence of function calls (which are implemented by hand in a library). The IR is fed into the
# desired target codegen, which outputs the C/C++ code which can be run on the target device.
@@ -139,7 +140,7 @@ def genCodeWithFuncCalls(self, ast):
compiler = irBuilder.IRBuilder(outputLog, self.intermediateScales, self.substitutions, self.scaleForX, self.variableToBitwidthMap, self.sparseMatrixSizes, self.demotedVarsList, self.demotedVarsOffsets)
res = compiler.visit(ast)

print(compiler.varScales)
util.getLogger().debug(compiler.varScales)
self.biasShifts = compiler.biasShifts
self.varScales = dict(compiler.varScales)

4 changes: 2 additions & 2 deletions tools/SeeDot/seedot/compiler/ir/ir.py
Original file line number Diff line number Diff line change
@@ -362,9 +362,9 @@ def getInt(x: int):
if x_np != x:
x_np = DataType.intType[target][wordLen * 2](x)
if x_np != x:
print('Warning: Integer overflow for %d' % (x))
getLogger().debug('Warning: Integer overflow for %d' % (x))
else:
print('Integer overflow for %d handled' % (x))
getLogger().debug('Integer overflow for %d handled' % (x))
return x_np

@staticmethod
10 changes: 5 additions & 5 deletions tools/SeeDot/seedot/compiler/ir/irBuilder.py
Original file line number Diff line number Diff line change
@@ -68,7 +68,7 @@ def __init__(self, outputLog, ddsScaleInfo = {}, substitutions = {}, scaleForX =
# Large value which makes MAX_SCALE ineffective.
elif getMaxScale() == None:
if forFloat():
print(
getLogger().info(
"Setting MAX_SCALE = 0. This value will not affect the generated code.")
self.MAX_SCALE = 0
else:
@@ -2818,7 +2818,7 @@ def visitTanh(self, node: AST.Func):
intv_out = self.updateTanhIntv(intv_in, tanh_intv)

scale_new = self.getScale(config.tanhLimit)
print("Scale changes in TanH operation: old = %d, new = %d, diff = %d" % (
getLogger().debug("Scale changes in TanH operation: old = %d, new = %d, diff = %d" % (
scale_in, scale_new, abs(scale_in - scale_new)))

expr_in.inputVar = False
@@ -3605,7 +3605,7 @@ def visitLet(self, node: AST.Let):

# TODO: When is this triggered and why is this required?
if forFixed() and idf in self.mutableVars:
print("TODO: Fix this if condition")
getLogger().warning("TODO: Fix this if condition")
idfs = idf
while idfs in self.substitutions.keys():
idfs = self.substitutions[idfs]
@@ -3965,7 +3965,7 @@ def getShrTreeSumAndDemoteParamsForMul(self,
# Last stage, adjusting scale to avoid invalid values.
demote = totalShr - shr_A - shr_B - H1
if height < H1:
print("Rolling back H1 in matrix multiplication. Current H1: %d height: %d" % (H1, height))
getLogger().info("Rolling back H1 in matrix multiplication. Current H1: %d height: %d" % (H1, height))
if height < H1:
diff = H1 - height
H1 = height
@@ -3976,7 +3976,7 @@ def getShrTreeSumAndDemoteParamsForMul(self,
shr_B += diff // 2
shr_A += diff - diff // 2
if demote < 0:
print("Rolling back shr in matrix multiplication. Current demote: %d" % (demote))
getLogger().info("Rolling back shr in matrix multiplication. Current demote: %d" % (demote))
if demote < 0:
if demote + H1 >= 0:
H1 += demote
10 changes: 10 additions & 0 deletions tools/SeeDot/seedot/config.py
Original file line number Diff line number Diff line change
@@ -110,3 +110,13 @@ class Source:
onnx = "onnx"
default = seedot
all = [seedot, tf, onnx]


class Log:
error = "error"
critical = "critical"
warning = "warning"
info = "info"
debug = "debug"
default = error
all = [error, critical, warning, info, debug]
114 changes: 64 additions & 50 deletions tools/SeeDot/seedot/main.py

Large diffs are not rendered by default.

24 changes: 12 additions & 12 deletions tools/SeeDot/seedot/predictor.py
Original file line number Diff line number Diff line change
@@ -89,7 +89,7 @@ def buildForWindows(self):
Builds using the Predictor.vcxproj project file and creates the executable.
The target platform is currently set to x64.
'''
print("Build...", end='')
Util.getLogger().debug("Build...")

projFile = "Predictor.vcxproj"
args = [config.msbuildPath, projFile, r"/t:Build",
@@ -100,14 +100,14 @@ def buildForWindows(self):
process = subprocess.call(args, stdout=file, stderr=subprocess.STDOUT)

if process == 1:
print("FAILED!!\n")
Util.getLogger().debug("FAILED!!\n")
return False
else:
print("success")
Util.getLogger().debug("SUCCESS")
return True

def buildForLinux(self):
print("Build...", end='')
Util.getLogger().debug("Build...")

args = ["make"]

@@ -116,10 +116,10 @@ def buildForLinux(self):
process = subprocess.call(args, stdout=file, stderr=subprocess.STDOUT)

if process == 1:
print("FAILED!!\n")
Util.getLogger().debug("FAILED!!\n\n")
return False
else:
print("success")
Util.getLogger().debug("SUCCESS\n")
return True

def build(self):
@@ -132,7 +132,7 @@ def executeForWindows(self):
'''
Invokes the executable with arguments.
'''
print("Execution...", end='')
Util.getLogger().debug("Execution...")

exeFile = os.path.join("x64", "Release", "Predictor.exe")
args = [exeFile, self.version, self.datasetType, self.problemType, str(self.numOutputs)]
@@ -142,15 +142,15 @@ def executeForWindows(self):
process = subprocess.call(args, stdout=file, stderr=subprocess.STDOUT)

if process == 1:
print("FAILED!!\n")
Util.getLogger().debug("FAILED!!\n\n")
return None
else:
print("success")
Util.getLogger().debug("SUCCESS\n")
execMap = self.readStatsFile()
return execMap

def executeForLinux(self):
print("Execution...", end='')
Util.getLogger().debug("Execution...")

exeFile = os.path.join("./Predictor")
args = [exeFile, self.version, self.datasetType, self.problemType, str(self.numOutputs)]
@@ -160,10 +160,10 @@ def executeForLinux(self):
process = subprocess.call(args, stdout=file, stderr=subprocess.STDOUT)

if process == 1:
print("FAILED!!\n")
Util.getLogger().debug("FAILED!!\n\n")
return None
else:
print("success")
Util.getLogger().debug("SUCCESS\n")
execMap = self.readStatsFile()
return execMap

7 changes: 7 additions & 0 deletions tools/SeeDot/seedot/util.py
Original file line number Diff line number Diff line change
@@ -5,6 +5,7 @@
import platform

import seedot.config as config
import logging


class Config:
@@ -163,3 +164,9 @@ def computeScalingFactorForFuncCalls(val):

def computeScalingFactorForInlineCodegen(val):
return int(np.ceil(np.log2(val) - np.log2((1 << (config.wordLength - 2)) - 1)))


# Logging Section
def getLogger():
log = logging.getLogger()
return log

0 comments on commit 30c7eeb

Please sign in to comment.