Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
nathanmartins committed Mar 19, 2020
1 parent f4e21ee commit daad6de
Show file tree
Hide file tree
Showing 8 changed files with 271 additions and 52 deletions.
8 changes: 6 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,14 @@ build:
docker build -t k2so.xnv.io/stripping .

test:
docker run -it --rm k2so.xnv.io/stripping python -m unittest discover -v tests
docker run -it --rm k2so.xnv.io/stripping python -m unittest discover -f -v tests

t:
# docker run -it --rm k2so.xnv.io/stripping python -m unittest -f tests/test_executor_serialization.py
docker run -it --rm k2so.xnv.io/stripping python -m unittest -f tests/test_weird_bug.py

ssh:
docker run -it --rm k2so.xnv.io/stripping /bin/bash
docker run -it --rm --name stripping k2so.xnv.io/stripping /bin/bash

build_package:
docker run -v $(shell pwd):/usr/src/app -it --rm k2so.xnv.io/stripping python3 setup.py sdist bdist_wheel
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
author='Adriano Marques, Nathan Martins, Thales Ribeiro',
author_email='[email protected], [email protected], [email protected]',
python_requires='>=3.7.0',
install_requires=['catalysis-client==1.0.1', 'numpy==1.18.1', 'pandas==1.0.1'],
install_requires=['catalysis-client>=1.0.1', 'numpy>=1.18.1', 'pandas>=1.0.1'],
include_package_data=True,
url='https://github.com/exponential-ventures/stripping',
packages=find_packages(exclude=['*tests*', 'test*']),
Expand Down
38 changes: 23 additions & 15 deletions stripping/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@ def __getattr__(self, attr_name):

def serialize(self) -> None:


for attr in dir(self):
if attr.startswith("_") or attr == 'self':
continue
Expand Down Expand Up @@ -92,16 +91,17 @@ def serialize(self) -> None:
else:

if isinstance(attribute, pd.DataFrame):
logging.debug(f"Context Attribute '{attr}' is a Pandas DataFrame")
attribute.to_pickle(context_file_name)
logging.debug(f"Context Attribute '{attr}' is a Pandas DataFrame")
attribute.to_pickle(context_file_name)
else:

with open(context_file_name, 'wb') as attr_file:
if isinstance(attribute, np.ndarray):
logging.debug(f"Context Attribute '{attr}' is a numpy array.")
np.save(attr_file, attribute)
else:
logging.debug(f" Context Attribute '{attr}' is a python object of type '{type(attribute)}'.")
logging.debug(
f" Context Attribute '{attr}' is a python object of type '{type(attribute)}'.")
pickle.dump(attribute, attr_file)

def deserialize(self) -> None:
Expand Down Expand Up @@ -131,17 +131,25 @@ def _deserialize(self, attr_file_name):
setattr(self, attr_file_name, np.load(attr_file))
logging.debug(f"Successfully deserialized '{attr_file_name}' as a numpy array.")
else:
with open(attr_file_name, 'rb') as attr_file:
try:
logging.debug(f"Attempting to deserialize '{attr_file_name}' with pickle...")
setattr(self, attr_file_name, pickle.load(attr_file))
logging.debug(
f"Successfully deserialized '{attr_file_name}' as a python object of "
f"type '{type(getattr(self, attr_file_name))}'")
except Exception:
logging.debug(f"Attempting to deserialize '{attr_file_name}' with numpy...")
setattr(self, attr_file_name, np.load(attr_file))
logging.debug(f"Successfully deserialized '{attr_file_name}' as a numpy array.")

try:
logging.debug(f"Attempting to deserialize '{attr_file_name}' with pd.read_pickle...")
setattr(self, attr_file_name, pd.read_pickle(attr_file_name))
logging.debug(
f"Successfully deserialized '{attr_file_name}' as a python object of "
f"type '{type(getattr(self, attr_file_name))}'")
except:
with open(attr_file_name, 'rb') as attr_file:
try:
logging.debug(f"Attempting to deserialize '{attr_file_name}' with pickle...")
setattr(self, attr_file_name, pickle.load(attr_file))
logging.debug(
f"Successfully deserialized '{attr_file_name}' as a python object of "
f"type '{type(getattr(self, attr_file_name))}'")
except Exception:
logging.debug(f"Attempting to deserialize '{attr_file_name}' with numpy...")
setattr(self, attr_file_name, np.load(attr_file))
logging.debug(f"Successfully deserialized '{attr_file_name}' as a numpy array.")


@SingletonDecorator
Expand Down
22 changes: 22 additions & 0 deletions tests/buggy_script.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
from pathlib import Path

import pandas as pd

from stripping import setup_stripping

file_path = "iris.csv"
current_dir = Path(__file__).parent.absolute()
st, ctx = setup_stripping('.stripping')


@st.step()
def load_dataset():
ctx.ds = pd.read_csv(file_path)


@st.step()
def save_new_version_9d122046_e311_4125_baa7_128eb59d048d():
new_name = f"iris_0_1_0.csv"
ctx.ds.to_csv(new_name, mode="w+", encoding='utf-8', index=False)

st.execute()
151 changes: 151 additions & 0 deletions tests/iris.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
sepal_length,sepal_width,petal_length,petal_width,species
5.1,3.5,1.4,0.2,setosa
4.9,3.0,1.4,0.2,setosa
4.7,3.2,1.3,0.2,setosa
4.6,3.1,1.5,0.2,setosa
5.0,3.6,1.4,0.2,setosa
5.4,3.9,1.7,0.4,setosa
4.6,3.4,1.4,0.3,setosa
5.0,3.4,1.5,0.2,setosa
4.4,2.9,1.4,0.2,setosa
4.9,3.1,1.5,0.1,setosa
5.4,3.7,1.5,0.2,setosa
4.8,3.4,1.6,0.2,setosa
4.8,3.0,1.4,0.1,setosa
4.3,3.0,1.1,0.1,setosa
5.8,4.0,1.2,0.2,setosa
5.7,4.4,1.5,0.4,setosa
5.4,3.9,1.3,0.4,setosa
5.1,3.5,1.4,0.3,setosa
5.7,3.8,1.7,0.3,setosa
5.1,3.8,1.5,0.3,setosa
5.4,3.4,1.7,0.2,setosa
5.1,3.7,1.5,0.4,setosa
4.6,3.6,1.0,0.2,setosa
5.1,3.3,1.7,0.5,setosa
4.8,3.4,1.9,0.2,setosa
5.0,3.0,1.6,0.2,setosa
5.0,3.4,1.6,0.4,setosa
5.2,3.5,1.5,0.2,setosa
5.2,3.4,1.4,0.2,setosa
4.7,3.2,1.6,0.2,setosa
4.8,3.1,1.6,0.2,setosa
5.4,3.4,1.5,0.4,setosa
5.2,4.1,1.5,0.1,setosa
5.5,4.2,1.4,0.2,setosa
4.9,3.1,1.5,0.1,setosa
5.0,3.2,1.2,0.2,setosa
5.5,3.5,1.3,0.2,setosa
4.9,3.1,1.5,0.1,setosa
4.4,3.0,1.3,0.2,setosa
5.1,3.4,1.5,0.2,setosa
5.0,3.5,1.3,0.3,setosa
4.5,2.3,1.3,0.3,setosa
4.4,3.2,1.3,0.2,setosa
5.0,3.5,1.6,0.6,setosa
5.1,3.8,1.9,0.4,setosa
4.8,3.0,1.4,0.3,setosa
5.1,3.8,1.6,0.2,setosa
4.6,3.2,1.4,0.2,setosa
5.3,3.7,1.5,0.2,setosa
5.0,3.3,1.4,0.2,setosa
7.0,3.2,4.7,1.4,versicolor
6.4,3.2,4.5,1.5,versicolor
6.9,3.1,4.9,1.5,versicolor
5.5,2.3,4.0,1.3,versicolor
6.5,2.8,4.6,1.5,versicolor
5.7,2.8,4.5,1.3,versicolor
6.3,3.3,4.7,1.6,versicolor
4.9,2.4,3.3,1.0,versicolor
6.6,2.9,4.6,1.3,versicolor
5.2,2.7,3.9,1.4,versicolor
5.0,2.0,3.5,1.0,versicolor
5.9,3.0,4.2,1.5,versicolor
6.0,2.2,4.0,1.0,versicolor
6.1,2.9,4.7,1.4,versicolor
5.6,2.9,3.6,1.3,versicolor
6.7,3.1,4.4,1.4,versicolor
5.6,3.0,4.5,1.5,versicolor
5.8,2.7,4.1,1.0,versicolor
6.2,2.2,4.5,1.5,versicolor
5.6,2.5,3.9,1.1,versicolor
5.9,3.2,4.8,1.8,versicolor
6.1,2.8,4.0,1.3,versicolor
6.3,2.5,4.9,1.5,versicolor
6.1,2.8,4.7,1.2,versicolor
6.4,2.9,4.3,1.3,versicolor
6.6,3.0,4.4,1.4,versicolor
6.8,2.8,4.8,1.4,versicolor
6.7,3.0,5.0,1.7,versicolor
6.0,2.9,4.5,1.5,versicolor
5.7,2.6,3.5,1.0,versicolor
5.5,2.4,3.8,1.1,versicolor
5.5,2.4,3.7,1.0,versicolor
5.8,2.7,3.9,1.2,versicolor
6.0,2.7,5.1,1.6,versicolor
5.4,3.0,4.5,1.5,versicolor
6.0,3.4,4.5,1.6,versicolor
6.7,3.1,4.7,1.5,versicolor
6.3,2.3,4.4,1.3,versicolor
5.6,3.0,4.1,1.3,versicolor
5.5,2.5,4.0,1.3,versicolor
5.5,2.6,4.4,1.2,versicolor
6.1,3.0,4.6,1.4,versicolor
5.8,2.6,4.0,1.2,versicolor
5.0,2.3,3.3,1.0,versicolor
5.6,2.7,4.2,1.3,versicolor
5.7,3.0,4.2,1.2,versicolor
5.7,2.9,4.2,1.3,versicolor
6.2,2.9,4.3,1.3,versicolor
5.1,2.5,3.0,1.1,versicolor
5.7,2.8,4.1,1.3,versicolor
6.3,3.3,6.0,2.5,virginica
5.8,2.7,5.1,1.9,virginica
7.1,3.0,5.9,2.1,virginica
6.3,2.9,5.6,1.8,virginica
6.5,3.0,5.8,2.2,virginica
7.6,3.0,6.6,2.1,virginica
4.9,2.5,4.5,1.7,virginica
7.3,2.9,6.3,1.8,virginica
6.7,2.5,5.8,1.8,virginica
7.2,3.6,6.1,2.5,virginica
6.5,3.2,5.1,2.0,virginica
6.4,2.7,5.3,1.9,virginica
6.8,3.0,5.5,2.1,virginica
5.7,2.5,5.0,2.0,virginica
5.8,2.8,5.1,2.4,virginica
6.4,3.2,5.3,2.3,virginica
6.5,3.0,5.5,1.8,virginica
7.7,3.8,6.7,2.2,virginica
7.7,2.6,6.9,2.3,virginica
6.0,2.2,5.0,1.5,virginica
6.9,3.2,5.7,2.3,virginica
5.6,2.8,4.9,2.0,virginica
7.7,2.8,6.7,2.0,virginica
6.3,2.7,4.9,1.8,virginica
6.7,3.3,5.7,2.1,virginica
7.2,3.2,6.0,1.8,virginica
6.2,2.8,4.8,1.8,virginica
6.1,3.0,4.9,1.8,virginica
6.4,2.8,5.6,2.1,virginica
7.2,3.0,5.8,1.6,virginica
7.4,2.8,6.1,1.9,virginica
7.9,3.8,6.4,2.0,virginica
6.4,2.8,5.6,2.2,virginica
6.3,2.8,5.1,1.5,virginica
6.1,2.6,5.6,1.4,virginica
7.7,3.0,6.1,2.3,virginica
6.3,3.4,5.6,2.4,virginica
6.4,3.1,5.5,1.8,virginica
6.0,3.0,4.8,1.8,virginica
6.9,3.1,5.4,2.1,virginica
6.7,3.1,5.6,2.4,virginica
6.9,3.1,5.1,2.3,virginica
5.8,2.7,5.1,1.9,virginica
6.8,3.2,5.9,2.3,virginica
6.7,3.3,5.7,2.5,virginica
6.7,3.0,5.2,2.3,virginica
6.3,2.5,5.0,1.9,virginica
6.5,3.0,5.2,2.0,virginica
6.2,3.4,5.4,2.3,virginica
5.9,3.0,5.1,1.8,virginica
22 changes: 22 additions & 0 deletions tests/non_buggy_script.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
from pathlib import Path

import pandas as pd

from stripping import setup_stripping

file_path = "iris.csv"
current_dir = Path(__file__).parent.absolute()
st, ctx = setup_stripping('.stripping')


@st.step()
def load_dataset():
ctx.ds = pd.read_csv(file_path)


# @st.step()
# def save_new_version_9d122046_e311_4125_baa7_128eb59d048d():
# new_name = f"iris_0_1_0.csv"
# ctx.ds.to_csv(new_name, mode="w+", encoding='utf-8', index=False)

st.execute()
34 changes: 0 additions & 34 deletions tests/test_executor_serialization.py

This file was deleted.

46 changes: 46 additions & 0 deletions tests/test_weird_bug.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import os
import shutil
import subprocess
from pathlib import Path
from shutil import copyfile

import asynctest

current_dir = Path(__file__).parent.absolute()


class TestWeirdBug(asynctest.TestCase):

def tearDown(self):
shutil.rmtree(os.path.join(current_dir, "log"), ignore_errors=True)
os.remove(os.path.join(current_dir, "bug.py"))

def setUp(self) -> None:
os.mkdir(os.path.join(current_dir, "log"))

def test_bug(self):
copyfile(os.path.join(current_dir, "non_buggy_script.py"), os.path.join(current_dir, "bug.py"))

proc = subprocess.Popen(
["python", "bug.py"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
cwd=current_dir,
)
_, error = proc.communicate()
self.assertEqual(error, b'')

copyfile(os.path.join(current_dir, "buggy_script.py"), os.path.join(current_dir, "bug.py"))

proc = subprocess.Popen(
["python", "bug.py"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
cwd=current_dir,
)

_, error = proc.communicate()
print(error)
# self.assertEqual(error, b'')

0 comments on commit daad6de

Please sign in to comment.