feat(env): create infra automatically (#17)

.github/workflows/docker.yml

@@ -21,18 +21,23 @@ on:
       model_version:
         required: true
         type: string
+      resource_group:
+        required: true
+        type: string
+      workspace_name:
+        required: true
+        type: string
     secrets:
       DOCKER_USERNAME:
         required: true
       DOCKER_PASSWORD:
         required: true
-      AZUREML_CREDENTIALS:
+      AZURE_CREDENTIALS:
         required: true
 
 jobs:
   build:
     runs-on: ubuntu-latest
-    environment: MLOpsPython
 
     steps:
     - uses: actions/checkout@v3
@@ -42,17 +47,18 @@ jobs:
     - name: Set up Python 3.11
       uses: actions/setup-python@v3
       with:
-        python-version: "3.11.1"
+        python-version: "3.10.13"
 
     - name: Setup Project
       run: |
+        pip install --user poetry
         chmod +x ./Makefile
         ./Makefile
 
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZUREML_CREDENTIALS}}
+        creds: ${{secrets.AZURE_CREDENTIALS}}
 
     - name: download model
       run: |
@@ -63,10 +69,13 @@ jobs:
       run: |
         az extension add -n ml
         MODEL_VERSION=${{ inputs.model_version }}
+        RESOURCE_GROUP=${{ inputs.resource_group }}
+        WORKSPACE_NAME=${{ inputs.workspace_name }}
         cd production/api/core/model
-        az ml model download --name cats-dogs-others --version $MODEL_VERSION --resource-group azure-ml --workspace-name cats-dogs
+        az ml model download --name cats-dogs-others --version $MODEL_VERSION --resource-group $RESOURCE_GROUP --workspace-name $WORKSPACE_NAME
         # find files recursively and copy them to the current directory root
-        find ./ -name '*.h5' -exec cp "{}" ./  \;
+        find ./ -name '*.keras' -exec cp "{}" ./  \;
+        ls 
         rm -r ./cats-dogs-others
     - name: Log in to Docker Hub
       uses: docker/login-action@v2

.gitignore

@@ -1 +1,2 @@
 .idea
+/get-pip.py

Makefile

@@ -1,20 +1,22 @@
-python -m pip install --upgrade pip
-python -m pip install --upgrade setuptools wheel
+install_mode=${1:-poetry}
+download_model_version=${2:-none}
 
-pip install -e packages/inference
+if [ $download_model_version != 'none' ]; then
+	# Download the model from here :
+	# https://github.com/guillaume-chervet/MLOpsPython/releases/download/v0.1.0/mlopspython_model.zip
+	# Unzip it
+	curl -L https://github.com/guillaume-chervet/MLOpsPython/releases/download/v0.1.0/mlopspython_model.zip --output model.zip
+	unzip model.zip -d ./production/api/core/model
+	rm model.zip
+fi
 
-cd packages/inference/
-python setup.py sdist bdist_wheel
-cd dist
-cp *.whl ../../../train/evaluate/packages
-cp *.whl ../../../production/api/packages
-cd ../../../
 
-pip install -e packages/extraction
-
-cd packages/extraction/
-python setup.py sdist bdist_wheel
-cd dist
-cp *.whl ../../../train/extraction/packages
-cp *.whl ../../../production/api/packages
-cd ../../../
+if [ $install_mode = 'pip' ]; then
+  echo "Installing with pip (degraded mode)"
+  chmod +x pip-install.sh
+  ./pip-install.sh
+else
+  echo "Installing with poetry (default mode)"
+  chmod +x poetry-install.sh
+  ./poetry-install.sh
+fi

README.md

@@ -8,7 +8,7 @@ A real demo of Deep Learning project with preprocessing from development to prod
 
 Requirements, Download and install:
 - Pycharm: https://www.jetbrains.com/pycharm/
-- Python 3.11.1 : https://www.python.org/downloads/
+- Python 3.10 : https://www.python.org/downloads/
 - Git: https://git-scm.com/downloads
 - Docker-desktop: https://www.docker.com/products/docker-desktop/
 - Postman: https://www.postman.com/downloads/
@@ -20,14 +20,10 @@ git clone https://www.github.com/guillaume-chervet/MLOpsPython
 
 cd MLOpsPython
 chmod +x Makefile
-./Makefile
-
-# Download the model from here :
-# https://github.com/guillaume-chervet/MLOpsPython/releases/download/v0.1.0/mlopspython_model.zip
-# Unzip it 
-curl -L https://github.com/guillaume-chervet/MLOpsPython/releases/download/v0.1.0/mlopspython_model.zip --output model.zip
-unzip model.zip -d ./production/api/core/model
-rm model.zip
+# If you have poetry installed
+./Makefile poetry 0.1.0
+# Else degraded mode
+./Makefile pip 0.1.0
 
 cd production
 docker-compose up

documentation/step_1_extraction.md

@@ -393,7 +393,7 @@ def score_data(
 def eval_model(
     scoring_result: Input(type="uri_folder"), eval_output: Output(type="uri_folder")
 ):
-    """A dummy evaluate component."""
+    """A dummy test component."""
 
     lines = [
         f"Scoring result path: {scoring_result}",

packages/extraction/.gitignore

@@ -3,4 +3,5 @@
 /build/
 /dist/
 **/*.egg-info/
-**/__pycache__/
+**/__pycache__/
+/.coverage

packages/extraction/pyproject.toml

@@ -0,0 +1,18 @@
+[tool.poetry]
+name = "mlopspython_extraction"
+version = "0.0.0"
+description = ""
+authors = ["Guillaume Chervet <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.8,<3.11"
+pymupdf = "1.23.3"
+
+
+[tool.poetry.group.dev.dependencies]
+coverage = "^7.3.1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

packages/extraction/requirements.txt

@@ -1 +1,2 @@
-pymupdf==1.21.1
+pymupdf==1.23.3 ; python_version >= "3.8" and python_version < "3.11"
+pymupdfb==1.23.3 ; python_version >= "3.8" and python_version < "3.11"

packages/extraction/setup.py

@@ -1,14 +1,13 @@
 import setuptools
-import version
 
 with open("requirements.txt") as f:
     requirements = f.read().splitlines()
 
 setuptools.setup(
     name="mlopspython-extraction",
-    version=version.VERSION,
+    version="0.0.0",
     packages=["mlopspython_extraction"],
-    package_dir={"": "src"},
+    package_dir={"": ""},
     package_data={"mlopspython_extraction": ["*"]},
     install_requires=requirements,
     author="Guillaume Chervet",
@@ -22,4 +21,4 @@
     classifiers=["Programming Language :: Python :: 3 :: Only",
                  "Programming Language :: Python :: 3.11",
                  ]
-)
+)

packages/inference/.gitignore

@@ -3,4 +3,5 @@
 /build/
 /dist/
 **/*.egg-info/
-**/__pycache__/
+**/__pycache__/
+/.coverage