adding basic solver logic

This will generate the base asp facts, along with an early set of rules to parse. I will need to add to this rules for min/max, along with the integration of cost APIs. Likely I will add AWS first, and the other list of stuffs in the README. Signed-off-by: vsoch <[email protected]>
converged-computing · Dec 4, 2022 · fb4a9bb · fb4a9bb
1 parent fdb5966
commit fb4a9bb
Show file tree

Hide file tree

Showing 19 changed files with 650 additions and 23 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -0,0 +1,6 @@
+FROM ghcr.io/autamus/clingo
+
+LABEL maintainer="Vanessasaurus <@vsoch>"
+
+# Pip not provided in this version
+RUN apt-get update && apt-get install -y python3 python3-pip git
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -0,0 +1,31 @@
+{
+    "name": "Cloud Select Development Environment",
+    "dockerFile": "Dockerfile",
+    "context": "../",
+
+    "customizations": {
+      "vscode": {
+        "settings": {
+          "terminal.integrated.defaultProfile.linux": "bash",
+
+          // Ensure that Python autocomplete works out of the box
+          "python.autoComplete.extraPaths": [
+            "/workspaces/cloud-select",
+            "/usr/local/lib/python3.8",
+            "/usr/local/lib/python3.8/site-packages"
+          ],
+          "python.analysis.extraPaths": [
+            "/workspaces/cloud-select",
+            "/usr/local/lib/python3.8",
+            "/usr/local/lib/python3.8/site-packages"
+          ]
+        },
+        // Note to Flux Developers! We can add extensions here that you like
+        "extensions": [
+		    	"ms-python.python"
+    		]
+      }
+    },
+    // Needed for git security feature (this assumes you locally cloned to flux-core)
+    "postStartCommand": "git config --global --add safe.directory /workspaces/cloud-select"
+  }
diff --git a/README.md b/README.md
@@ -1,5 +1,7 @@
 # Cloud Select
 
+![docs/assets/img/cloud-select.png](docs/assets/img/cloud-select.png)
+
 This is a tool that helps a user select a cloud. It will make it easy for an HPC user to say:
 
 > I need 4 nodes with these criteria, to run in the cloud.
@@ -123,6 +125,43 @@ I think I'm still going to use Python for faster prototyping.
 - add tests and testing workflow (and linting)
 - Add Docker build / automated builds
 - finish algorithm - should first filter based on common standard, then use clingo solver
+- ensure that required set of attributes for each instance are returned (e.g., name, cpu, memory)
+
+
+Planning for min/max stuff
+
+```lp
+need_at_least("cpus", 8).
+
+#constant max_cpus = 128.
+
+select(Cloud, Instance) :-
+  need_at_least(Name, N),
+  has_attr(Cloud, Instance, Name, M),
+  M >= N,
+  M <= max_cpus, M >= 0,
+  N <= max_cpus, N >= 0.
+```
+
+And for minimizing cost:
+
+```lp
+% generate a bunch of candidate_instance() predicates for each instance type that matches the user request
+candidate_instance(Cloud, Instance) :-
+  cloud_instance_type(Cloud, Instance),
+  instance_attr(Cloud, Instance, Name, Value) : requested_attr(Name, Value).
+
+% Tell clingo to select exactly one (at least one and at most one) of them
+1 { select(Cloud, Instance) : candidate_instance(Cloud, Instance) } 1.
+
+% associate the cost from your input facts with every candidate instance
+selected_instance_cost(Cloud, Instance, Cost) :-
+  select(Cloud, Instance),
+  instance_cost(Cloud, Instance, Cost).
+
+% tell clingo to find the solution (the one select() it got to choose with minimal cost
+#minimize { Cost,Cloud,Instance : selected_instance_cost(Cloud, Instance, Cost) }.cv
+```
 
 ## License
 

diff --git a/cloud_select/client/__init__.py b/cloud_select/client/__init__.py
@@ -41,7 +41,7 @@ def add_instance_arguments(command):
             default_type = bool
             # Assume default is always false for now
             action = "store_true"
-            default = False
+            default = None
 
         elif typ == "array":
             typ = attrs["items"]["type"]
@@ -102,6 +102,12 @@ def get_parser():
         help="directory for data cache (defaults to ~/.cloud-select/cache).",
     )
 
+    parser.add_argument(
+        "--max-results",
+        dest="max_results",
+        help="Maximum results to return per cloud provider.",
+        type=int,
+    )
     parser.add_argument(
         "--cloud",
         dest="clouds",
@@ -169,6 +175,11 @@ def get_parser():
         formatter_class=argparse.RawTextHelpFormatter,
     )
 
+    instance.add_argument(
+        "--outfile-asp",
+        dest="out",
+        help="Write ASP atoms to output file.",
+    )
     # Add attributes from spec
     add_instance_arguments(instance)
     return parser

diff --git a/cloud_select/client/instance.py b/cloud_select/client/instance.py
@@ -23,6 +23,14 @@ def main(args, parser, extra, subparser):
     # Update config settings on the fly
     cli.settings.update_params(args.config_params)
 
+    # Are we writing to an output file?
+    out = None
+    if args.out is not None and not args.quiet:
+        out = open(args.out, "w")
+    delattr(args, "out")
+
     # And select the instance
-    instances = cli.instance_select(**args.__dict__)
+    instances = cli.instance_select(**args.__dict__, out=out)
     print(json.dumps(instances, indent=4))
+    if out:
+        out.close()
diff --git a/cloud_select/main/cache.py b/cloud_select/main/cache.py
@@ -86,7 +86,7 @@ def is_expired(self, cloud_name, datatype):
         stats = os.stat(cache_file)
 
         # Convert cache_expire hours to seconds
-        expire_seconds = self._cache_expire_hours * 60
+        expire_seconds = self._cache_expire_hours * 60 * 60
 
         # And determine if the time now since modified is greater than expire
         return (time.time() - stats.st_mtime) > expire_seconds

diff --git a/cloud_select/main/client.py b/cloud_select/main/client.py
@@ -4,9 +4,13 @@
 # SPDX-License-Identifier: (MIT)
 
 
+import sys
+
 import cloud_select.defaults as defaults
 import cloud_select.main.cache as cache
 import cloud_select.main.cloud as clouds
+import cloud_select.main.schemas as schemas
+import cloud_select.main.solve as solve
 from cloud_select.logger import logger
 
 from .settings import Settings
@@ -19,6 +23,7 @@ class Client:
 
     def __init__(self, **kwargs):
         validate = kwargs.get("validate", True)
+        self.quiet = kwargs.get("quiet", False)
         self.settings = Settings(kwargs.get("settings_file"), validate)
         self.set_clouds(kwargs.get("clouds"))
 
@@ -108,7 +113,7 @@ def update_from_cache(self, items, datatype):
                 items[cloud_name] = updated
         return items
 
-    def instance_select(self, **kwargs):
+    def instance_select(self, max_results=20, out=None, **kwargs):
         """
         Select an instance.
 
@@ -122,16 +127,18 @@ def instance_select(self, **kwargs):
             )
 
         # By here we have a lookup *by cloud) of instance groups
+        # Filter down kwargs to those relevant to instances
+        properties = solve.Properties(schemas.instance_properties, **kwargs)
+        solver = solve.Solver(out=sys.stdout if not self.quiet else None)
+        max_results = max_results or self.settings.max_results or 20
 
-        # TODO
         # 1. write mapping of common features into functions
         # 2. filter down to desired set based on these common functions
-        print("VANESSA TODO")
-        import IPython
-
-        IPython.embed()
-
         for cloud_name, instance_group in instances.items():
+            # Generate facts for instances
+            solver.add_instances(cloud_name, instance_group)
+        solver.add_properties(properties.defined)
 
-            # TODO filter down to desired set based on provided attributes
-            instance_group.select(**kwargs)
+        # TODO parse instance facts from here
+        # use lookup to return to user, likely add costs
+        return solver.solve()
diff --git a/cloud_select/main/cloud/base.py b/cloud_select/main/cloud/base.py
@@ -68,10 +68,31 @@ class Instance(CloudData):
     Base of an instance.
 
     This class defines the different common attributes that we want
-    to expose. Likely we should add them via a data file.
+    to expose. If a cloud instance (json) result differs, it should
+    override this function.
     """
 
-    pass
+    @property
+    def attribute_getters(self):
+        """
+        function names we can call to get an instance attribute.
+        """
+        fields = []
+        for func in dir(self):
+            if func.startswith("attr_"):
+                fields.append(func)
+        return fields
+
+    # Attributes shared between clouds (maybe)
+    def attr_description(self):
+        return self.data.get("description")
+
+    def attr_zone(self):
+        return self.data.get("zone")
+
+    @property
+    def name(self):
+        return self.data.get("name")
 
 
 class InstanceGroup(CloudData):

diff --git a/cloud_select/main/cloud/google/instance.py b/cloud_select/main/cloud/google/instance.py
@@ -7,7 +7,38 @@
 
 
 class GoogleCloudInstance(Instance):
-    pass
+    def attr_cpus(self):
+        return self.data.get("guestCpus")
+
+    def attr_memory_gb(self):
+        return self.data["memoryMb"] / 1024
+
+    def attr_free_tier(self):
+        """
+        Determine if an instance is free tier.
+        https://cloud.google.com/free/docs/free-cloud-features#compute
+
+        This is a best effort ESTIMATION based on the name of the instance,
+        saying it _could_ be free tier but isn't necessarily if you've used
+        your monthly allowance. We will want to add that preamble somewhere.
+        """
+        # We treat booleans as lowercase strings
+        return str("micro" in self.name).lower()
+
+
+# select(Cloud, Instance) :-
+#    has_attr(Cloud, Instance, "gpu", "false"),
+#    has_attr(Cloud, Instance, "memory", 4),
+#    has_attr(Cloud, Instance, "free_tier", "false"),
+#    has_attr(Cloud, Instance, "ipv6", "false").
+
+#        "imageSpaceGb": 0,
+#        "maximumPersistentDisks": 128,
+#        "maximumPersistentDisksSizeGb": "263168",
+#        "selfLink": "https://www.googleapis.com/compute/v1/projects/dinodev/zones/us-west1-a/machineTypes/t2d-standard-8",
+#        "isSharedCpu": false
+
+# TODO logic for gpus https://cloud.google.com/compute/docs/gpus
 
 
 class GoogleCloudInstanceGroup(InstanceGroup):

diff --git a/cloud_select/main/schemas.py b/cloud_select/main/schemas.py
@@ -59,11 +59,6 @@
         "description": "Hypervisor.",
         "enum": ["xen", "nitro"],
     },
-    "max-results": {
-        "type": "number",
-        "description": "Maximum results to return per cloud provider.",
-        "default": 20,
-    },
     "memory": {
         "type": "number",
         "description": "Total memory needed and sets memory min and max to the same value.",
@@ -88,15 +83,15 @@
         "type": "number",
         "description": "Maximum amount of local instance storage in GiB. If --instance-storage-min not set, is 0.",
     },
-    "vcpus": {
+    "cpus": {
         "type": "number",
         "description": "Number of vcpus available to the instance type. Sets min and -max to the same value.",
     },
-    "vcpus-min": {
+    "cpus-min": {
         "type": "number",
         "description": "Minimum number of vcpus available to the instance type. If max not set, is infinity.",
     },
-    "vcpus-max": {
+    "cpus-max": {
         "type": "number",
         "description": "Maximum number of vcpus available to the instance type. If min not set, is 0.",
     },
@@ -165,6 +160,11 @@
 settingsProperties = {
     "cache_dir": {"type": "string"},
     "config_editor": {"type": "string"},
+    "max-results": {
+        "type": "number",
+        "description": "Maximum results to return per cloud provider.",
+        "default": 20,
+    },
     "instances": {
         "type": "object",
         "properties": instance_properties,

diff --git a/cloud_select/main/solve/__init__.py b/cloud_select/main/solve/__init__.py
@@ -0,0 +1,2 @@
+from .properties import Properties
+from .solver import Solver
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		from .properties import Properties
		from .solver import Solver