Merge pull request #405 from fabric-testbed/rel1.8

Rel1.8 - changes

.gitignore

@@ -138,10 +138,7 @@ logs
 *_creds
 *.lock
 *.log
-neo4j1/
-neo4j2/
-neo4j3/
-neo4j4/
+neo4j*/
 pdp/
 pg_data/
 schema/

Dockerfile-auth

@@ -1,7 +1,7 @@
 FROM python:3.11.0
 MAINTAINER Komal Thareja<[email protected]>
 
-ARG HANDLERS_VER=1.7.1
+ARG HANDLERS_VER=1.8.0
 
 RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app

fabric_cf/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "1.7.0"
+__version__ = "1.8.0"
 __VERSION__ = __version__

fabric_cf/actor/boot/configuration.py

@@ -45,6 +45,10 @@ def __init__(self, *, config: dict):
         if Constants.CONFIG_SECTION_O_AUTH in config:
             self.oauth = config.get(Constants.CONFIG_SECTION_O_AUTH)
 
+        self.smtp = {}
+        if Constants.CONFIG_SECTION_SMTP in config:
+            self.smtp = config.get(Constants.CONFIG_SECTION_SMTP)
+
         self.database = {}
         if Constants.CONFIG_SECTION_DATABASE in config:
             self.database = config.get(Constants.CONFIG_SECTION_DATABASE)
@@ -87,6 +91,12 @@ def get_oauth(self) -> dict:
         """
         return self.oauth
 
+    def get_smtp(self) -> dict:
+        """
+        Return smtp config
+        """
+        return self.smtp
+
     def get_database(self) -> dict:
         """
         Return database config
@@ -425,6 +435,10 @@ def get_oauth_config(self) -> dict:
             return self.global_config.get_oauth()
         return None
 
+    def get_smtp_config(self) -> dict:
+        if self.global_config:
+            return self.global_config.get_smtp()
+
     def get_actor_config(self) -> ActorConfig:
         """
         Return Actor Config

fabric_cf/actor/core/apis/abc_actor_management_object.py

@@ -238,8 +238,8 @@ def get_sites(self, *, caller: AuthToken, site: str) -> ResultSitesAvro:
     def get_reservations(self, *, caller: AuthToken, states: List[int] = None,
                          slice_id: ID = None, rid: ID = None, oidc_claim_sub: str = None,
                          email: str = None, rid_list: List[str] = None, type: str = None,
-                         site: str = None, node_id: str = None,
-                         host: str = None, ip_subnet: str = None, full: bool = False) -> ResultReservationAvro:
+                         site: str = None, node_id: str = None, host: str = None, ip_subnet: str = None,
+                         full: bool = False, start: datetime = None, end: datetime = None) -> ResultReservationAvro:
         """
         Get Reservations
         @param states states
@@ -256,10 +256,30 @@ def get_reservations(self, *, caller: AuthToken, states: List[int] = None,
         @param host host
         @param ip_subnet ip subnet
         @param full
+        @param start: start time
+        @param end: end time
 
         @return returns list of the reservations
         """
 
+    def get_components(self, *, node_id: str, rsv_type: list[str], states: list[int],
+                       component: str = None, bdf: str = None, start: datetime = None,
+                       end: datetime = None, excludes: List[str] = None) -> Dict[str, List[str]]:
+        """
+        Returns components matching the search criteria
+        @param node_id: Worker Node ID to which components belong
+        @param states: list of states used to find reservations
+        @param rsv_type: type of reservations
+        @param component: component name
+        @param bdf: Component's PCI address
+        @param start: start time
+        @param end: end time
+        @param excludes: Excludes the list of reservations
+        NOTE# For P4 switches; node_id=node+renc-p4-sw  component=ip+192.168.11.8 bdf=p1
+
+        @return Dictionary with component name as the key and value as list of associated PCI addresses in use.
+        """
+
     def get_slices(self, *, slice_id: ID, caller: AuthToken, slice_name: str = None, email: str = None,
                    states: List[int] = None, project: str = None, limit: int = None,
                    offset: int = None, user_id: str = None, search: str = None,

fabric_cf/actor/core/apis/abc_mgmt_actor.py

@@ -26,6 +26,7 @@
 from __future__ import annotations
 
 from abc import abstractmethod
+from datetime import datetime
 from typing import TYPE_CHECKING, List, Tuple, Dict
 
 from fabric_mb.message_bus.messages.delegation_avro import DelegationAvro
@@ -151,7 +152,8 @@ def accept_update_slice(self, *, slice_id: ID) -> bool:
     def get_reservations(self, *, states: List[int] = None, slice_id: ID = None,
                          rid: ID = None, oidc_claim_sub: str = None, email: str = None, rid_list: List[str] = None,
                          type: str = None, site: str = None, node_id: str = None,
-                         host: str = None, ip_subnet: str = None, full: bool = False) -> List[ReservationMng]:
+                         host: str = None, ip_subnet: str = None, full: bool = False,
+                         start: datetime = None, end: datetime = None) -> List[ReservationMng]:
         """
         Get Reservations
         @param states states
@@ -166,10 +168,31 @@ def get_reservations(self, *, states: List[int] = None, slice_id: ID = None,
         @param ip_subnet ip subnet
         @param host host
         @param full
+        @param start: start time
+        @param end: end time
         Obtains all reservations
         @return returns list of the reservations
         """
 
+    def get_components(self, *, node_id: str, rsv_type: list[str], states: list[int],
+                       component: str = None, bdf: str = None, start: datetime = None,
+                       end: datetime = None, excludes: List[str] = None) -> Dict[str, List[str]]:
+        """
+        Returns components matching the search criteria
+        @param node_id: Worker Node ID to which components belong
+        @param states: list of states used to find reservations
+        @param rsv_type: type of reservations
+        @param component: component name
+        @param bdf: Component's PCI address
+        @param start: start time
+        @param end: end time
+        @param excludes: Excludes the list of reservations
+        NOTE# For P4 switches; node_id=node+renc-p4-sw  component=ip+192.168.11.8 bdf=p1
+
+        @return Dictionary with component name as the key and value as list of associated PCI addresses in use.
+        """
+        raise NotImplementedError
+
     @abstractmethod
     def get_sites(self, *, site: str) -> List[SiteAvro] or None:
         """

fabric_cf/actor/core/common/constants.py

@@ -173,6 +173,8 @@ class Constants:
     PROPERTY_CONF_O_AUTH_TRL_REFRESH = "trl-refresh"
     PROPERTY_CONF_O_AUTH_VERIFY_EXP = "verify-exp"
 
+    CONFIG_SECTION_SMTP = "smtp"
+
     CONFIG_SECTION_DATABASE = "database"
     PROPERTY_CONF_DB_USER = "db-user"
     PROPERTY_CONF_DB_PASSWORD = "db-password"
@@ -302,10 +304,11 @@ class Constants:
 
     USER_SSH_KEY = "user.ssh.key"
     ALGORITHM = 'algorithm'
+    CORE_CAPACITY_THRESHOLD = "core_capacity_threshold"
 
     # Orchestrator Lease params
     TWO_WEEKS = timedelta(days=15)
-    DEFAULT_MAX_DURATION = TWO_WEEKS
+    DEFAULT_MAX_DURATION_IN_WEEKS = TWO_WEEKS
     LEASE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S %z"
     DEFAULT_LEASE_IN_HOURS = 24
     LONG_LIVED_SLICE_TIME_WEEKS = timedelta(weeks=26)

fabric_cf/actor/core/core/policy.py

@@ -23,6 +23,9 @@
 #
 #
 # Author: Komal Thareja ([email protected])
+import enum
+from enum import Enum
+
 from fabric_cf.actor.boot.configuration import ActorConfig
 from fabric_cf.actor.core.apis.abc_actor_mixin import ABCActorMixin
 from fabric_cf.actor.core.apis.abc_delegation import ABCDelegation
@@ -36,6 +39,19 @@
 from fabric_cf.actor.core.kernel.resource_set import ResourceSet
 
 
+class AllocationAlgorithm(Enum):
+    FirstFit = enum.auto()
+    BestFit = enum.auto()
+    WorstFit = enum.auto()
+    Random = enum.auto()
+
+    def __repr__(self):
+        return self.name
+
+    def __str__(self):
+        return self.name
+
+
 class Policy(ABCPolicy):
     """
     Base class for all policy implementations.

fabric_cf/actor/core/kernel/reservation_client.py

@@ -1004,7 +1004,8 @@ def probe_join_state(self):
 
                 # This is a regular request for modifying network resources to an upstream broker.
                 self.sequence_ticket_out += 1
-                print(f"Issuing an extend ticket {sliver_to_str(sliver=self.get_requested_resources().get_sliver())}")
+                self.logger.debug(f"Issuing an extend ticket "
+                                  f"{sliver_to_str(sliver=self.get_requested_resources().get_sliver())}")
                 RPCManagerSingleton.get().extend_ticket(reservation=self)
 
                 # Update ASM with Reservation Info

fabric_cf/actor/core/manage/actor_management_object.py

@@ -428,11 +428,17 @@ def get_sites(self, *, caller: AuthToken, site: str) -> ResultSitesAvro:
 
         return result
 
+    def get_components(self, *, node_id: str, rsv_type: list[str], states: list[int],
+                       component: str = None, bdf: str = None, start: datetime = None,
+                       end: datetime = None, excludes: List[str] = None) -> Dict[str, List[str]]:
+        return self.db.get_components(node_id=node_id, rsv_type=rsv_type, states=states,
+                                      component=component, bdf=bdf, start=start, end=end, excludes=excludes)
+
     def get_reservations(self, *, caller: AuthToken, states: List[int] = None,
                          slice_id: ID = None, rid: ID = None, oidc_claim_sub: str = None,
                          email: str = None, rid_list: List[str] = None, type: str = None,
-                         site: str = None, node_id: str = None, host: str = None,
-                         ip_subnet: str = None, full: bool = False) -> ResultReservationAvro:
+                         site: str = None, node_id: str = None, host: str = None, ip_subnet: str = None,
+                         full: bool = False, start: datetime = None, end: datetime = None) -> ResultReservationAvro:
         result = ResultReservationAvro()
         result.status = ResultAvro()
 
@@ -452,7 +458,8 @@ def get_reservations(self, *, caller: AuthToken, states: List[int] = None,
                 else:
                     res_list = self.db.get_reservations(slice_id=slice_id, rid=rid, email=email,
                                                         states=states, rsv_type=rsv_type, site=site,
-                                                        graph_node_id=node_id, host=host, ip_subnet=ip_subnet)
+                                                        graph_node_id=node_id, host=host, ip_subnet=ip_subnet,
+                                                        start=start, end=end)
             except Exception as e:
                 self.logger.error("getReservations:db access {}".format(e))
                 result.status.set_code(ErrorCodes.ErrorDatabaseError.value)

fabric_cf/actor/core/manage/kafka/kafka_actor.py

@@ -25,6 +25,7 @@
 # Author: Komal Thareja ([email protected])
 from __future__ import annotations
 
+from datetime import datetime
 from typing import List
 
 from fabric_mb.message_bus.messages.close_delegations_avro import CloseDelegationsAvro
@@ -132,7 +133,8 @@ def delete_slice(self, *, slice_id: ID) -> bool:
     def get_reservations(self, *, states: List[int] = None, slice_id: ID = None,
                          rid: ID = None, oidc_claim_sub: str = None, email: str = None, rid_list: List[str] = None,
                          type: str = None, site: str = None, node_id: str = None,
-                         host: str = None, ip_subnet: str = None, full: bool = False) -> List[ReservationMng]:
+                         host: str = None, ip_subnet: str = None, full: bool = False,
+                         start: datetime = None, end: datetime = None) -> List[ReservationMng]:
         request = GetReservationsRequestAvro()
         request = self.fill_request_by_id_message(request=request, slice_id=slice_id,
                                                   states=states, email=email, rid=rid,

fabric_cf/actor/core/manage/local/local_actor.py

@@ -26,6 +26,7 @@
 from __future__ import annotations
 
 import traceback
+from datetime import datetime
 from typing import TYPE_CHECKING, List, Tuple, Dict
 
 from fabric_mb.message_bus.messages.delegation_avro import DelegationAvro
@@ -44,12 +45,11 @@
     from fabric_mb.message_bus.messages.reservation_mng import ReservationMng
     from fabric_mb.message_bus.messages.reservation_state_avro import ReservationStateAvro
     from fabric_mb.message_bus.messages.slice_avro import SliceAvro
-    from fabric_cf.actor.core.manage.management_object import ManagementObject
     from fabric_cf.actor.security.auth_token import AuthToken
 
 
 class LocalActor(LocalProxy, ABCMgmtActor):
-    def __init__(self, *, manager: ManagementObject, auth: AuthToken):
+    def __init__(self, *, manager: ActorManagementObject, auth: AuthToken):
         super().__init__(manager=manager, auth=auth)
 
         if not isinstance(manager, ActorManagementObject):
@@ -111,13 +111,14 @@ def remove_slice(self, *, slice_id: ID) -> bool:
     def get_reservations(self, *, states: List[int] = None, slice_id: ID = None,
                          rid: ID = None, oidc_claim_sub: str = None, email: str = None, rid_list: List[str] = None,
                          type: str = None, site: str = None, node_id: str = None,
-                         host: str = None, ip_subnet: str = None, full: bool = False) -> List[ReservationMng]:
+                         host: str = None, ip_subnet: str = None, full: bool = False,
+                         start: datetime = None, end: datetime = None) -> List[ReservationMng]:
         self.clear_last()
         try:
             result = self.manager.get_reservations(caller=self.auth, states=states, slice_id=slice_id, rid=rid,
                                                    oidc_claim_sub=oidc_claim_sub, email=email, rid_list=rid_list,
                                                    type=type, site=site, node_id=node_id, host=host,
-                                                   ip_subnet=ip_subnet, full=full)
+                                                   ip_subnet=ip_subnet, full=full, start=start, end=end)
             self.last_status = result.status
 
             if result.status.get_code() == 0:
@@ -126,6 +127,16 @@ def get_reservations(self, *, states: List[int] = None, slice_id: ID = None,
         except Exception as e:
             self.on_exception(e=e, traceback_str=traceback.format_exc())
 
+    def get_components(self, *, node_id: str, rsv_type: list[str], states: list[int],
+                       component: str = None, bdf: str = None, start: datetime = None,
+                       end: datetime = None, excludes: List[str] = None) -> Dict[str, List[str]]:
+        try:
+            return self.manager.get_components(node_id=node_id, rsv_type=rsv_type, states=states,
+                                               component=component, bdf=bdf, start=start,
+                                               end=end, excludes=excludes)
+        except Exception as e:
+            self.on_exception(e=e, traceback_str=traceback.format_exc())
+
     def get_sites(self, *, site: str) -> List[SiteAvro] or None:
         self.clear_last()
         try:

fabric_cf/actor/core/manage/management_utils.py

@@ -67,6 +67,7 @@ class ManagementUtils:
     @staticmethod
     def update_slice(*, slice_obj: ABCSlice, slice_mng: SliceAvro) -> ABCSlice:
         slice_obj.set_graph_id(graph_id=slice_mng.get_graph_id())
+        slice_obj.set_lease_start(lease_start=slice_mng.get_lease_start())
         slice_obj.set_lease_end(lease_end=slice_mng.get_lease_end())
         slice_obj.set_config_properties(value=slice_mng.get_config_properties())
         return slice_obj