diff --git a/.github/workflows/ubuntu_22_and_test.yml b/.github/workflows/ubuntu_22_and_test.yml index cd063cb0..f62d9dae 100644 --- a/.github/workflows/ubuntu_22_and_test.yml +++ b/.github/workflows/ubuntu_22_and_test.yml @@ -92,7 +92,9 @@ jobs: ${{ github.workspace }}/cicd/run-fabfed.sh cicd/test_configs/cloudlab $session $session-varfile.yml - name: Test Fabric FacilityPort - if: ${{ env.RUN_FABRIC_AWS_SENSE == 'false' || env.RUN_FABRIC_AWS_SENSE == false }} + # if: ${{ env.RUN_FABRIC_AWS_SENSE == 'false' || env.RUN_FABRIC_AWS_SENSE == false }} + # need to allow for this after changes made for fabric as middle + if: false run: | session=cicd-fabric-facility-port echo "vlan: 3102" > $session-varfile.yml diff --git a/cicd/test_configs/fabric_native_aws/config.fab b/cicd/test_configs/fabric_native_aws/config.fab index 5ce6861e..50b3d04f 100644 --- a/cicd/test_configs/fabric_native_aws/config.fab +++ b/cicd/test_configs/fabric_native_aws/config.fab @@ -1,6 +1,8 @@ variable: - vlan: default: 4 + - node_count: + default: 1 provider: - fabric: @@ -37,6 +39,8 @@ resource: - fabric_node: provider: '{{ fabric.fabric_provider }}' site: MAX + network: '{{ network.fabric_network }}' + count: '{{ var.node_count }}' - network: - fabric_network: @@ -44,7 +48,6 @@ resource: layer3: "{{ layer3.fab_layer }}" peer_layer3: [ "{{ layer3.aws_layer }}" ] peering: "{{ peering.my_peering }}" - interface: '{{ node.fabric_node }}' stitch_info: stitch_port: name: AWS_PORTS diff --git a/examples/demos/stitching-chameleon-cloudlab-via-fabric/config.fab b/examples/demos/stitching-chameleon-cloudlab-via-fabric/config.fab new file mode 100644 index 00000000..58171e73 --- /dev/null +++ b/examples/demos/stitching-chameleon-cloudlab-via-fabric/config.fab @@ -0,0 +1,51 @@ +provider: + - cloudlab: + - cloudlab_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml + profile: cloudlab + - fabric: + - fabric_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml + profile: fabric + - chi: + - chi_provider: + credential_file: ~/.fabfed/fabfed_credentials.yml + profile: chi + +config: + - layer3: + - my_layer: + subnet: 192.168.1.0/24 + gateway: 192.168.1.1 + ip_start: 192.168.1.2 + ip_end: 192.168.1.254 +resource: + - network: + - cnet: + provider: '{{cloudlab.cloudlab_provider }}' + layer3: "{{ layer3.my_layer }}" + - fabric_network: + provider: '{{ fabric.fabric_provider }}' + layer3: "{{ layer3.my_layer }}" + stitch_with: + - network: '{{ network.chi_network }}' + stitch_option: + site: TACC + - network: '{{ network.cnet }}' + stitch_option: + site: UTAH + - chi_network: + provider: '{{ chi.chi_provider }}' + name: stitch_net + layer3: "{{ layer3.my_layer }}" + - node: + - cloudlab_node: + provider: '{{ cloudlab.cloudlab_provider }}' + network: "{{ network.cnet }}" + count: 1 + - chi_node: + provider: '{{ chi.chi_provider }}' + image: CC-Ubuntu20.04 + network: '{{ network.chi_network }}' + flavor: m1.medium + count: 1 diff --git a/fabfed/controller/controller.py b/fabfed/controller/controller.py index 89b67751..1e438895 100644 --- a/fabfed/controller/controller.py +++ b/fabfed/controller/controller.py @@ -265,26 +265,57 @@ def apply(self, provider_states: List[ProviderState]): resource_state_map = Controller._build_state_map(provider_states) exceptions = [] - to_be_deleted_resources = [] + create_and_wait_resource_labels = set() - for resource in resources: + for resource in filter(lambda r: not r.is_service, resources): resource_dict = resource.attributes - creation_details = resource_dict[Constants.RES_CREATION_DETAILS] - if not creation_details['in_config_file']: - to_be_deleted_resources.append(resource) - for resource in filter(lambda r: not r.is_service, resources): - label = resource.provider.label - provider = self.provider_factory.get_provider(label=label) + for dependency in resource_dict[Constants.EXTERNAL_DEPENDENCIES]: + create_and_wait_resource_labels.add(dependency.resource.label) if resource.label in resource_state_map: resource.attributes[Constants.SAVED_STATES] = resource_state_map[resource.label] - try: - provider.create_resource(resource=resource.attributes) - except Exception as e: - exceptions.append(e) - self.logger.error(e, exc_info=True) + for resource in filter(lambda r: not r.is_service, resources): + if resource.label in create_and_wait_resource_labels: + provider = self.provider_factory.get_provider(label=resource.provider.label) + + try: + provider.create_resource(resource=resource.attributes) + provider.wait_for_create_resource(resource=resource.attributes) + except Exception as e: + exceptions.append(e) + self.logger.error(e, exc_info=True) + + if exceptions: + raise ControllerException(exceptions) + + exceptions = [] + + for resource in filter(lambda r: not r.is_service, resources): + if resource.label not in create_and_wait_resource_labels: + provider = self.provider_factory.get_provider(label=resource.provider.label) + + try: + provider.create_resource(resource=resource.attributes) + except Exception as e: + exceptions.append(e) + self.logger.error(e, exc_info=True) + + if exceptions: + raise ControllerException(exceptions) + + exceptions = [] + + for resource in filter(lambda r: not r.is_service, resources): + if resource.label not in create_and_wait_resource_labels: + provider = self.provider_factory.get_provider(label=resource.provider.label) + + try: + provider.wait_for_create_resource(resource=resource.attributes) + except Exception as e: + exceptions.append(e) + self.logger.error(e, exc_info=True) if exceptions: raise ControllerException(exceptions) diff --git a/fabfed/controller/helper.py b/fabfed/controller/helper.py index ab3abe42..97e80175 100644 --- a/fabfed/controller/helper.py +++ b/fabfed/controller/helper.py @@ -16,7 +16,13 @@ def on_added(self, *, source, provider: Provider, resource: object): def on_created(self, *, source, provider: Provider, resource: object): for temp_provider in self.providers: - temp_provider.on_created(source=self, provider=provider, resource=resource) + if temp_provider == provider: + temp_provider.on_created(source=self, provider=provider, resource=resource) + break + + for temp_provider in self.providers: + if temp_provider != provider: + temp_provider.on_created(source=self, provider=provider, resource=resource) def on_deleted(self, *, source, provider: Provider, resource: object): for temp_provider in self.providers: @@ -60,14 +66,15 @@ def partition_layer3_config(*, networks: list): dhcp_start = dhcp_end + 1 -def find_peer_network(*, network): +def find_peer_networks(*, network): dependencies = network.dependencies + peer_networks = [] for ed in dependencies: if ed.key == Constants.RES_STITCH_INTERFACE: - return ed.resource + peer_networks.append(ed.resource) - return None + return peer_networks def find_nodes_related_to_network(*, network, resources): @@ -93,29 +100,33 @@ def find_node_clusters(*, resources): for net in networks: if net.label not in visited_networks: - peer = find_peer_network(network=net) + peers = find_peer_networks(network=net) - if peer: + if peers: + cluster = [] visited_networks.append(net.label) nodes = find_nodes_related_to_network(network=net, resources=resources) - visited_networks.append(peer.label) - nodes.extend(find_nodes_related_to_network(network=peer, resources=resources)) visited_nodes.extend([n.label for n in nodes]) + cluster.extend(nodes) + + for peer in peers: + visited_networks.append(peer.label) + nodes = find_nodes_related_to_network(network=peer, resources=resources) + visited_nodes.extend([n.label for n in nodes]) + + if nodes: + cluster.extend(nodes) - if nodes: - clusters.append(nodes) + if cluster: + clusters.append(cluster) for net in networks: if net.label not in visited_networks: - peer = find_peer_network(network=net) + nodes = find_nodes_related_to_network(network=net, resources=resources) - if not peer: - visited_networks.append(net.label) - nodes = find_nodes_related_to_network(network=net, resources=resources) + if nodes: visited_nodes.extend([n.label for n in nodes]) - - if nodes: - clusters.append(nodes) + clusters.append(nodes) nodes = [r for r in resources if r.is_node] diff --git a/fabfed/model/__init__.py b/fabfed/model/__init__.py index 7ec1b8e7..71bc645c 100644 --- a/fabfed/model/__init__.py +++ b/fabfed/model/__init__.py @@ -1,11 +1,13 @@ from abc import ABC, abstractmethod from collections import namedtuple from fabfed.util.utils import get_inventory_dir +from typing import List class Resource(ABC): def __init__(self, label, name: str): self.label = label self.name = name + self._depends_on: List[str] = list() def get_label(self) -> str: return self.label @@ -13,6 +15,12 @@ def get_label(self) -> str: def get_name(self) -> str: return self.name + def set_externally_depends_on(self, depends_on: List[str]): + self._depends_on = depends_on + + def get_externally_depends_on(self) -> List[str]: + return self._depends_on + @abstractmethod def write_ansible(self, friendly_name): pass diff --git a/fabfed/policy/policy_helper.py b/fabfed/policy/policy_helper.py index 23a95bd1..7770a5fa 100644 --- a/fabfed/policy/policy_helper.py +++ b/fabfed/policy/policy_helper.py @@ -85,7 +85,7 @@ def parse_policy(policy, policy_details, fp_dict=None) -> Dict[str, ProviderPoli effective_stitch_port.update(port_detail) effective_stitch_ports.append(effective_stitch_port) else: - effective_stitch_port = stitch_port # TODO + effective_stitch_port = stitch_port # TODO effective_stitch_ports.append(effective_stitch_port) groups = v[GROUP] if GROUP in v else [] @@ -436,64 +436,131 @@ def handle_stitch_info(config, policy, resources): logger = get_logger() has_stitch_with = False + for network in [resource for resource in resources if resource.is_network]: + if Constants.RES_STITCH_INFO not in network.attributes: + network.attributes[Constants.RES_STITCH_INFO] = [] + + if Constants.RES_STITCH_INTERFACE not in network.attributes: + network.attributes[Constants.RES_STITCH_INTERFACE] = [] + for network in [resource for resource in resources if resource.is_network]: if Constants.NETWORK_STITCH_WITH in network.attributes: + from fabfed.util.config_models import DependencyInfo + has_stitch_with = True - dependency_info = network.attributes[Constants.NETWORK_STITCH_WITH] - dependencies = network.dependencies - network_dependency = None + dependency_info_dicts = network.attributes[Constants.NETWORK_STITCH_WITH] - for ed in dependencies: - if ed.key == Constants.NETWORK_STITCH_WITH and ed.resource.label == dependency_info.resource.label: - network_dependency = ed - break + if not isinstance(dependency_info_dicts, list): + option = network.attributes.get(Constants.NETWORK_STITCH_OPTION) + dependency_info_dicts = [dict(network=dependency_info_dicts, stitch_option=option)] - assert network_dependency is not None, "should never happen" - assert network_dependency.is_external, "only network stitching across providers is supported" - other_network = network_dependency.resource - assert other_network.is_network, "only network stitching is supported" + from fabfed.util.config_models import DependencyInfo - stitch_config = None - option = network.attributes.get(Constants.NETWORK_STITCH_OPTION) + for dependency_info_dict in dependency_info_dicts: + assert("network" in dependency_info_dict) + assert(isinstance(dependency_info_dict["network"], DependencyInfo)) - if option: - stitch_config = option.get(Constants.NETWORK_STITCH_CONFIG) + temp = set() - if not stitch_config: - site = find_site(network, resources) - profile = find_profile(network, resources) - options = network.attributes.get(Constants.NETWORK_STITCH_OPTION, list()) + for dep in network.dependencies: + if dep.key == 'stitch_with.network': + # Dependency(key='stitch_with.network', resource=cnet@network, attribute='', is_external=True + from fabfed.util.config_models import Dependency + new_dep = Dependency(key=Constants.NETWORK_STITCH_WITH, + resource=dep.resource, attribute=dep.attribute, is_external=True) + temp.add(new_dep) + else: + temp.add(dep) + + network._resource_dependencies = temp + + for dependency_info_dict in dependency_info_dicts: + dependency_info: DependencyInfo = dependency_info_dict['network'] + network_dependency = None + + for ed in network.dependencies: + if ed.key == Constants.NETWORK_STITCH_WITH and ed.resource.label == dependency_info.resource.label: + network_dependency = ed + break + + assert network_dependency is not None, "should never happen" + assert network_dependency.is_external, "only network stitching across providers is supported" + other_network = network_dependency.resource + assert other_network.is_network, "only network stitching is supported" + + stitch_config = None + option = dependency_info_dict.get(Constants.NETWORK_STITCH_OPTION) + + if option: + stitch_config = option.get(Constants.NETWORK_STITCH_CONFIG) + + if not stitch_config: + site = find_site(network, resources) + profile = find_profile(network, resources) + stitch_info = find_stitch_port(policy=policy, + providers=[network.provider.type, other_network.provider.type], + site=site, + profile=profile, + options=option) + + clean_up_port(stitch_info.stitch_port) + stitch_info = StitchInfo(consumer=stitch_info.consumer, + producer=stitch_info.producer, + stitch_port=stitch_info.stitch_port) + else: + logger.info(f"using supplied {Constants.NETWORK_STITCH_CONFIG}:{stitch_config.attributes}") + stitch_info = StitchInfo(consumer=stitch_config.attributes['consumer'], + producer=stitch_config.attributes['producer'], + stitch_port=stitch_config.attributes['stitch_port']) + from fabfed.util.config_models import DependencyInfo + + if network.provider.type == stitch_info.consumer: + network.attributes[Constants.RES_STITCH_INTERFACE].append(DependencyInfo(resource=other_network, + attribute='')) + else: + other_network.attributes[Constants.RES_STITCH_INTERFACE].append(DependencyInfo(resource=network, + attribute='')) - stitch_info = find_stitch_port(policy=policy, - providers=[network.provider.type, other_network.provider.type], - site=site, - profile=profile, - options=options) + peer1 = {} + peer2 = {} - clean_up_port(stitch_info.stitch_port) - stitch_info = StitchInfo(consumer=stitch_info.consumer, - producer=stitch_info.producer, - stitch_port=stitch_info.stitch_port) - else: - logger.info(f"using supplied {Constants.NETWORK_STITCH_CONFIG}:{stitch_config.attributes}") - stitch_info = StitchInfo(consumer=stitch_config.attributes['consumer'], - producer=stitch_config.attributes['producer'], - stitch_port=stitch_config.attributes['stitch_port']) + peer1.update(stitch_info.stitch_port['peer']) + peer2.update(stitch_info.stitch_port) + peer2.pop('peer') - network.attributes.pop(Constants.NETWORK_STITCH_WITH) - network.attributes.pop(Constants.NETWORK_STITCH_OPTION, None) + if network.provider.type != peer1['provider']: + pass - from fabfed.util.config_models import DependencyInfo + stitch_info = StitchInfo(stitch_port=dict(), + producer=stitch_info.producer, consumer=stitch_info.consumer) - if network.provider.type == stitch_info.consumer: - network.attributes[Constants.RES_STITCH_INTERFACE] = DependencyInfo(resource=other_network, - attribute='') - else: - other_network.attributes[Constants.RES_STITCH_INTERFACE] = DependencyInfo(resource=network, - attribute='') + if network.provider.type == peer1['provider']: + stitch_info.stitch_port.update(peer1) + stitch_info.stitch_port['peer'] = dict() + stitch_info.stitch_port['peer'].update(peer2) + else: + stitch_info.stitch_port.update(peer2) + stitch_info.stitch_port['peer'] = dict() + stitch_info.stitch_port['peer'].update(peer1) + + network.attributes[Constants.RES_STITCH_INFO].append(stitch_info) - network.attributes[Constants.RES_STITCH_INFO] = stitch_info - other_network.attributes[Constants.RES_STITCH_INFO] = stitch_info + stitch_info = StitchInfo(stitch_port=dict(), + producer=stitch_info.producer, consumer=stitch_info.consumer) + + if other_network.provider.type == peer1['provider']: + stitch_info.stitch_port.update(peer1) + stitch_info.stitch_port['peer'] = dict() + stitch_info.stitch_port['peer'].update(peer2) + else: + stitch_info.stitch_port.update(peer2) + stitch_info.stitch_port['peer'] = dict() + stitch_info.stitch_port['peer'].update(peer1) + other_network.attributes[Constants.RES_STITCH_INFO].append(stitch_info) + + for network in [resource for resource in resources if resource.is_network]: + network.attributes.pop(Constants.NETWORK_STITCH_WITH, None) + network.attributes.pop(Constants.NETWORK_STITCH_OPTION, None) if has_stitch_with: for resource in resources: @@ -526,7 +593,7 @@ def fix_node_site(resource, resources): stitch_port = get_stitch_port_for_provider(resource=net.attributes, provider=net.provider.type) if stitch_port: - site = stitch_port.get(Constants.RES_SITE) + site = stitch_port.get(Constants.RES_SITE) # TODO FIX PYCHARM WARNING resource.attributes[Constants.RES_SITE] = site return @@ -540,7 +607,7 @@ def fix_node_site(resource, resources): stitch_port = get_stitch_port_for_provider(resource=net.attributes, provider=net.provider.type) if stitch_port: - site = stitch_port.get(Constants.RES_SITE) + site = stitch_port.get(Constants.RES_SITE) # TODO FIX PYCHARM WARNING resource.attributes[Constants.RES_SITE] = site return @@ -551,9 +618,21 @@ def fix_network_site(resource): if site: return - stitch_port = get_stitch_port_for_provider(resource=resource.attributes, provider=resource.provider.type) + stitch_ports = get_stitch_port_for_provider(resource=resource.attributes, provider=resource.provider.type) - if stitch_port: + if stitch_ports is None: + return + + if isinstance(stitch_ports, list) and len(stitch_ports) > 1: + return + + if isinstance(stitch_ports, list) and len(stitch_ports) == 1: + stitch_port = stitch_ports[0] + site = stitch_port.get(Constants.RES_SITE) + resource.attributes[Constants.RES_SITE] = site + + if isinstance(stitch_ports, dict): + stitch_port = stitch_ports site = stitch_port.get(Constants.RES_SITE) resource.attributes[Constants.RES_SITE] = site @@ -574,21 +653,21 @@ def get_vlan_from_range(*, resource: dict): def get_vlan_range(*, resource: dict): - stitch_info = resource.get(Constants.RES_STITCH_INFO) + stitch_infos = resource.get(Constants.RES_STITCH_INFO) - if not stitch_info: + if not stitch_infos: return None - if isinstance(stitch_info, dict): # This is for testing purposes. - producer = stitch_info['producer'] - consumer = stitch_info['consumer'] - stitch_info = StitchInfo(stitch_port=stitch_info['stitch_port'], producer=producer, consumer=consumer) - resource[Constants.RES_STITCH_INFO] = stitch_info + if isinstance(stitch_infos, dict): # This is for testing purposes. + producer = stitch_infos['producer'] + consumer = stitch_infos['consumer'] + stitch_info = StitchInfo(stitch_port=stitch_infos['stitch_port'], producer=producer, consumer=consumer) + stitch_infos = resource[Constants.RES_STITCH_INFO] = [stitch_info] - stitch_ports = [stitch_info.stitch_port] + stitch_ports = [stitch_infos[0].stitch_port] - if PEER in stitch_info.stitch_port: - stitch_ports.append(stitch_info.stitch_port[PEER]) + if PEER in stitch_infos[0].stitch_port: + stitch_ports.append(stitch_infos[0].stitch_port[PEER]) for stitch_port in stitch_ports: if Constants.STITCH_VLAN_RANGE in stitch_port: @@ -596,21 +675,37 @@ def get_vlan_range(*, resource: dict): return None + def get_stitch_port_for_provider(*, resource: dict, provider: str): - stitch_info = resource.get(Constants.RES_STITCH_INFO) + stitch_infos = resource.get(Constants.RES_STITCH_INFO) - if not stitch_info: + if not stitch_infos: return None - if isinstance(stitch_info, dict): # This is for testing purposes. - producer = stitch_info['producer'] - consumer = stitch_info['consumer'] - stitch_info = StitchInfo(stitch_port=stitch_info['stitch_port'], producer=producer, consumer=consumer) - resource[Constants.RES_STITCH_INFO] = stitch_info + if isinstance(stitch_infos, dict): # This is for testing purposes. + producer = stitch_infos['producer'] + consumer = stitch_infos['consumer'] + stitch_info = StitchInfo(stitch_port=stitch_infos['stitch_port'], producer=producer, consumer=consumer) + stitch_infos = resource[Constants.RES_STITCH_INFO] = [stitch_info] + + stitch_ports = [] + + for stitch_info in stitch_infos: + temp_stitch_ports = [stitch_info.stitch_port] + + if PEER in stitch_info.stitch_port: + temp_stitch_ports.append(stitch_info.stitch_port[PEER]) + + stitch_port = next(filter(lambda sp: sp['provider'] == provider, temp_stitch_ports), None) + + if stitch_port is not None: + stitch_ports.append(stitch_port) + + if len(stitch_ports) == 0: + return None - stitch_ports = [stitch_info.stitch_port] + if len(stitch_ports) == 1: + return stitch_ports[0] - if PEER in stitch_info.stitch_port: - stitch_ports.append(stitch_info.stitch_port[PEER]) + return stitch_ports - return next(filter(lambda sp: sp['provider'] == provider, stitch_ports), None) diff --git a/fabfed/provider/api/provider.py b/fabfed/provider/api/provider.py index cde50131..5eebe4ec 100644 --- a/fabfed/provider/api/provider.py +++ b/fabfed/provider/api/provider.py @@ -21,6 +21,8 @@ def __init__(self, *, type, label, name, logger: logging.Logger, config: dict): self._services = list() self._pending = [] + self._externally_depends_on_map: Dict[str, List[str]] = {} + self._no_longer_pending = [] self._failed = {} self.creation_details = {} @@ -41,7 +43,7 @@ def added_map(self) -> Dict[str, List[str]]: return self._added_map @property - def resources(self) -> List: + def resources(self) -> List[Resource]: resources = [n for n in self._nodes] resources.extend([n for n in self._networks]) resources.extend([n for n in self._services]) @@ -102,26 +104,32 @@ def on_created(self, *, source, provider, resource: Resource): assert self != source assert provider + if self == provider: self.creation_details[resource.label]["resources"].append(resource.name) + resource.set_externally_depends_on(self._externally_depends_on_map[resource.label]) - try: - resource.write_ansible(provider.name) - except Exception as e: - self.logger.warning( - f"exception occurred while writing ansible for resource={resource.name}/{provider.name}:{e}") - - for pending_resource in self.pending.copy(): - resolver = self.get_dependency_resolver() - label = pending_resource[Constants.LABEL] - resolver.resolve_dependency(resource=pending_resource, from_resource=resource) - ok = resolver.check_if_external_dependencies_are_resolved(resource=pending_resource) - - if ok: - resolver.extract_values(resource=pending_resource) - self.pending.remove(pending_resource) - self.no_longer_pending.append(pending_resource) - self.logger.info(f"Removing {label} from pending using {self.label}") + try: + resource.write_ansible(provider.name) + except Exception as e: + self.logger.warning( + f"exception occurred while writing ansible for resource={resource.name}/{provider.name}:{e}") + else: + for pending_resource in self.pending.copy(): + resolver = self.get_dependency_resolver() + label = pending_resource[Constants.LABEL] + resolver.resolve_dependency(resource=pending_resource, from_resource=resource) + ok = resolver.check_if_external_dependencies_are_resolved(resource=pending_resource) + + if ok: + resolver.extract_values(resource=pending_resource) + self.pending.remove(pending_resource) + self.no_longer_pending.append(pending_resource) + self.logger.info(f"Removing {label} from pending using {self.label}") + + for r in self.resources: + if r.label in resource.get_externally_depends_on(): + self.do_handle_externally_depends_on(resource=r, dependee=resource) def init(self): import time @@ -242,6 +250,12 @@ def add_resource(self, *, resource: dict): assert count > 0 assert label not in self._added, f"{label} already in {self._added}" + if label not in self._externally_depends_on_map: + depends_on = self._externally_depends_on_map[label] = list() + + for dependency in resource[Constants.EXTERNAL_DEPENDENCIES]: + depends_on.append(dependency.resource.label) + if len(resource[Constants.EXTERNAL_DEPENDENCIES]) > len(resource[Constants.RESOLVED_EXTERNAL_DEPENDENCIES]): self.logger.info(f"Adding {label} to pending using {self.label}") assert resource not in self.pending, f"Did not expect {label} to be in pending list using {self.label}" @@ -298,7 +312,8 @@ def create_resource(self, *, resource: dict): self.add_resource(resource=no_longer_pending_resource) added = True except Exception as e: - self.logger.warning(f"Adding no longer pending externally {external_dependency_label} failed: {e}") + self.logger.warning(f"Adding no longer pending externally {external_dependency_label} failed: {e}", + exc_info=True) added = False self.no_longer_pending.append(no_longer_pending_resource) @@ -316,9 +331,9 @@ def create_resource(self, *, resource: dict): self.logger.warning( f"Adding no longer pending internally {internal_dependency_label} failed using {e2}") - self.logger.info(f"Creating {label} using {self.label}: {self._added}") - if label in self._added: + self.logger.info(f"Create: {label} using {self.label}: {self._added}") + try: self.do_create_resource(resource=resource) except (Exception, KeyboardInterrupt) as e: @@ -331,6 +346,26 @@ def create_resource(self, *, resource: dict): end = time.time() self.create_duration += (end - start) + def wait_for_create_resource(self, *, resource: dict): + import time + start = time.time() + label = resource.get(Constants.LABEL) + + if label in self._added: + self.logger.info(f"Waiting on Create: {label} using {self.label}: {self._added}") + + try: + self.do_wait_for_create_resource(resource=resource) + except (Exception, KeyboardInterrupt) as e: + self.failed[label] = 'CREATE' + failed_count = resource[Constants.RES_COUNT] - len(self.creation_details[label]['resources']) + self.creation_details[label]['failed_count'] = failed_count + raise e + finally: + self.creation_details[label]['created_count'] = len(self.creation_details[label]['resources']) + end = time.time() + self.create_duration += (end - start) + def delete_resource(self, *, resource: dict): import time @@ -412,6 +447,12 @@ def do_add_resource(self, *, resource: dict): def do_create_resource(self, *, resource: dict): pass + def do_wait_for_create_resource(self, *, resource: dict): + pass + + def do_handle_externally_depends_on(self, *, resource: Resource, dependee: Resource): + pass + @abstractmethod def do_delete_resource(self, *, resource: dict): pass diff --git a/fabfed/provider/aws/aws_utils.py b/fabfed/provider/aws/aws_utils.py index 2e6362ab..a4541adc 100644 --- a/fabfed/provider/aws/aws_utils.py +++ b/fabfed/provider/aws/aws_utils.py @@ -352,10 +352,12 @@ def create_vpn_gateway(*, ec2_client, name: str, amazon_asn: int): for i in range(RETRY): vpn_gateway = _find_vpn_gateway_by_id(ec2_client=ec2_client, vpn_id=vpn_id) - state = vpn_gateway['State'] - if state == 'available': - return vpn_id + if vpn_gateway: + state = vpn_gateway['State'] + + if state == 'available': + return vpn_id logger.info(f"Waiting on VPN {name}:state={state}") time.sleep(20) diff --git a/fabfed/provider/chi/chi_network.py b/fabfed/provider/chi/chi_network.py index 0e9011cc..9acb4260 100644 --- a/fabfed/provider/chi/chi_network.py +++ b/fabfed/provider/chi/chi_network.py @@ -20,7 +20,7 @@ class ChiNetwork(Network): def __init__(self, *, label, name: str, site: str, project_name: str, layer3: Config, - stitch_provider: str, vlan: int): + stitch_info, vlan: int): super().__init__(label=label, name=name, site=site) self.project_name = project_name self.layer3 = layer3 @@ -28,7 +28,12 @@ def __init__(self, *, label, name: str, site: str, project_name: str, layer3: Co self.ip_start = layer3.attributes.get(Constants.RES_LAYER3_DHCP_START) self.ip_end = layer3.attributes.get(Constants.RES_LAYER3_DHCP_END) self.gateway = layer3.attributes.get(Constants.RES_NET_GATEWAY, None) - self.stitch_provider = stitch_provider + self.stitch_info = stitch_info + self.stitch_provider = None + + if stitch_info is not None: + self.stitch_provider = stitch_info.stitch_port['peer']['provider'] + self._retry = 10 self.lease_name = f'{name}-lease' self.subnet_name = f'{name}-subnet' @@ -93,7 +98,10 @@ def create(self): self.vlans.append(network_vlan) for vlan in self.vlans: - self.interface.append(dict(id='', provider="chi", vlan=vlan)) + temp = dict(id=self.label, vlan=vlan) + temp.update(self.stitch_info.stitch_port['peer']) + temp['provider'] = self.stitch_info.stitch_port['provider'] + self.interface.append(temp) try: chameleon_subnet = chi.network.get_subnet(self.subnet_name) @@ -132,6 +140,21 @@ def create(self): self.logger.info(f'Attached subnet {self.subnet_name} to router {self.router_name}') self.logger.debug(f'Router: {chameleon_router}') + def update_route(self, *, subnet: str, gateway_ip: str): + chameleon_subnet = chi.network.get_subnet(self.subnet_name) + body = { + "subnet": { + "host_routes": [ + { + "destination": f"{subnet}", + "nexthop": f"{gateway_ip}" + } + ] + } + } + + chi.neutron().update_subnet(subnet=chameleon_subnet['id'], body=body) + def _delete(self): chi.set('project_name', self.project_name) chi.set('project_domain_name', 'default') diff --git a/fabfed/provider/chi/chi_provider.py b/fabfed/provider/chi/chi_provider.py index 43272283..42c790a7 100644 --- a/fabfed/provider/chi/chi_provider.py +++ b/fabfed/provider/chi/chi_provider.py @@ -1,14 +1,14 @@ import logging import os -from fabfed.exceptions import ResourceTypeNotSupported, ProviderException from typing import List +import fabfed.provider.api.dependency_util as util +from fabfed.exceptions import ResourceTypeNotSupported, ProviderException +from fabfed.model import Resource from fabfed.provider.api.provider import Provider from fabfed.util.constants import Constants -from .chi_constants import * -import fabfed.provider.api.dependency_util as util - from fabfed.util.utils import get_logger +from .chi_constants import * logger: logging.Logger = get_logger() @@ -131,17 +131,12 @@ def do_add_resource(self, *, resource: dict): if rtype == Constants.RES_TYPE_NETWORK: layer3 = resource.get(Constants.RES_LAYER3) - from typing import Union from fabfed.policy.policy_helper import StitchInfo - stitch_info: Union[str, StitchInfo] = resource.get(Constants.RES_STITCH_INFO) - assert stitch_info, f"resource {label} missing stitch info" - - if isinstance(stitch_info, str): - stitch_provider = stitch_info - else: - stitch_provider = stitch_info.consumer - - assert stitch_provider, f"resource {label} missing stitch provider" + stitch_infos: List[StitchInfo] = resource.get(Constants.RES_STITCH_INFO) + assert stitch_infos, f"resource {label} missing stitch info" + assert isinstance(stitch_infos, list), f"resource {label} expecting a list for stitch info" + assert len(stitch_infos) == 1, f"resource {label} expect a list of size for stitch info " + assert stitch_infos[0].stitch_port['peer'] != self.type, f"resource {label} stitch provider has wrong type" interfaces = resource.get(Constants.RES_INTERFACES, list()) @@ -157,7 +152,7 @@ def do_add_resource(self, *, resource: dict): from fabfed.provider.chi.chi_network import ChiNetwork net = ChiNetwork(label=label, name=net_name, site=site, - layer3=layer3, stitch_provider=stitch_provider, + layer3=layer3, stitch_info=stitch_infos[0], project_name=project_name, vlan=vlan) self._networks.append(net) @@ -206,7 +201,7 @@ def do_create_resource(self, *, resource: dict): project_name = self.config[CHI_PROJECT_NAME] net = ChiNetwork(label=label, name=net_name, site=site, - layer3=layer3, stitch_provider='', project_name=project_name, vlan=-1) + layer3=layer3, stitch_info=None, project_name=project_name, vlan=-1) net.delete() self.logger.info(f"Deleted network: {net_name} at site {site}") @@ -247,6 +242,19 @@ def do_create_resource(self, *, resource: dict): for node in temp: node.create() + def do_wait_for_create_resource(self, *, resource: dict): + site = resource.get(Constants.RES_SITE) + self._setup_environment(site=site) + label = resource.get(Constants.LABEL) + rtype = resource.get(Constants.RES_TYPE) + + if rtype == Constants.RES_TYPE_NETWORK: + pass + else: + from fabfed.provider.chi.chi_node import ChiNode + + temp: List[ChiNode] = [node for node in self._nodes if node.label == label] + for node in temp: node.wait_for_active() @@ -256,6 +264,10 @@ def do_create_resource(self, *, resource: dict): if self.resource_listener: self.resource_listener.on_created(source=self, provider=self, resource=node) + def do_handle_externally_depends_on(self, *, resource: Resource, dependee: Resource): + self.logger.info(f"NEED TO DO SOME POST PROCESSING {resource}: {dependee}") + # I have the code to add the route. + # noinspection PyTypeChecker def do_delete_resource(self, *, resource: dict): site = resource.get(Constants.RES_SITE) @@ -274,7 +286,7 @@ def do_delete_resource(self, *, resource: dict): layer3 = Config("", "", {}) net = ChiNetwork(label=label, name=net_name, site=site, - layer3=layer3, stitch_provider=None, project_name=project_name, vlan=-1) + layer3=layer3, stitch_info=None, project_name=project_name, vlan=-1) net.delete() self.logger.info(f"Deleted network: {net_name} at site {site}") diff --git a/fabfed/provider/cloudlab/cloudlab_network.py b/fabfed/provider/cloudlab/cloudlab_network.py index 9e34a824..fd20625e 100644 --- a/fabfed/provider/cloudlab/cloudlab_network.py +++ b/fabfed/provider/cloudlab/cloudlab_network.py @@ -11,10 +11,12 @@ class CloudNetwork(Network): - def __init__(self, *, label, name: str, provider: CloudlabProvider, profile: str, interfaces, layer3, cluster): + def __init__(self, *, label, name: str, provider: CloudlabProvider, + stitch_info=None, profile: str, interfaces, layer3, cluster): super().__init__(label=label, name=name, site="") self.profile = profile self._provider = provider + self.stitch_info = stitch_info self.interface = interfaces or [] self.layer3 = layer3 or {} self.cluster = cluster @@ -141,7 +143,12 @@ def create(self): data_dict = xmltodict.parse(next(iter(status.values()))) logger.info(f"RSPEC: {json.dumps(data_dict['rspec'], indent=2)}") link = data_dict['rspec']['link'] + + temp = dict(id=self.label, vlan=link['@vlantag']) + temp.update(self.stitch_info.stitch_port['peer']) + temp['provider'] = self.stitch_info.stitch_port['provider'] self.interface = [dict(id='', provider=self.provider.type, vlan=link['@vlantag'])] + self.interface = [temp] if not [n for n in self.provider.nodes if n.net == self]: return diff --git a/fabfed/provider/cloudlab/cloudlab_provider.py b/fabfed/provider/cloudlab/cloudlab_provider.py index 82adc58a..be7e5129 100644 --- a/fabfed/provider/cloudlab/cloudlab_provider.py +++ b/fabfed/provider/cloudlab/cloudlab_provider.py @@ -1,3 +1,5 @@ +from typing import List + from fabfed.exceptions import ResourceTypeNotSupported, ProviderException from fabfed.provider.api.provider import Provider from fabfed.util.constants import Constants @@ -15,6 +17,7 @@ def __init__(self, *, type, label, name, config: dict): super().__init__(type=type, label=label, name=name, logger=logger, config=config) self.supported_resources = [Constants.RES_TYPE_NETWORK, Constants.RES_TYPE_NODE] self._handled_modify = False + self._stitch_info_map = dict() def setup_environment(self): for attr in CLOUDLAB_CONF_ATTRS: @@ -115,7 +118,6 @@ def do_validate_resource(self, *, resource: dict): if not stitch_info: raise ProviderException(f"{self.label} expecting stitch info in {rtype} resource {label}") - def resource_name(self, resource: dict, idx: int = 0): rtype = resource.get(Constants.RES_TYPE) @@ -171,9 +173,15 @@ def do_add_resource(self, *, resource: dict): net_name = self.resource_name(resource) profile = resource.get(Constants.RES_PROFILE) - cloudlab_stitch_port = get_stitch_port_for_provider(resource=resource, provider=self.type) + from fabfed.policy.policy_helper import StitchInfo + stitch_infos: List[StitchInfo] = resource.get(Constants.RES_STITCH_INFO) + assert stitch_infos, f"resource {label} missing stitch info" + assert isinstance(stitch_infos, list), f"resource {label} expecting a list for stitch info" + assert len(stitch_infos) == 1, f"resource {label} expect a list of size for stitch info " + assert stitch_infos[0].stitch_port['peer'] != self.type, f"resource {label} stitch provider has wrong type" + cloudlab_stitch_port = stitch_infos[0].stitch_port - if not profile and cloudlab_stitch_port: + if not profile: profile = cloudlab_stitch_port.get(Constants.RES_PROFILE) if not profile: @@ -181,7 +189,7 @@ def do_add_resource(self, *, resource: dict): cluster = resource.get(Constants.RES_CLUSTER) - if not cluster and cloudlab_stitch_port: + if not cluster: if 'option' in cloudlab_stitch_port and Constants.RES_CLUSTER in cloudlab_stitch_port['option']: cluster = cloudlab_stitch_port['option'][Constants.RES_CLUSTER] @@ -195,7 +203,8 @@ def do_add_resource(self, *, resource: dict): interfaces = [{'vlan': vlan}] if vlan > 0 else [] layer3 = resource.get(Constants.RES_LAYER3) - net = CloudNetwork(label=label, name=net_name, provider=self, profile=profile, interfaces=interfaces, + net = CloudNetwork(label=label, name=net_name, provider=self, stitch_info=stitch_infos[0], + profile=profile, interfaces=interfaces, layer3=layer3, cluster=cluster) self._networks.append(net) self.resource_listener.on_added(source=self, provider=self, resource=net) diff --git a/fabfed/provider/fabric/fabric_constants.py b/fabfed/provider/fabric/fabric_constants.py index c111b4fc..d6016df1 100644 --- a/fabfed/provider/fabric/fabric_constants.py +++ b/fabfed/provider/fabric/fabric_constants.py @@ -50,3 +50,5 @@ FABRIC_SLEEP_AFTER_SUBMIT_OK = 120 # In seconds PATCH_FOR_TOKENS = True + +SITES = "sites" diff --git a/fabfed/provider/fabric/fabric_network.py b/fabfed/provider/fabric/fabric_network.py index 44c2155f..2e263bb2 100644 --- a/fabfed/provider/fabric/fabric_network.py +++ b/fabfed/provider/fabric/fabric_network.py @@ -6,7 +6,6 @@ from fabfed.model import Network from fabfed.util.constants import Constants from ...util.config_models import Config -from fabfed.exceptions import FabfedException from .fabric_provider import FabricProvider from fabfed.policy.policy_helper import get_stitch_port_for_provider @@ -31,7 +30,6 @@ def __init__(self, *, label, delegate: NetworkService, layer3: Config, peering: ns = self._delegate.get_fim_network_service() self.interface = [] - # TODO This is only needed for sense-aws and aws if self.peering and Constants.RES_CLOUD_ACCOUNT in self.peering.attributes: account_id = self.peering.attributes[Constants.RES_CLOUD_ACCOUNT] @@ -51,6 +49,11 @@ def subnet(self): @property def gateway(self): + fabric_gateway_ip = self.delegate.get_gateway() + + if fabric_gateway_ip is not None: + return fabric_gateway_ip + return self.layer3.attributes.get(Constants.RES_NET_GATEWAY) if self.layer3 else None @property @@ -83,46 +86,64 @@ def get_site(self): class NetworkBuilder: def __init__(self, label, provider: FabricProvider, slice_object: Slice, name, resource: dict): self.slice_object = slice_object - self.stitch_info = resource.get(Constants.RES_STITCH_INFO) + self.stitch_infos = resource.get(Constants.RES_STITCH_INFO) self.stitch_port = get_stitch_port_for_provider(resource=resource, provider=provider.type) self.vlan = None self.site = resource.get(Constants.RES_SITE) - self.interfaces = [] + self.facility_port_interfaces = [] self.net_name = name self.layer3 = resource.get(Constants.RES_LAYER3) self.peering = resource.get(Constants.RES_PEERING) self.peer_layer3 = resource.get(Constants.RES_PEER_LAYER3) - self.stitching_net = None + self.stitching_nets = [] self.label = label self.net = None - self.type = resource.get('net_type') # TODO: type - self.discovered_stitch_info = {} + self.type = resource.get(Constants.RES_FLAVOR) + self.discovered_stitch_infos = [] self.device = resource.get(Constants.STITCH_PORT_DEVICE_NAME) self.site = resource.get(Constants.STITCH_PORT_SITE) + self.sites = set() interface = resource.get(Constants.RES_INTERFACES) if isinstance(interface, list): interface = interface[0] - + if isinstance(interface, dict) and 'vlan' in interface: logger.info(f'Network {self.net_name} found interface {interface}') self.vlan = interface.get('vlan') - if self.stitch_port: + if isinstance(self.stitch_port, dict): self.device = self.stitch_port.get(Constants.STITCH_PORT_DEVICE_NAME) self.site = self.stitch_port.get(Constants.STITCH_PORT_SITE) import fabfed.provider.api.dependency_util as util if util.has_resolved_external_dependencies(resource=resource, attribute=Constants.RES_STITCH_INTERFACE): - self.stitching_net = util.get_single_value_for_dependency(resource=resource, - attribute=Constants.RES_STITCH_INTERFACE) + self.stitching_nets = util.get_values_for_dependency(resource=resource, + attribute=Constants.RES_STITCH_INTERFACE) + + for stitching_net in self.stitching_nets: + interface = stitching_net.interface - if not isinstance(self.stitching_net, Network): - raise FabfedException(f"Expecting Network. Got {type(self.stitching_net)}") + if isinstance(interface, list): + temp = [i for i in interface if isinstance(i, dict) and 'provider' in i] - self.check_stitch_net() + if temp: + interface = temp[0] + + if isinstance(interface, dict) and 'provider' in interface: + logger.info( + f'Network {self.net_name} found stitching interface {interface} from {stitching_net.label}') + + discovered_stitch_info = dict() + + discovered_stitch_info.update(interface) + self.discovered_stitch_infos.append(discovered_stitch_info) + + for discovered_stitch_info in self.discovered_stitch_infos: + logger.info( + f'{self.net_name} will use stitch info: {discovered_stitch_info}') if self.peering: from .plugins import Plugins @@ -135,29 +156,10 @@ def __init__(self, label, provider: FabricProvider, slice_object: Slice, name, r logger.info( f'{self.net_name}:vlan={self.vlan},stitch_port={self.stitch_port},device={self.device},site={self.site}') - def check_stitch_net(self): - assert self.stitching_net - - if hasattr(self.stitching_net, 'interface'): - interface = self.stitching_net.interface - - if isinstance(interface, list): - temp = [i for i in interface if isinstance(i, dict) and 'provider' in i] - - if temp: - interface = temp[0] - - if isinstance(interface, dict) and 'provider' in interface: - logger.info(f'Network {self.net_name} found stitching interface {interface}') - self.vlan = interface.get('vlan') - self.discovered_stitch_info = interface - - def handle_facility_port(self): + def handle_facility_port(self, *, sites): from fim.slivers.capacities_labels import Labels, Capacities - if not self.vlan and not self.peering: - logger.warning(f"Network {self.net_name} has no vlan and no peering so no facility port will be added ") - return + self.sites.update(sites) if self.peering: cloud = self.peering.attributes.get(Constants.RES_CLOUD_FACILITY) @@ -165,11 +167,11 @@ def handle_facility_port(self): account_id = self.peering.attributes.get(Constants.RES_CLOUD_ACCOUNT) if account_id is None: - account_id = self.discovered_stitch_info.get("id") + account_id = self.discovered_stitch_infos[0].get("id") # GCP subnet = self.peering.attributes.get(Constants.RES_LOCAL_ADDRESS) peer_subnet = self.peering.attributes.get(Constants.RES_REMOTE_ADDRESS) - region= self.peering.attributes.get(Constants.RES_CLOUD_REGION) + region = self.peering.attributes.get(Constants.RES_CLOUD_REGION) device = self.peering.attributes.get(Constants.RES_LOCAL_DEVICE) port = self.peering.attributes.get(Constants.RES_LOCAL_PORT) vlan = self.peering.attributes.get('cloud_vlan') @@ -186,10 +188,10 @@ def handle_facility_port(self): if not cloud: cloud = self.stitch_port.get(Constants.STITCH_PORT_SITE) - self.peering.attributes[Constants.RES_CLOUD_FACILITY] = cloud # TODO WORKAROUND FOR NOW + self.peering.attributes[Constants.RES_CLOUD_FACILITY] = cloud # TODO WORKAROUND FOR NOW if not vlan: - vlan = self.stitch_port.get('vlan') # TODO WORKAROUND GCP NEEDS THIS + vlan = self.stitch_port.get('vlan') labels = Labels(ipv4_subnet=subnet) @@ -234,24 +236,42 @@ def handle_facility_port(self): peer_labels=peer_labels, capacities=Capacities(bw=int(bw), mtu=mtu_size)) + facility_port_interface = facility_port.get_interfaces()[0] + self.facility_port_interfaces.append(facility_port_interface) logger.info("CreatedFacilityPort:" + facility_port.toJson()) - else: - logger.info(f"Creating Facility Port: name={self.device}:site={self.site}:vlan={self.vlan}") - + elif self.discovered_stitch_infos: + for discovered_stitch_info in self.discovered_stitch_infos: + logger.info( + f'{self.net_name} will use stitch info: {discovered_stitch_info}') + + device = discovered_stitch_info[Constants.STITCH_PORT_DEVICE_NAME] + site = discovered_stitch_info[Constants.STITCH_PORT_SITE] + vlan = discovered_stitch_info[Constants.STITCH_PORT_VLAN] + self.sites.add(site) + + logger.info(f"Adding Facility Port to slice: name={device}:site={site}:vlan={vlan}") + + facility_port = self.slice_object.add_facility_port(name=device, + site=site, + vlan=str(vlan)) + logger.info(f"Added Facility Port to slice: name={device}:site={site}:vlan={vlan}") + facility_port_interface = facility_port.get_interfaces()[0] + self.facility_port_interfaces.append(facility_port_interface) + elif self.device and self.vlan and self.site: # USED FOR TESTING + logger.info(f"Adding Facility Port to slice: name={self.device}:site={self.site}:vlan={self.vlan}") facility_port = self.slice_object.add_facility_port(name=self.device, site=self.site, vlan=str(self.vlan)) - logger.info(f"Created Facility Port: name={self.device}:site={self.site}:vlan={self.vlan}") - - facility_port_interface = facility_port.get_interfaces()[0] - self.interfaces.append(facility_port_interface) + logger.info(f"Added Facility Port to slice: name={self.device}:site={self.site}:vlan={self.vlan}") + facility_port_interface = facility_port.get_interfaces()[0] + self.facility_port_interfaces.append(facility_port_interface) def handle_network(self, nodes): from fim.slivers.capacities_labels import Labels, Capacities if not self.peering: interfaces = [] - interfaces.extend(self.interfaces) + interfaces.extend(self.facility_port_interfaces) for node in nodes: node_interfaces = [i for i in node.get_interfaces() if not i.get_network()] @@ -262,19 +282,31 @@ def handle_network(self, nodes): else: logger.warning(f"Node {node.name} has no available interface to stitch to network {self.net_name} ") - # Use type='L2STS'? - logger.info(f"Creating Layer2 Network:{self.net_name}:interfaces={[i.get_name() for i in interfaces]}") - self.net: NetworkService = self.slice_object.add_l2network(name=self.net_name, interfaces=interfaces) + if self.type is not None: + net_type = self.type + elif len(self.sites) == 2: + net_type = 'L2STS' + else: + net_type = 'L2Bridge' + logger.info( + f"Adding Network:{self.net_name} to slice:ifaces={[i.get_name() for i in interfaces]}:type={net_type}") + + if net_type == "L3VPN": + self.net: NetworkService = self.slice_object.add_l3network(name=self.net_name, + interfaces=interfaces) + else: + self.net: NetworkService = self.slice_object.add_l2network(name=self.net_name, + interfaces=interfaces) return tech = 'AL2S' net_type = 'L3VPN' - port_name= self.interfaces[0].get_name() + port_name = self.facility_port_interfaces[0].get_name() logger.info(f"Creating Network:{self.net_name}:FacilityPort:{port_name},type={net_type}:techonolgy={tech}") self.net = self.slice_object.add_l3network(name=self.net_name, - interfaces=self.interfaces, type=net_type, + interfaces=self.facility_port_interfaces, type=net_type, technology=tech) interfaces = [] @@ -287,7 +319,6 @@ def handle_network(self, nodes): else: logger.warning(f"Node {node.name} has no available interface to stitch to network {self.net_name} ") - # TODO DO WE NEED REALLY THIS? if not interfaces: return diff --git a/fabfed/provider/fabric/fabric_provider.py b/fabfed/provider/fabric/fabric_provider.py index 8f76660f..57dbcba1 100644 --- a/fabfed/provider/fabric/fabric_provider.py +++ b/fabfed/provider/fabric/fabric_provider.py @@ -15,7 +15,6 @@ def __init__(self, *, type, label, name, config: dict): self.slice = None self.retry = 5 self.slice_init = False - # TODO Should not be needed for fablib 1.6.4 from fabrictestbed_extensions.fablib.constants import Constants as FC from fabfed.util.utils import get_base_dir @@ -23,6 +22,8 @@ def __init__(self, *, type, label, name, config: dict): FC.DEFAULT_FABRIC_CONFIG_DIR = get_base_dir(self.name) FC.DEFAULT_FABRIC_RC = f"{FC.DEFAULT_FABRIC_CONFIG_DIR}/fabric_rc" + + def _to_abs_for(self, env_var: str, config: dict): path = config.get(env_var) @@ -83,7 +84,6 @@ def setup_environment(self): fabric_slice_helper.patch_for_token() - def _init_slice(self, destroy_phase=False): if not self.slice_init: self.logger.info(f"Initializing slice {self.name}") @@ -128,8 +128,12 @@ def _init_slice(self, destroy_phase=False): def supports_modify(self): return True - def do_add_resource(self, *, resource: dict): + def do_validate_resource(self, *, resource: dict): self._init_slice() + assert self.slice.slice_object is not None + self.slice.validate_resource(resource=resource) + + def do_add_resource(self, *, resource: dict): assert self.slice.slice_object is not None self.slice.add_resource(resource=resource) @@ -137,6 +141,10 @@ def do_create_resource(self, *, resource: dict): assert self.slice.slice_object is not None self.slice.create_resource(resource=resource) + def do_wait_for_create_resource(self, *, resource: dict): + assert self.slice.slice_object is not None + self.slice.wait_for_create_resource(resource=resource) + def do_delete_resource(self, *, resource: dict): self._init_slice(True) self.slice.delete_resource(resource=resource) diff --git a/fabfed/provider/fabric/fabric_slice.py b/fabfed/provider/fabric/fabric_slice.py index fa5d1094..17a715ed 100644 --- a/fabfed/provider/fabric/fabric_slice.py +++ b/fabfed/provider/fabric/fabric_slice.py @@ -18,6 +18,8 @@ def __init__(self, *, provider: FabricProvider, logger: logging.Logger): self.notified_create = False self.slice_created = False self.slice_modified = False + self.submitted = False + self.network_to_sites_mapping = dict() from fabrictestbed_extensions.fablib.slice import Slice @@ -72,6 +74,36 @@ def networks(self) -> List[Network]: def pending(self): return self.provider.pending + def validate_resource(self, *, resource: dict): + rtype = resource.get(Constants.RES_TYPE) + + if rtype == Constants.RES_TYPE_NETWORK.lower(): + label = resource[Constants.LABEL] + if label not in self.network_to_sites_mapping: + self.network_to_sites_mapping[label] = set() + + if Constants.RES_SITE in resource: + self.network_to_sites_mapping[label].add(resource[Constants.RES_SITE]) + + dependencies = resource[Constants.INTERNAL_DEPENDENCIES] + + for dependency in dependencies: + if Constants.RES_SITE in dependency.resource.attributes: + self.network_to_sites_mapping[label].add(dependency.resource.attributes[Constants.RES_SITE]) + + elif rtype == Constants.RES_TYPE_NODE.lower(): + dependencies = resource[Constants.INTERNAL_DEPENDENCIES] + + for dependency in dependencies: + label = dependency.resource.label + + if label not in self.network_to_sites_mapping: + self.network_to_sites_mapping[label] = set() + + self.network_to_sites_mapping[label].add(resource[Constants.RES_SITE]) + else: + raise Exception("Unknown resource ....") + def _add_network(self, resource: dict): label = resource[Constants.LABEL] net_name = self.provider.resource_name(resource) @@ -96,7 +128,7 @@ def _add_network(self, resource: dict): return network_builder = NetworkBuilder(label, self.provider, self.slice_object, net_name, resource) - network_builder.handle_facility_port() + network_builder.handle_facility_port(sites=self.network_to_sites_mapping[label]) temp = [] if util.has_resolved_internal_dependencies(resource=resource, attribute='interface'): temp = util.get_values_for_dependency(resource=resource, attribute='interface') @@ -270,42 +302,6 @@ def add_resource(self, *, resource: dict): else: raise Exception("Unknown resource ....") - def _submit_and_wait(self) -> str or None: - self.logger.info(f"Submitting request for slice {self.name}") - slice_id = self.slice_object.submit(wait=False) - self.logger.info(f"Waiting for slice {self.name} to be stable") - - # TODO Timeout exceeded(360 sec).Slice: aes - chi - tacc - seq - 3(Configuring) - - try: - self.slice_object.wait(timeout=24 * 60, progress=True) - except Exception as e: - state = self.slice_object.get_state() - self.logger.warning(f"Exception occurred while waiting state={state}:{e}") - raise e - - try: - self.slice_object.wait_ssh() - except Exception as e: - self.logger.warning(f"Exception occurred while waiting on ssh: {e}") - - try: - self.slice_object.post_boot_config() - except Exception as e: - self.logger.warning(f"Exception occurred while update/post_boot_config: {e}") - - self.logger.info(f"Slice provisioning successful {self.slice_object.get_state()}") - - # days = DEFAULT_RENEWAL_IN_DAYS - # try: - # import datetime - # end_date = (datetime.datetime.now() + datetime.timedelta(days=days)).strftime("%Y-%m-%d %H:%M:%S %z") - # self.slice_object.renew(end_date) - # except Exception as e: - # self.logger.warning(f"Exception occurred while renewing for {days}: {e}") - - return slice_id - def _reload_nodes(self): from fabrictestbed_extensions.fablib.fablib import fablib @@ -455,6 +451,9 @@ def create_resource(self, *, resource: dict): self.logger.warning(f"still have pending {len(self.pending)} resources") return + if self.submitted: + return + if self.slice_created: aset = set(self.existing_nodes) bset = {n.name for n in self.nodes} @@ -522,30 +521,61 @@ def create_resource(self, *, resource: dict): self.slice_modified = True if self.slice_created and not self.slice_modified: - state = self.slice_object.get_state() + return - if not self.notified_create: - self.logger.info(f"already provisioned. {self.name}: state={state}") - else: - self.logger.debug(f"already provisioned. {self.name}: state={state}") + self.logger.info(f"Submitting request for slice {self.name}") + slice_id = self.slice_object.submit(wait=False) + self.logger.info(f"Done Submitting request for slice {self.name}:{slice_id}") + self.submitted = True + + def wait_for_create_resource(self, *, resource: dict): + if self.slice_created and not self.slice_modified and not self.notified_create: + self._handle_node_networking() - if not self.notified_create: - self._handle_node_networking() + for node in self.nodes: + self.resource_listener.on_created(source=self, provider=self.provider, resource=node) - for node in self.nodes: - self.resource_listener.on_created(source=self, provider=self.provider, resource=node) + for net in self.networks: + self.resource_listener.on_created(source=self, provider=self.provider, resource=net) - for net in self.networks: - self.resource_listener.on_created(source=self, provider=self.provider, resource=net) + self.notified_create = True + return - self.notified_create = True + if not self.submitted: return - self._submit_and_wait() + self.logger.info(f"Waiting for slice {self.name} to be stable") + + try: + self.slice_object.wait(timeout=24 * 60, progress=True) + except Exception as e: + state = self.slice_object.get_state() + self.logger.warning(f"Exception occurred while waiting state={state}:{e}") + raise e + + try: + self.slice_object.wait_ssh() + except Exception as e: + self.logger.warning(f"Exception occurred while waiting on ssh: {e}") + + try: + self.slice_object.post_boot_config() + except Exception as e: + self.logger.warning(f"Exception occurred while update/post_boot_config: {e}") + + self.logger.info(f"Slice provisioning successful {self.slice_object.get_state()}") + + # days = DEFAULT_RENEWAL_IN_DAYS + # try: + # import datetime + # end_date = (datetime.datetime.now() + datetime.timedelta(days=days)).strftime("%Y-%m-%d %H:%M:%S %z") + # self.slice_object.renew(end_date) + # except Exception as e: + # self.logger.warning(f"Exception occurred while renewing for {days}: {e}") + self.slice_created = True self.slice_modified = False self.existing_nodes = [n.name for n in self.nodes] - # self.existing_networks = [n.get_name() for n in self.networks] self.existing_networks = [] for net in self.slice_object.get_networks(): diff --git a/fabfed/util/constants.py b/fabfed/util/constants.py index 664dc4f4..436c28ba 100644 --- a/fabfed/util/constants.py +++ b/fabfed/util/constants.py @@ -68,6 +68,7 @@ class Constants: STITCH_PORT_REGION = 'region' STITCH_PORT_SITE = 'site' STITCH_VLAN_RANGE = 'vlan_range' + STITCH_PORT_VLAN = 'vlan' RES_STITCH_INFO = "stitch_info" RES_STITCH_INTERFACE = "stitch_interface" diff --git a/tools/fabfed.py b/tools/fabfed.py index b915a739..8064ff9d 100644 --- a/tools/fabfed.py +++ b/tools/fabfed.py @@ -185,11 +185,15 @@ def manage_workflow(args): stitch_info_network_info_map = {} for network in filter(lambda n: n.is_network and n.attributes.get(Constants.RES_STITCH_INFO), resources): - stitch_info = network.attributes.get(Constants.RES_STITCH_INFO) + stitch_infos = network.attributes.get(Constants.RES_STITCH_INFO) - if stitch_info: + for stitch_info in stitch_infos: network_info = NetworkInfo(label=network.label, provider_label=network.provider.label) - stitch_port_name = stitch_info.stitch_port['name'] + if 'name' in stitch_info.stitch_port: + stitch_port_name = stitch_info.stitch_port['name'] + else: + stitch_port_name = stitch_info.stitch_port['peer']['name'] + stitch_info_map[stitch_port_name] = stitch_info if stitch_port_name not in stitch_info_network_info_map: