Feat(cv_deploy): Expose list of targeted inactive devices if Workspace submission failed due to ResponseCode.INACTIVE_DEVICES_EXIST

ansible_collections/arista/avd/molecule/cv_deploy/workspace_force_false.yml

@@ -45,12 +45,18 @@
             cv_devices: [ avd-ci-leaf1, avd-ci-core1 ]
             cv_submit_workspace_force: false
       rescue:
+
+        - name: Display CVP result
+          run_once: true
+          ansible.builtin.debug:
+            msg: '{{ cv_deploy_results }}'
+
         - name: Check CVP returns
           run_once: true
           ansible.builtin.assert:
             that:
               # errors and warnings
-              - "'Failed to submit workspace' in cv_deploy_results.errors[0]"
+              - "'Failed to submit CloudVision Workspace due to the presence of inactive devices. Use force to override' in cv_deploy_results.errors[0]"
 
     - name: Cleanup orphan workspace
       run_once: true

ansible_collections/arista/avd/molecule/cv_deploy/workspace_force_true.yml

@@ -54,6 +54,7 @@
         that:
           # errors and warnings
           - cv_deploy_results.errors == []
+          - "'Inactive devices present' in cv_deploy_results.warnings[0]"
 
     - name: Cleanup orphan CC
       run_once: true

python-avd/pyavd/_cv/client/exceptions.py

@@ -66,7 +66,11 @@ class CVWorkspaceBuildFailed(CVClientException):
 
 
 class CVWorkspaceSubmitFailed(CVClientException):
-    """Build of CloudVision Workspace failed."""
+    """Failed to submit CloudVision Workspace."""
+
+
+class CVWorkspaceSubmitFailedInactiveDevices(CVClientException):
+    """Failed to submit CloudVision Workspace due to the presence of inactive devices. Use force to override."""
 
 
 class CVWorkspaceStateTimeout(CVClientException):
@@ -84,3 +88,7 @@ class CVMessageSizeExceeded(CVClientException):
     """Maximum GRPC message size"""
     size: int
     """Actual GRPC message size"""
+
+
+class CVInactiveDevices(CVClientException):
+    """Inactive devices present."""

python-avd/pyavd/_cv/workflows/deploy_to_cv.py

@@ -131,17 +131,20 @@ async def deploy_to_cv(
             # Create workspace
             await create_workspace_on_cv(workspace=result.workspace, cv_client=cv_client)
 
+            # Form the list of targeted CVDevices (list may contain duplicated items)
+            devices = (
+                [tag.device for tag in device_tags if tag.device is not None]
+                + [tag.device for tag in interface_tags if tag.device is not None]
+                + [config.device for config in configs if config.device is not None]
+            )
+
             try:
                 # Verify devices exist and update CVDevice objects with _exists_on_cv.
                 # Depending on skip_missing_devices we will raise or skip missing devices.
                 # Since verify_devices will silently return if _exists_on_cv is already set,
                 # we can just send all the items even if we have duplicate device objects.
-                await verify_devices_on_cv(
-                    devices=(
-                        [tag.device for tag in device_tags if tag.device is not None]
-                        + [tag.device for tag in interface_tags if tag.device is not None]
-                        + [config.device for config in configs if config.device is not None]
-                    ),
+                existing_deduplicated_devices = await verify_devices_on_cv(
+                    devices=devices,
                     workspace_id=result.workspace.id,
                     skip_missing_devices=skip_missing_devices,
                     warnings=result.warnings,
@@ -201,7 +204,7 @@ async def deploy_to_cv(
                 result.workspace.state = "abandoned"
                 return result
 
-            await finalize_workspace_on_cv(workspace=result.workspace, cv_client=cv_client)
+            await finalize_workspace_on_cv(workspace=result.workspace, cv_client=cv_client, devices=existing_deduplicated_devices, warnings=result.warnings)
 
             # Create/update CVChangeControl object with ID created by workspace.
             if result.workspace.change_control_id is not None:

python-avd/pyavd/_cv/workflows/finalize_workspace_on_cv.py

@@ -6,13 +6,13 @@
 from logging import getLogger
 from typing import TYPE_CHECKING
 
-from pyavd._cv.api.arista.workspace.v1 import ResponseStatus, WorkspaceState
-from pyavd._cv.client.exceptions import CVWorkspaceBuildFailed, CVWorkspaceSubmitFailed
+from pyavd._cv.api.arista.workspace.v1 import ResponseCode, ResponseStatus, WorkspaceState
+from pyavd._cv.client.exceptions import CVInactiveDevices, CVWorkspaceBuildFailed, CVWorkspaceSubmitFailed, CVWorkspaceSubmitFailedInactiveDevices
 
 if TYPE_CHECKING:
     from pyavd._cv.client import CVClient
 
-    from .models import CVWorkspace
+    from .models import CVDevice, CVWorkspace
 
 LOGGER = getLogger(__name__)
 
@@ -26,7 +26,7 @@
 }
 
 
-async def finalize_workspace_on_cv(workspace: CVWorkspace, cv_client: CVClient) -> None:
+async def finalize_workspace_on_cv(workspace: CVWorkspace, cv_client: CVClient, devices: list[CVDevice], warnings: list) -> None:
     """
     Finalize a Workspace from the given result.CVWorkspace object.
 
@@ -61,11 +61,43 @@ async def finalize_workspace_on_cv(workspace: CVWorkspace, cv_client: CVClient)
             workspace_id=workspace.id,
             request_id=workspace_config.request_params.request_id,
         )
+        # Form a list of known inactive existing devices
+        inactive_devices = [device for device in devices if not device._streaming]
         if submit_result.status != ResponseStatus.SUCCESS:
             workspace.state = "submit failed"
+            # Unforced Workspace submission failed due to inactive devices.
+            if submit_result.code == ResponseCode.INACTIVE_DEVICES_EXIST and not workspace.force:
+                # Usecase where some of the devices that we targeted were known to be inactive prior to Workspace submission
+                if inactive_devices:
+                    msg = (
+                        f"Failed to submit CloudVision Workspace due to the presence of inactive devices. "
+                        f"Use force to override. Inactive devices: {inactive_devices}."
+                    )
+                    LOGGER.warning(msg)
+                    raise CVWorkspaceSubmitFailedInactiveDevices(msg)
+                # Usecase where all devices were actively streaming prior to Workspace submission
+                msg = (
+                    "Failed to submit CloudVision Workspace due to the presence of inactive devices. "
+                    "Use force to override. Exact list of inactive devices is unknown."
+                )
+                LOGGER.warning(msg)
+                raise CVWorkspaceSubmitFailedInactiveDevices(msg)
+
+            # If Workspace submission failed for any other reason and known inactive devices were present - append information to warnings.
+            if inactive_devices:
+                msg = f"Inactive devices present: {inactive_devices}"
+                LOGGER.warning(msg)
+                warnings.append(CVInactiveDevices(msg))
+
+            # If Workspace submission failed for any other reason - raise general exception.
             LOGGER.info("finalize_workspace_on_cv: %s", workspace)
             msg = f"Failed to submit workspace {workspace.id}: {submit_result}"
             raise CVWorkspaceSubmitFailed(msg)
+        # If successful Workspace submission with inactive devices was forced - append information to warnings.
+        if inactive_devices and workspace.force:
+            msg = f"Inactive devices present: {inactive_devices}"
+            LOGGER.warning(msg)
+            warnings.append(CVInactiveDevices(msg))
 
         workspace.state = "submitted"
         if cv_workspace.cc_ids.values:

python-avd/pyavd/_cv/workflows/models.py

@@ -141,6 +141,11 @@ class CVDevice:
     system_mac_address: str | None = None
     _exists_on_cv: bool | None = None
     """ Do not set this manually. """
+    _streaming: bool | None = None
+    """
+    Device's streaming status.
+    Do not set this manually.
+    """
 
 
 @dataclass

python-avd/pyavd/_cv/workflows/verify_devices_on_cv.py

@@ -6,31 +6,56 @@
 from logging import getLogger
 from typing import TYPE_CHECKING
 
+from pyavd._cv.api.arista.inventory.v1 import StreamingStatus
 from pyavd._cv.client.exceptions import CVResourceNotFound
 
 from .models import CVDevice
 
 if TYPE_CHECKING:
     from pyavd._cv.client import CVClient
 
+
 LOGGER = getLogger(__name__)
 
 
 async def verify_devices_on_cv(
-    *, devices: list[CVDevice], workspace_id: str, skip_missing_devices: bool, warnings: list[Exception], cv_client: CVClient
-) -> None:
-    """Verify that the given Devices are already present in the CloudVision Inventory & I&T Studio."""
+    *,
+    devices: list[CVDevice],
+    workspace_id: str,
+    skip_missing_devices: bool,
+    warnings: list[Exception],
+    cv_client: CVClient,
+) -> list[CVDevice]:
+    """
+    Verify that the given Devices are already present in the CloudVision Inventory & I&T Studio.
+
+    Returns a list of deduplicated CVDevice objects found on CloudVision.
+    """
     LOGGER.info("verify_devices_on_cv: %s", len(devices))
 
     # Return if we have nothing to do.
     if not devices:
-        return
+        return []
 
     existing_devices = await verify_devices_in_cloudvision_inventory(
-        devices=devices, skip_missing_devices=skip_missing_devices, warnings=warnings, cv_client=cv_client
+        devices=devices,
+        skip_missing_devices=skip_missing_devices,
+        warnings=warnings,
+        cv_client=cv_client,
     )
     await verify_devices_in_topology_studio(existing_devices, workspace_id, cv_client)
-    return
+
+    # Form deduplicated list of CVDevices found to exist on CV
+    unique_existing_device_tuples: list[tuple[str | None, str | None, str | None]] = []
+    existing_deduplicated_devices: list[CVDevice] = []
+    for existing_device in existing_devices:
+        if (
+            unique_existing_device_tuple := (existing_device.serial_number, existing_device.system_mac_address, existing_device.hostname)
+        ) not in unique_existing_device_tuples:
+            unique_existing_device_tuples.append(unique_existing_device_tuple)
+            existing_deduplicated_devices.append(existing_device)
+
+    return existing_deduplicated_devices
 
 
 async def verify_devices_in_cloudvision_inventory(
@@ -51,6 +76,8 @@ async def verify_devices_in_cloudvision_inventory(
 
     Skip checks for devices where _exists_on_cv is already filled out on the device.
 
+    Populate current streaming status for all existing devices.
+
     Returns a list of CVDevice objects found to exist on CloudVision.
     """
     # Using set to only include a device once.
@@ -80,6 +107,8 @@ async def verify_devices_in_cloudvision_inventory(
                 continue
             device._exists_on_cv = True
             device.system_mac_address = found_device_dict_by_serial[device.serial_number].system_mac_address
+            # Update streaming status
+            device._streaming = found_device_dict_by_serial[device.serial_number].streaming_status == StreamingStatus.ACTIVE
             continue
 
         # Use system_mac_address as unique ID if set.
@@ -89,6 +118,8 @@ async def verify_devices_in_cloudvision_inventory(
                 continue
             device._exists_on_cv = True
             device.serial_number = found_device_dict_by_system_mac[device.system_mac_address].key.device_id
+            # Update streaming status
+            device._streaming = found_device_dict_by_system_mac[device.system_mac_address].streaming_status == StreamingStatus.ACTIVE
             continue
 
         # Finally use hostname as unique ID.
@@ -98,6 +129,8 @@ async def verify_devices_in_cloudvision_inventory(
         device._exists_on_cv = True
         device.serial_number = found_device_dict_by_hostname[device.hostname].key.device_id
         device.system_mac_address = found_device_dict_by_hostname[device.hostname].system_mac_address
+        # Update streaming status
+        device._streaming = found_device_dict_by_hostname[device.hostname].streaming_status == StreamingStatus.ACTIVE
 
     # Now we know which devices are on CV, so we can dig deeper and check for them in I&T Studio
     # If a device is found, we will ensure hostname is correct and if not, update the hostname.