Skip to content

Commit

Permalink
[TEST]: Improvement: Switches: New interaction approach: Phase2
Browse files Browse the repository at this point in the history
  • Loading branch information
Yuliia Miroshnychenko committed Jan 17, 2025
1 parent e0df6d0 commit cc3e394
Show file tree
Hide file tree
Showing 9 changed files with 251 additions and 203 deletions.
41 changes: 37 additions & 4 deletions ...l-tests/src/main/groovy/org/openkilda/functionaltests/helpers/model/SwitchExtended.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ import org.openkilda.northbound.dto.v1.switches.SwitchPropertiesDto
import org.openkilda.northbound.dto.v1.switches.SwitchSyncResult
import org.openkilda.northbound.dto.v2.switches.LagPortResponse
import org.openkilda.northbound.dto.v2.switches.MeterInfoDtoV2
import org.openkilda.northbound.dto.v2.switches.SwitchConnectEntry
import org.openkilda.northbound.dto.v2.switches.SwitchConnectedDevicesResponse
import org.openkilda.northbound.dto.v2.switches.SwitchFlowsPerPortResponse
import org.openkilda.northbound.dto.v2.switches.SwitchLocationDtoV2
import org.openkilda.northbound.dto.v2.switches.SwitchPatchDto
Expand All @@ -82,6 +84,7 @@ import org.openkilda.testing.service.floodlight.model.Floodlight
import org.openkilda.testing.service.floodlight.model.FloodlightConnectMode
import org.openkilda.testing.service.lockkeeper.LockKeeperService
import org.openkilda.testing.service.lockkeeper.model.FloodlightResourceAddress
import org.openkilda.testing.service.lockkeeper.model.TrafficControlData
import org.openkilda.testing.service.northbound.NorthboundService
import org.openkilda.testing.service.northbound.NorthboundServiceV2
import org.openkilda.testing.service.northbound.payloads.SwitchValidationExtendedResult
Expand Down Expand Up @@ -151,8 +154,8 @@ class SwitchExtended {

@Override
String toString() {
return String.format("Switch: %s, islPorts: %s, traffGen(s) port(s) %s, nbDetails: %s",
switchId, islPorts, traffGenPorts, nbDetails)
return String.format("Switch: %s, islPorts: %s, traffGen(s) port(s) %s, nbDetails: %s, FL regions: %s",
switchId, islPorts, traffGenPorts, nbDetails, regions)
}

@JsonIgnore
Expand All @@ -175,6 +178,10 @@ class SwitchExtended {
sw.ofVersion
}

List<String> getRegions(){
sw.regions
}

/**
*
* Get list of switch ports excluding the ports which are busy with ISLs or s42.
Expand Down Expand Up @@ -512,12 +519,24 @@ class SwitchExtended {
return northbound.deleteSwitch(sw.dpId, force)
}

SwitchConnectedDevicesResponse getConnectedDevices() {
northboundV2.getConnectedDevices(switchId)
}

boolean isS42FlowRttEnabled() {
def swProps = northbound.getSwitchProperties(sw.dpId)
def featureToggles = northbound.getFeatureToggles()
swProps.server42FlowRtt && featureToggles.server42FlowRtt
}

/***
* getting switch connections details to floodlight
* @return list of floodlight connections
*/
List<SwitchConnectEntry> getConnectedFloodLights() {
northboundV2.getSwitchConnections(sw.dpId).connections
}

/***
* Floodlight interaction
*/
Expand Down Expand Up @@ -546,9 +565,10 @@ class SwitchExtended {
* @param FL mode
* @param waitForRelatedLinks make sure that all switch related ISLs are FAILED
*/
List<FloodlightResourceAddress> knockout(FloodlightConnectMode mode, boolean waitForRelatedLinks, double timeout = WAIT_OFFSET) {
List<FloodlightResourceAddress> knockout(FloodlightConnectMode mode, boolean waitForRelatedLinks,
boolean waitForRelatedLinksWhenRecover = true, double timeout = WAIT_OFFSET) {
def blockData = lockKeeper.knockoutSwitch(sw, mode)
cleanupManager.addAction(REVIVE_SWITCH, { revive(blockData, true) }, CleanupAfter.TEST)
cleanupManager.addAction(REVIVE_SWITCH, { revive(blockData, waitForRelatedLinksWhenRecover) }, CleanupAfter.TEST)
Wrappers.wait(timeout) {
assert northbound.getSwitch(sw.dpId).state == DEACTIVATED
}
Expand All @@ -565,6 +585,10 @@ class SwitchExtended {
knockout(mode, false)
}

List<FloodlightResourceAddress> knockoutWithoutLinksCheckWhenRecover(FloodlightConnectMode mode) {
knockout(mode, false, false)
}

List<FloodlightResourceAddress> knockout(List<String> regions) {
def blockData = lockKeeper.knockoutSwitch(sw, regions)
cleanupManager.addAction(REVIVE_SWITCH, { revive(blockData, true) }, CleanupAfter.TEST)
Expand Down Expand Up @@ -609,6 +633,15 @@ class SwitchExtended {
relatedLinks.each { isl -> assert isl.state == expectedState }
}

void shapeTraffic(TrafficControlData tcData) {
cleanupManager.addAction(OTHER, { cleanupTrafficShaperRules() })
lockKeeper.shapeSwitchesTraffic([sw], tcData)
}

void cleanupTrafficShaperRules() {
lockKeeper.cleanupTrafficShaperRules(sw.regions.flatten())
}

void waitForS42FlowRttRulesSetup(boolean isS42ToggleOn = true) {
SwitchPropertiesDto switchDetails = getProps()
Wrappers.wait(RULES_INSTALLATION_TIME) {
Expand Down
25 changes: 25 additions & 0 deletions ...ctional-tests/src/main/groovy/org/openkilda/functionaltests/helpers/model/Switches.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ import static org.openkilda.functionaltests.model.switches.Manufacturer.CENTEC
import static org.openkilda.functionaltests.model.switches.Manufacturer.NOVIFLOW
import static org.openkilda.functionaltests.model.switches.Manufacturer.OVS
import static org.openkilda.functionaltests.model.switches.Manufacturer.WB5164
import static org.openkilda.testing.service.floodlight.model.FloodlightConnectMode.RW
import static org.springframework.beans.factory.config.ConfigurableBeanFactory.SCOPE_PROTOTYPE

import org.openkilda.functionaltests.helpers.factory.SwitchFactory
import org.openkilda.functionaltests.model.switches.Manufacturer
import org.openkilda.model.SwitchId
import org.openkilda.northbound.dto.v1.switches.SwitchDto
import org.openkilda.testing.model.topology.TopologyDefinition
import org.openkilda.testing.service.floodlight.FloodlightsHelper
import org.openkilda.testing.service.northbound.NorthboundService
import org.openkilda.testing.service.northbound.NorthboundServiceV2

Expand All @@ -36,6 +38,8 @@ class Switches {
@Qualifier("islandNbV2")
NorthboundServiceV2 northboundV2
@Autowired
FloodlightsHelper flHelper
@Autowired
SwitchFactory switchFactory

List<SwitchExtended> switches
Expand Down Expand Up @@ -77,6 +81,27 @@ class Switches {
return this
}


Switches withConnectedToExactlyNManagementFls(int flAmount) {
switches.findAll { flHelper.filterRegionsByMode(it.regions, RW).size() == flAmount }
return this
}

Switches withConnectedToAtLeastNFls(int flAmount) {
switches.findAll { flHelper.filterRegionsByMode(it.sw.regions, RW).size() >= flAmount }
return this
}

Switches withVxlanEnabled() {
switches.findAll { it.isVxlanEnabled() }
return this
}

Switches withTraffGens() {
switches.findAll { !it.traffGenPorts.isEmpty() }
return this
}

SwitchExtended random() {
assumeFalse(switches.isEmpty(), "No suiting switch found")
switches.shuffled().first()
Expand Down
57 changes: 30 additions & 27 deletions ...s/src/test/groovy/org/openkilda/functionaltests/spec/switches/MultiFloodlightsSpec.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit

import static org.junit.jupiter.api.Assumptions.assumeTrue
import static org.openkilda.functionaltests.extension.tags.Tag.LOCKKEEPER
import static org.openkilda.functionaltests.extension.tags.Tag.SWITCH_RECOVER_ON_FAIL
import static org.openkilda.functionaltests.helpers.Wrappers.wait
Expand All @@ -36,29 +35,30 @@ All switch floodlights can be checked via 'GET /api/v2/switches/{switchId}/conne
class MultiFloodlightsSpec extends HealthCheckSpecification {
@Shared ExecutorService executor = Executors.newFixedThreadPool(2)

@Tags([SWITCH_RECOVER_ON_FAIL])
def "Switch remains online only if at least one of multiple RW floodlights is available"() {
given: "Switch simultaneously connected to 2 management floodlights"
def cleanupActions = []
def sw = topology.switches.find { flHelper.filterRegionsByMode(it.regions, RW).size() == 2 }
assumeTrue(sw.asBoolean(), "Require a switch with 2 active regions")
def sw = switches.all().withConnectedToExactlyNManagementFls(2).first()

and: "Background observer monitoring the state of switch and its ISLs"
def relatedIsls = topology.getRelatedIsls(sw).collectMany { [it, it.reversed] }
def relatedIsls = topology.getRelatedIsls(sw.switchId).collectMany { [it, it.reversed] }

def islObserver = new LoopTask({
def soft = new SoftAssertions()
relatedIsls.each { isl -> soft.checkSucceeds { assert northbound.getLink(isl).state == DISCOVERED } }
soft.verify()
})
def swObserver = new LoopTask({ assert northbound.getSwitch(sw.dpId).state == ACTIVATED })
def swObserver = new LoopTask({ assert sw.getDetails().state == ACTIVATED })
def islObserverFuture = executor.submit(islObserver, "ok")
def swObserverFuture = executor.submit(swObserver, "ok")

when: "Switch loses connection to one of the regions"
def knockout1 = lockKeeper.knockoutSwitch(sw, [flHelper.filterRegionsByMode(sw.regions, RW)[0]])
cleanupActions << { lockKeeper.reviveSwitch(sw, knockout1) }
def knockout1 = sw.knockout([flHelper.filterRegionsByMode(sw.regions, RW)[0]])
cleanupActions << { sw.revive(knockout1) }

then: "Switch can still return its rules"
Wrappers.retry(2, 0.5) { !northbound.getSwitchRules(sw.dpId).flowEntries.empty }
Wrappers.retry(2, 0.5) { !sw.rulesManager.getRules().isEmpty() }

when: "Broken connection gets fixed, but the second floodlight loses connection to kafka"
cleanupActions.pop().call() //lockKeeper.reviveSwitch(sw, knockout1)
Expand All @@ -67,42 +67,42 @@ class MultiFloodlightsSpec extends HealthCheckSpecification {
cleanupActions << { lockKeeper.reviveFloodlight(sw.regions[1]) }

then: "Switch can still return its rules"
Wrappers.retry(2, 0.5) { !northbound.getSwitchRules(sw.dpId).flowEntries.empty }
Wrappers.retry(2, 0.5) { !sw.rulesManager.getRules().isEmpty() }

and: "All this time switch monitor saw switch as Active"
swObserver.stop()
swObserverFuture.get()

when: "Switch loses connection to the last active region"
def knockout2 = lockKeeper.knockoutSwitch(sw, [flHelper.filterRegionsByMode(sw.regions, RW)[0]])
cleanupActions << { lockKeeper.reviveSwitch(sw, knockout2) }
def knockout2 = sw.knockout([flHelper.filterRegionsByMode(sw.regions, RW)[0]])
cleanupActions << { sw.revive(knockout2) }

then: "Switch is marked as inactive"
wait(WAIT_OFFSET) {
northbound.getSwitch(sw.dpId).state == DEACTIVATED
sw.getDetails().state == DEACTIVATED
}

when: "Try getting switch rules"
northbound.getSwitchRules(sw.dpId)
sw.rulesManager.getRules()

then: "Human readable error is returned"
def e = thrown(HttpClientErrorException)
new SwitchNotFoundExpectedError(
"Switch $sw.dpId was not found", ~/The switch was not found when requesting a rules dump./).matches(e)
"Switch $sw.switchId was not found", ~/The switch was not found when requesting a rules dump./).matches(e)

when: "Broken region restores connection to kafka"
cleanupActions.pop().call() //lockKeeper.reviveFloodlight(sw.regions[1])

then: "Switch becomes activated and can respond"
wait(WAIT_OFFSET) { assert northbound.getSwitch(sw.dpId).state == ACTIVATED }
!northbound.getSwitchRules(sw.dpId).flowEntries.empty
wait(WAIT_OFFSET) { assert sw.getDetails().state == ACTIVATED }
!sw.rulesManager.getRules().isEmpty()

when: "Switch restores connection to the other region"
cleanupActions.pop().call() //lockKeeper.reviveSwitch(sw, knockout2)

then: "Switch can still return its rules and remains active"
!northbound.getSwitchRules(sw.dpId).flowEntries.empty
northbound.getSwitch(sw.dpId).state == ACTIVATED
!sw.rulesManager.getRules().isEmpty()
sw.getDetails().state == ACTIVATED

and: "ISL monitor reports that all switch-related ISLs have been Up all the time"
islObserver.stop()
Expand All @@ -117,28 +117,31 @@ class MultiFloodlightsSpec extends HealthCheckSpecification {
@Tags([LOCKKEEPER, SWITCH_RECOVER_ON_FAIL])
def "System supports case when switch uses different networks to connect to FLs, i.e. has diff ips"() {
given: "A switch with at least 2 regions available"
def sw = topology.activeSwitches.find { it.regions.size() >= 2 }
assumeTrue(sw.asBoolean(), "Couldn't find a switch with at least 2 regions available")
def switchToInteract = switches.all().withConnectedToAtLeastNFls(2).first()

and: "The switch's ip in first region is different from other"
def originalSwIp = sw.nbFormat().getAddress()
def region = sw.getRegions().first()
def originalSwIp = switchToInteract.nbFormat().getAddress()
def region = switchToInteract.getRegions().first()
lockKeeper.changeSwIp(region, originalSwIp, DUMMY_SW_IP_1)

and: "The switch is currently disconnected"
def blockData = switchHelper.knockoutSwitch(sw, RW)
def blockData = lockKeeper.knockoutSwitch(switchToInteract.sw, RW)
wait(WAIT_OFFSET) {
assert switchToInteract.getDetails().state == DEACTIVATED
}

when: "Test switch gets connected to both regions"
switchHelper.reviveSwitch(sw, blockData)
switchToInteract.revive(blockData)

then: "Get switch connections API returns different ips for regions"
def connections = northboundV2.getSwitchConnections(sw.dpId)
connections.connections.each {
def connections = switchToInteract.getConnectedFloodLights()
connections.each {
assert it.switchAddress.startsWith(DUMMY_SW_IP_1) == (it.regionName == region)
}

cleanup:
region && lockKeeper.cleanupIpChanges(region)
sw && switchHelper.reviveSwitch(sw, switchHelper.knockoutSwitch(sw, RW))
//this additional knockout is required to revive switch with the correct IP
switchToInteract && switchToInteract.revive(lockKeeper.knockoutSwitch(switchToInteract.sw, RW))
}
}
5 changes: 2 additions & 3 deletions ...-tests/src/test/groovy/org/openkilda/functionaltests/spec/switches/PortHistorySpec.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import static org.openkilda.functionaltests.helpers.model.PortHistoryEvent.ANTI_
import static org.openkilda.functionaltests.helpers.model.PortHistoryEvent.ANTI_FLAP_DEACTIVATED
import static org.openkilda.functionaltests.helpers.model.PortHistoryEvent.ANTI_FLAP_PERIODIC_STATS
import static org.openkilda.functionaltests.helpers.model.PortHistoryEvent.PORT_DOWN
import static org.openkilda.functionaltests.model.cleanup.CleanupActionType.RESTORE_FEATURE_TOGGLE
import static org.openkilda.testing.Constants.NON_EXISTENT_SWITCH_ID
import static org.openkilda.testing.Constants.WAIT_OFFSET
import static org.openkilda.testing.service.floodlight.model.FloodlightConnectMode.RW
Expand Down Expand Up @@ -153,8 +152,8 @@ class PortHistorySpec extends HealthCheckSpecification {
northboundV2.getPortHistory(isl.srcSwitch.dpId, isl.srcPort, timestampBefore, timestampAfter).size() == 4
and: "Deactivate the src switch"
def switchToDisconnect = isl.srcSwitch
switchHelper.knockoutSwitch(switchToDisconnect, RW)
def switchToDisconnect = switches.all().findSpecific(isl.srcSwitch.dpId)
switchToDisconnect.knockout(RW)
then: "Port history on the src switch is still available"
northboundV2.getPortHistory(isl.srcSwitch.dpId, isl.srcPort, timestampBefore, timestampAfter).size() == 4
Expand Down
10 changes: 5 additions & 5 deletions ...sts/src/test/groovy/org/openkilda/functionaltests/spec/switches/PortPropertiesSpec.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,9 @@ class PortPropertiesSpec extends HealthCheckSpecification {
@Tags(ISL_RECOVER_ON_FAIL)
def "System doesn't discover link when port discovery property is disabled"() {
given: "A deleted link"
def sw = topology.activeSwitches.first()
def relatedIsls = topology.getRelatedIsls(sw)
def islToManipulate = relatedIsls.first()
def srcSw = switches.all().first()
def relatedIsls = topology.getRelatedIsls(srcSw.switchId)
def islToManipulate = relatedIsls.find{ it.srcSwitch.dpId == srcSw.switchId }
def isRtl = [islToManipulate.srcSwitch, islToManipulate.dstSwitch]
.any { it.features.contains(SwitchFeature.NOVIFLOW_COPY_FIELD) }

Expand All @@ -134,8 +134,8 @@ class PortPropertiesSpec extends HealthCheckSpecification {
}

when: "Deactivate/activate src switch"
def blockData = switchHelper.knockoutSwitch(sw, RW)
switchHelper.reviveSwitch(sw, blockData)
def blockData = srcSw.knockout(RW)
srcSw.revive(blockData)

then: "Link is still not detected"
Wrappers.timedLoop(discoveryInterval) {
Expand Down
Loading

0 comments on commit cc3e394

Please sign in to comment.