From b5ddf3b27e1af68ec99eeae43aa726613ccb916c Mon Sep 17 00:00:00 2001 From: fanshilun Date: Sun, 9 Feb 2025 20:15:32 +0800 Subject: [PATCH 1/4] YARN-11760. Upgrade JUnit from 4 to 5 in hadoop-yarn-applications-distributedshell. --- .../DistributedShellBaseTest.java | 54 +++--- .../distributedshell/TestClient.java | 4 +- .../distributedshell/TestDSAppMaster.java | 62 +++---- .../distributedshell/TestDSTimelineV10.java | 156 +++++++++--------- .../distributedshell/TestDSTimelineV15.java | 8 +- .../distributedshell/TestDSTimelineV20.java | 46 +++--- .../TestDSWithMultipleNodeManager.java | 40 ++--- 7 files changed, 186 insertions(+), 184 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java index 28cdf8f8223d2..408a14cca4c4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java @@ -34,11 +34,11 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.Rule; import org.junit.rules.TemporaryFolder; import org.junit.rules.TestName; @@ -117,7 +117,7 @@ public abstract class DistributedShellBaseTest { // location of the filesystem timeline writer for timeline service v.2 private String timelineV2StorageDir = null; - @BeforeClass + @BeforeAll public static void setupUnitTests() throws Exception { URL url = Thread.currentThread().getContextClassLoader().getResource( "yarn-site.xml"); @@ -134,7 +134,7 @@ public static void setupUnitTests() throws Exception { StandardCopyOption.REPLACE_EXISTING); } - @AfterClass + @AfterAll public static void tearDownUnitTests() throws Exception { // shutdown the clusters. shutdownYarnCluster(); @@ -215,12 +215,12 @@ public void setTimelineV2StorageDir() throws Exception { timelineV2StorageDir = tmpFolder.newFolder().getAbsolutePath(); } - @Before + @BeforeEach public void setup() throws Exception { setupInternal(NUM_NMS, new YarnConfiguration()); } - @After + @AfterEach public void tearDown() throws IOException { cleanUpDFSClient(); FileContext fsContext = FileContext.getLocalFSFileContext(); @@ -329,7 +329,7 @@ protected void baseTestDSShell(boolean haveDomain, boolean defaultFlow) YarnClient yarnClient; dsClient = setAndGetDSClient(new Configuration(yarnCluster.getConfig())); boolean initSuccess = dsClient.init(args); - Assert.assertTrue(initSuccess); + Assertions.assertTrue(initSuccess); LOG.info("Running DS Client"); final AtomicBoolean result = new AtomicBoolean(false); Thread t = new Thread(() -> { @@ -379,11 +379,11 @@ protected void baseTestDSShell(boolean haveDomain, boolean defaultFlow) t.join(); if (waitResult.get() == 2) { // Exception was raised - Assert.fail("Exception in getting application report. Failed"); + Assertions.fail("Exception in getting application report. Failed"); } if (waitResult.get() == 1) { - Assert.assertEquals("Failed waiting for expected rpc port to be -1.", - -1, appReportRef.get().getRpcPort()); + Assertions.assertEquals( + -1, appReportRef.get().getRpcPort(), "Failed waiting for expected rpc port to be -1."); } checkTimeline(appIdRef.get(), defaultFlow, haveDomain, appReportRef.get()); } @@ -399,22 +399,22 @@ protected void checkTimeline(ApplicationId appId, if (haveDomain) { domain = yarnCluster.getApplicationHistoryServer() .getTimelineStore().getDomain("TEST_DOMAIN"); - Assert.assertNotNull(domain); - Assert.assertEquals("reader_user reader_group", domain.getReaders()); - Assert.assertEquals("writer_user writer_group", domain.getWriters()); + Assertions.assertNotNull(domain); + Assertions.assertEquals("reader_user reader_group", domain.getReaders()); + Assertions.assertEquals("writer_user writer_group", domain.getWriters()); } TimelineEntities entitiesAttempts = yarnCluster .getApplicationHistoryServer() .getTimelineStore() .getEntities(ApplicationMaster.DSEntity.DS_APP_ATTEMPT.toString(), null, null, null, null, null, null, null, null, null); - Assert.assertNotNull(entitiesAttempts); - Assert.assertEquals(1, entitiesAttempts.getEntities().size()); - Assert.assertEquals(2, entitiesAttempts.getEntities().get(0).getEvents() + Assertions.assertNotNull(entitiesAttempts); + Assertions.assertEquals(1, entitiesAttempts.getEntities().size()); + Assertions.assertEquals(2, entitiesAttempts.getEntities().get(0).getEvents() .size()); - Assert.assertEquals(entitiesAttempts.getEntities().get(0).getEntityType(), + Assertions.assertEquals(entitiesAttempts.getEntities().get(0).getEntityType(), ApplicationMaster.DSEntity.DS_APP_ATTEMPT.toString()); - Assert.assertEquals(haveDomain ? domain.getId() : "DEFAULT", + Assertions.assertEquals(haveDomain ? domain.getId() : "DEFAULT", entitiesAttempts.getEntities().get(0).getDomainId()); String currAttemptEntityId = entitiesAttempts.getEntities().get(0).getEntityId(); @@ -428,9 +428,9 @@ protected void checkTimeline(ApplicationId appId, .getTimelineStore() .getEntities(ApplicationMaster.DSEntity.DS_CONTAINER.toString(), null, null, null, null, null, primaryFilter, null, null, null); - Assert.assertNotNull(entities); - Assert.assertEquals(2, entities.getEntities().size()); - Assert.assertEquals(entities.getEntities().get(0).getEntityType(), + Assertions.assertNotNull(entities); + Assertions.assertEquals(2, entities.getEntities().size()); + Assertions.assertEquals(entities.getEntities().get(0).getEntityType(), ApplicationMaster.DSEntity.DS_CONTAINER.toString()); String entityId = entities.getEntities().get(0).getEntityId(); @@ -438,9 +438,9 @@ protected void checkTimeline(ApplicationId appId, yarnCluster.getApplicationHistoryServer().getTimelineStore() .getEntity(entityId, ApplicationMaster.DSEntity.DS_CONTAINER.toString(), null); - Assert.assertNotNull(entity); - Assert.assertEquals(entityId, entity.getEntityId()); - Assert.assertEquals(haveDomain ? domain.getId() : "DEFAULT", + Assertions.assertNotNull(entity); + Assertions.assertEquals(entityId, entity.getEntityId()); + Assertions.assertEquals(haveDomain ? domain.getId() : "DEFAULT", entities.getEntities().get(0).getDomainId()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestClient.java index 074f2e8422bc5..4132675b5d572 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestClient.java @@ -18,12 +18,12 @@ package org.apache.hadoop.yarn.applications.distributedshell; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.HashMap; import java.util.Map; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; public class TestClient { @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java index af095bc918717..8c3b8fc432b0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java @@ -38,8 +38,8 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter; import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; @@ -105,12 +105,12 @@ public void testDSAppMasterAllocateHandler() throws Exception { // first allocate a single container, everything should be fine handler.onContainersAllocated(containers); - Assert.assertEquals("Wrong container allocation count", 1, - master.getAllocatedContainers()); - Assert.assertEquals("Incorrect number of threads launched", 1, - master.threadsLaunched); - Assert.assertEquals("Incorrect YARN Shell IDs", - Arrays.asList("1"), master.yarnShellIds); + Assertions.assertEquals(1 +, master.getAllocatedContainers(), "Wrong container allocation count"); + Assertions.assertEquals(1 +, master.threadsLaunched, "Incorrect number of threads launched"); + Assertions.assertEquals( + Arrays.asList("1"), master.yarnShellIds, "Incorrect YARN Shell IDs"); // now send 3 extra containers containers.clear(); @@ -121,14 +121,14 @@ public void testDSAppMasterAllocateHandler() throws Exception { ContainerId id4 = BuilderUtils.newContainerId(1, 1, 1, 4); containers.add(generateContainer(id4)); handler.onContainersAllocated(containers); - Assert.assertEquals("Wrong final container allocation count", 2, - master.getAllocatedContainers()); + Assertions.assertEquals(2 +, master.getAllocatedContainers(), "Wrong final container allocation count"); - Assert.assertEquals("Incorrect number of threads launched", 2, - master.threadsLaunched); + Assertions.assertEquals(2 +, master.threadsLaunched, "Incorrect number of threads launched"); - Assert.assertEquals("Incorrect YARN Shell IDs", - Arrays.asList("1", "2"), master.yarnShellIds); + Assertions.assertEquals( + Arrays.asList("1", "2"), master.yarnShellIds, "Incorrect YARN Shell IDs"); // make sure we handle completion events correctly List status = new ArrayList<>(); status.add(generateContainerStatus(id1, ContainerExitStatus.SUCCESS)); @@ -137,25 +137,25 @@ public void testDSAppMasterAllocateHandler() throws Exception { status.add(generateContainerStatus(id4, ContainerExitStatus.ABORTED)); handler.onContainersCompleted(status); - Assert.assertEquals("Unexpected number of completed containers", - targetContainers, master.getNumCompletedContainers()); - Assert.assertTrue("Master didn't finish containers as expected", - master.getDone()); + Assertions.assertEquals( + targetContainers, master.getNumCompletedContainers(), "Unexpected number of completed containers"); + Assertions.assertTrue( + master.getDone(), "Master didn't finish containers as expected"); // test for events from containers we know nothing about // these events should be ignored status = new ArrayList<>(); ContainerId id5 = BuilderUtils.newContainerId(1, 1, 1, 5); status.add(generateContainerStatus(id5, ContainerExitStatus.ABORTED)); - Assert.assertEquals("Unexpected number of completed containers", - targetContainers, master.getNumCompletedContainers()); - Assert.assertTrue("Master didn't finish containers as expected", - master.getDone()); + Assertions.assertEquals( + targetContainers, master.getNumCompletedContainers(), "Unexpected number of completed containers"); + Assertions.assertTrue( + master.getDone(), "Master didn't finish containers as expected"); status.add(generateContainerStatus(id5, ContainerExitStatus.SUCCESS)); - Assert.assertEquals("Unexpected number of completed containers", - targetContainers, master.getNumCompletedContainers()); - Assert.assertTrue("Master didn't finish containers as expected", - master.getDone()); + Assertions.assertEquals( + targetContainers, master.getNumCompletedContainers(), "Unexpected number of completed containers"); + Assertions.assertTrue( + master.getDone(), "Master didn't finish containers as expected"); } private Container generateContainer(ContainerId cid) { @@ -200,15 +200,15 @@ private void runTimelineClientInDSAppMaster(boolean v1Enabled, private void validateAppMasterTimelineService(boolean v1Enabled, boolean v2Enabled, ApplicationMaster appMaster) { if (v1Enabled) { - Assert.assertEquals(appMaster.appSubmitterUgi, + Assertions.assertEquals(appMaster.appSubmitterUgi, ((TimelineClientImpl)appMaster.timelineClient).getUgi()); } else { - Assert.assertNull(appMaster.timelineClient); + Assertions.assertNull(appMaster.timelineClient); } if (v2Enabled) { - Assert.assertNotNull(appMaster.timelineV2Client); + Assertions.assertNotNull(appMaster.timelineV2Client); } else { - Assert.assertNull(appMaster.timelineV2Client); + Assertions.assertNull(appMaster.timelineV2Client); } } @@ -227,7 +227,7 @@ private ApplicationMaster createAppMasterWithStartedTimelineService( private Configuration getTimelineServiceConf(boolean v1Enabled, boolean v2Enabled) { Configuration conf = new YarnConfiguration(new Configuration(false)); - Assert.assertFalse(YarnConfiguration.timelineServiceEnabled(conf)); + Assertions.assertFalse(YarnConfiguration.timelineServiceEnabled(conf)); if (v1Enabled || v2Enabled) { conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV10.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV10.java index ed7ed543b73d9..e7edcd9e4b106 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV10.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV10.java @@ -35,8 +35,8 @@ import java.util.concurrent.atomic.AtomicReference; import net.jodah.failsafe.RetryPolicy; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,9 +65,7 @@ import javax.ws.rs.ProcessingException; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -123,7 +121,7 @@ public void testDSRestartWithPreviousRunningContainers() throws Exception { boolean result = getDSClient().run(); LOG.info("Client run completed. Result={}", result); // application should succeed - Assert.assertTrue(result); + Assertions.assertTrue(result); } /* @@ -160,7 +158,7 @@ public void testDSAttemptFailuresValidityIntervalSuccess() throws Exception { LOG.info("Client run completed. Result = {}.", result); // application should succeed - Assert.assertTrue(result); + Assertions.assertTrue(result); } /* @@ -197,7 +195,7 @@ public void testDSAttemptFailuresValidityIntervalFailed() throws Exception { LOG.info("Client run completed. Result=" + result); // application should be failed - Assert.assertFalse(result); + Assertions.assertFalse(result); } @Test @@ -210,7 +208,7 @@ public void testDSShellWithCustomLogPropertyFile() throws Exception { customLogProperty.delete(); } if (!customLogProperty.createNewFile()) { - Assert.fail("Can not create custom log4j property file."); + Assertions.fail("Can not create custom log4j property file."); } PrintWriter fileWriter = new PrintWriter(customLogProperty); // set the output to DEBUG level @@ -238,26 +236,26 @@ public void testDSShellWithCustomLogPropertyFile() throws Exception { // Before run the DS, the default the log level is INFO final Logger LOG_Client = LoggerFactory.getLogger(Client.class); - Assert.assertTrue(LOG_Client.isInfoEnabled()); - Assert.assertFalse(LOG_Client.isDebugEnabled()); + Assertions.assertTrue(LOG_Client.isInfoEnabled()); + Assertions.assertFalse(LOG_Client.isDebugEnabled()); final Logger LOG_AM = LoggerFactory.getLogger(ApplicationMaster.class); - Assert.assertTrue(LOG_AM.isInfoEnabled()); - Assert.assertFalse(LOG_AM.isDebugEnabled()); + Assertions.assertTrue(LOG_AM.isInfoEnabled()); + Assertions.assertFalse(LOG_AM.isDebugEnabled()); LOG.info("Initializing DS Client"); setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); boolean initSuccess = getDSClient().init(args); - Assert.assertTrue(initSuccess); + Assertions.assertTrue(initSuccess); LOG.info("Running DS Client"); boolean result = getDSClient().run(); LOG.info("Client run completed. Result=" + result); - Assert.assertTrue(verifyContainerLog(3, null, true, "DEBUG") > 10); + Assertions.assertTrue(verifyContainerLog(3, null, true, "DEBUG") > 10); //After DS is finished, the log level should be DEBUG - Assert.assertTrue(LOG_Client.isInfoEnabled()); - Assert.assertTrue(LOG_Client.isDebugEnabled()); - Assert.assertTrue(LOG_AM.isInfoEnabled()); - Assert.assertTrue(LOG_AM.isDebugEnabled()); + Assertions.assertTrue(LOG_Client.isInfoEnabled()); + Assertions.assertTrue(LOG_Client.isDebugEnabled()); + Assertions.assertTrue(LOG_AM.isInfoEnabled()); + Assertions.assertTrue(LOG_AM.isDebugEnabled()); } @Test @@ -270,7 +268,7 @@ public void testSpecifyingLogAggregationContext() throws Exception { regex ); setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); - Assert.assertTrue(getDSClient().init(args)); + Assertions.assertTrue(getDSClient().init(args)); ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class); @@ -302,7 +300,7 @@ public void testDSShellWithMultipleArgs() throws Exception { LOG.info("Initializing DS Client"); setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); boolean initSuccess = getDSClient().init(args); - Assert.assertTrue(initSuccess); + Assertions.assertTrue(initSuccess); LOG.info("Running DS Client"); boolean result = getDSClient().run(); @@ -322,7 +320,7 @@ public void testDSShellWithShellScript() throws Exception { customShellScript.delete(); } if (!customShellScript.createNewFile()) { - Assert.fail("Can not create custom shell script file."); + Assertions.fail("Can not create custom shell script file."); } PrintWriter fileWriter = new PrintWriter(customShellScript); // set the output to DEBUG level @@ -346,7 +344,7 @@ public void testDSShellWithShellScript() throws Exception { LOG.info("Initializing DS Client"); setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); - Assert.assertTrue(getDSClient().init(args)); + Assertions.assertTrue(getDSClient().init(args)); LOG.info("Running DS Client"); assertTrue(getDSClient().run()); List expectedContent = new ArrayList<>(); @@ -579,9 +577,9 @@ public void testContainerLaunchFailureHandling() throws Exception { LOG.info("Initializing DS Client"); setAndGetDSClient(ContainerLaunchFailAppMaster.class.getName(), new Configuration(getYarnClusterConfiguration())); - Assert.assertTrue(getDSClient().init(args)); + Assertions.assertTrue(getDSClient().init(args)); LOG.info("Running DS Client"); - Assert.assertFalse(getDSClient().run()); + Assertions.assertFalse(getDSClient().run()); } @Test @@ -604,9 +602,9 @@ public void testDebugFlag() throws Exception { LOG.info("Initializing DS Client"); setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); - Assert.assertTrue(getDSClient().init(args)); + Assertions.assertTrue(getDSClient().init(args)); LOG.info("Running DS Client"); - Assert.assertTrue(getDSClient().run()); + Assertions.assertTrue(getDSClient().run()); } private int verifyContainerLog(int containerNum, @@ -617,7 +615,7 @@ private int verifyContainerLog(int containerNum, YarnConfiguration.DEFAULT_NM_LOG_DIRS)); File[] listOfFiles = logFolder.listFiles(); - Assert.assertNotNull(listOfFiles); + Assertions.assertNotNull(listOfFiles); int currentContainerLogFileIndex = -1; for (int i = listOfFiles.length - 1; i >= 0; i--) { if (listOfFiles[i].listFiles().length == containerNum + 1) { @@ -625,7 +623,7 @@ private int verifyContainerLog(int containerNum, break; } } - Assert.assertTrue(currentContainerLogFileIndex != -1); + Assertions.assertTrue(currentContainerLogFileIndex != -1); File[] containerFiles = listOfFiles[currentContainerLogFileIndex].listFiles(); @@ -648,8 +646,8 @@ private int verifyContainerLog(int containerNum, } } else if (output.getName().trim().equals("stdout")) { if (!Shell.WINDOWS) { - Assert.assertEquals("The current is" + sCurrentLine, - expectedContent.get(numOffline), sCurrentLine.trim()); + Assertions.assertEquals( + expectedContent.get(numOffline), sCurrentLine.trim(), "The current is" + sCurrentLine); numOffline++; } else { stdOutContent.add(sCurrentLine.trim()); @@ -664,7 +662,7 @@ private int verifyContainerLog(int containerNum, */ if (Shell.WINDOWS && !count && output.getName().trim().equals("stdout")) { - Assert.assertTrue(stdOutContent.containsAll(expectedContent)); + Assertions.assertTrue(stdOutContent.containsAll(expectedContent)); } } catch (IOException e) { LOG.error("Exception reading the buffer", e); @@ -696,7 +694,7 @@ public void testDistributedShellResourceProfiles() throws Exception { for (int i = 0; i < args.length; ++i) { LOG.info("Initializing DS Client[{}]", i); setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); - Assert.assertTrue(getDSClient().init(args[i])); + Assertions.assertTrue(getDSClient().init(args[i])); LOG.info("Running DS Client[{}]", i); LambdaTestUtils.intercept(Exception.class, () -> getDSClient().run()); @@ -727,36 +725,39 @@ public void testDSShellWithOpportunisticContainers() throws Exception { assertTrue(getDSClient().run()); } - @Test(expected = ResourceNotFoundException.class) + @Test public void testDistributedShellAMResourcesWithUnknownResource() throws Exception { - String[] args = createArgumentsWithAppName( - "--num_containers", - "1", - "--shell_command", - getListCommand(), - "--master_resources", - "unknown-resource=5" - ); - setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); - assertTrue(getDSClient().init(args)); - getDSClient().run(); + assertThrows(ResourceNotFoundException.class, () -> { + String[] args = createArgumentsWithAppName( + "--num_containers", + "1", + "--shell_command", + getListCommand(), + "--master_resources", + "unknown-resource=5" + ); + setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); + assertTrue(getDSClient().init(args)); + getDSClient().run(); + }); } - @Test(expected = IllegalArgumentException.class) - public void testDistributedShellNonExistentQueue() - throws Exception { - String[] args = createArgumentsWithAppName( - "--num_containers", - "1", - "--shell_command", - getListCommand(), - "--queue", - "non-existent-queue" - ); - setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); - assertTrue(getDSClient().init(args)); - getDSClient().run(); + @Test + public void testDistributedShellNonExistentQueue() throws Exception { + assertThrows(IllegalArgumentException.class, () -> { + String[] args = createArgumentsWithAppName( + "--num_containers", + "1", + "--shell_command", + getListCommand(), + "--queue", + "non-existent-queue" + ); + setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); + assertTrue(getDSClient().init(args)); + getDSClient().run(); + }); } @Test @@ -775,7 +776,7 @@ public void testDistributedShellWithSingleFileLocalization() setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); assertTrue(getDSClient().init(args)); - assertTrue("Client exited with an error", getDSClient().run()); + assertTrue(getDSClient().run(), "Client exited with an error"); } @Test @@ -794,26 +795,27 @@ public void testDistributedShellWithMultiFileLocalization() setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); assertTrue(getDSClient().init(args)); - assertTrue("Client exited with an error", getDSClient().run()); + assertTrue(getDSClient().run(), "Client exited with an error"); } - @Test(expected = UncheckedIOException.class) + @Test public void testDistributedShellWithNonExistentFileLocalization() throws Exception { - String[] args = createArgumentsWithAppName( - "--num_containers", - "1", - "--shell_command", - getCatCommand(), - "--localize_files", - "/non/existing/path/file.txt", - "--shell_args", - "file.txt" - ); - - setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); - assertTrue(getDSClient().init(args)); - assertTrue(getDSClient().run()); + assertThrows(UncheckedIOException.class, () -> { + String[] args = createArgumentsWithAppName( + "--num_containers", + "1", + "--shell_command", + getCatCommand(), + "--localize_files", + "/non/existing/path/file.txt", + "--shell_args", + "file.txt" + ); + setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); + assertTrue(getDSClient().init(args)); + assertTrue(getDSClient().run()); + }); } @Test @@ -845,7 +847,7 @@ public void testDistributedShellCleanup() } }, 10, 60000); - assertFalse("Distributed Shell Cleanup failed", fs1.exists(path)); + assertFalse(fs1.exists(path), "Distributed Shell Cleanup failed"); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV15.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV15.java index 634bac4df4326..3274a03af6ff9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV15.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV15.java @@ -20,8 +20,8 @@ import java.util.concurrent.atomic.AtomicReference; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,8 +83,8 @@ protected void checkTimeline(ApplicationId appId, return true; } }, scanInterval * 2, TEST_TIME_WINDOW_EXPIRE); - Assert.assertNull("Exception in getting listing status", - exceptionRef.get()); + Assertions.assertNull( + exceptionRef.get(), "Exception in getting listing status"); super.checkTimeline(appId, defaultFlow, haveDomain, appReport); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV20.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV20.java index caf9d3b8de7f1..0b07ddd84c4a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV20.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV20.java @@ -26,9 +26,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import org.junit.Assert; +import org.junit.jupiter.api.Assertions; import org.junit.Assume; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -137,18 +137,18 @@ public void testDSShellWithEnforceExecutionType() throws Exception { waitForContainersLaunch(yarnClient, 3, appAttemptReportRef, containersListRef, appAttemptIdRef, thrownError); if (thrownError.get() != null) { - Assert.fail(thrownError.get().getMessage()); + Assertions.fail(thrownError.get().getMessage()); } ContainerId amContainerId = appAttemptReportRef.get().getAMContainerId(); for (ContainerReport container : containersListRef.get()) { if (!container.getContainerId().equals(amContainerId)) { - Assert.assertEquals(container.getExecutionType(), + Assertions.assertEquals(container.getExecutionType(), ExecutionType.OPPORTUNISTIC); } } } catch (Exception e) { LOG.error("Job execution with enforce execution type failed.", e); - Assert.fail("Exception. " + e.getMessage()); + Assertions.fail("Exception. " + e.getMessage()); } finally { if (yarnClient != null) { yarnClient.stop(); @@ -209,7 +209,7 @@ private void doTestDistributedShellWithResources(boolean largeContainers) LOG.info("Initializing DS Client"); setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); - Assert.assertTrue(getDSClient().init(args)); + Assertions.assertTrue(getDSClient().init(args)); LOG.info("Running DS Client"); final AtomicBoolean result = new AtomicBoolean(false); Thread dsClientRunner = new Thread(() -> { @@ -228,19 +228,19 @@ private void doTestDistributedShellWithResources(boolean largeContainers) waitForContainersLaunch(yarnClient, 2, appAttemptReportRef, containersListRef, appAttemptIdRef, thrownExceptionRef); if (thrownExceptionRef.get() != null) { - Assert.fail(thrownExceptionRef.get().getMessage()); + Assertions.fail(thrownExceptionRef.get().getMessage()); } ContainerId amContainerId = appAttemptReportRef.get().getAMContainerId(); ContainerReport report = yarnClient.getContainerReport(amContainerId); Resource masterResource = report.getAllocatedResource(); - Assert.assertEquals(memVars[0], masterResource.getMemorySize()); - Assert.assertEquals(1, masterResource.getVirtualCores()); + Assertions.assertEquals(memVars[0], masterResource.getMemorySize()); + Assertions.assertEquals(1, masterResource.getVirtualCores()); for (ContainerReport container : containersListRef.get()) { if (!container.getContainerId().equals(amContainerId)) { Resource containerResource = container.getAllocatedResource(); - Assert.assertEquals(memVars[1], + Assertions.assertEquals(memVars[1], containerResource.getMemorySize()); - Assert.assertEquals(1, containerResource.getVirtualCores()); + Assertions.assertEquals(1, containerResource.getVirtualCores()); } } } finally { @@ -294,7 +294,7 @@ protected void checkTimeline(ApplicationId appId, boolean defaultFlow, File tmpRootFolder = new File(tmpRoot); try { - Assert.assertTrue(tmpRootFolder.isDirectory()); + Assertions.assertTrue(tmpRootFolder.isDirectory()); String basePath = tmpRoot + YarnConfiguration.DEFAULT_RM_CLUSTER_ID + File.separator + UserGroupInformation.getCurrentUser().getShortUserName() + @@ -436,21 +436,21 @@ private void verifyEntityForTimeline(File entityFile, String expectedEvent, TimelineEntity entity = FileSystemTimelineReaderImpl. getTimelineRecordFromJSON(entityLine, TimelineEntity.class); TimelineEvent event = entity.getEvents().pollFirst(); - Assert.assertNotNull(event); - Assert.assertTrue("diagnostics", - event.getInfo().containsKey(ApplicationMaster.DIAGNOSTICS)); + Assertions.assertNotNull(event); + Assertions.assertTrue( + event.getInfo().containsKey(ApplicationMaster.DIAGNOSTICS), "diagnostics"); } if (checkIdPrefix) { TimelineEntity entity = FileSystemTimelineReaderImpl. getTimelineRecordFromJSON(entityLine, TimelineEntity.class); - Assert.assertTrue("Entity ID prefix expected to be > 0", - entity.getIdPrefix() > 0); + Assertions.assertTrue( + entity.getIdPrefix() > 0, "Entity ID prefix expected to be > 0"); if (idPrefix == -1) { idPrefix = entity.getIdPrefix(); } else { - Assert.assertEquals( - "Entity ID prefix should be same across each publish of " - + "same entity", idPrefix, entity.getIdPrefix()); + Assertions.assertEquals( + idPrefix, entity.getIdPrefix(), "Entity ID prefix should be same across each publish of " + + "same entity"); } } } @@ -463,7 +463,7 @@ private void verifyEntityForTimeline(File entityFile, String expectedEvent, }, sleepTime, (checkTimes + 1) * sleepTime); if (thrownExceptionRef.get() != null) { - Assert.fail("verifyEntityForTimeline failed " + Assertions.fail("verifyEntityForTimeline failed " + thrownExceptionRef.get().getMessage()); } } @@ -475,10 +475,10 @@ private File verifyEntityTypeFileExists(String basePath, String entityType, LOG.info("verifyEntityTypeFileExists output path for entityType {}: {}", entityType, outputDirPathForEntity); File outputDirForEntity = new File(outputDirPathForEntity); - Assert.assertTrue(outputDirForEntity.isDirectory()); + Assertions.assertTrue(outputDirForEntity.isDirectory()); String entityFilePath = outputDirPathForEntity + entityFileName; File entityFile = new File(entityFilePath); - Assert.assertTrue(entityFile.exists()); + Assertions.assertTrue(entityFile.exists()); return entityFile; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java index 19f04237f09f4..43136e4c1569e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java @@ -28,13 +28,13 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.Rule; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.junit.rules.TestName; import org.junit.rules.Timeout; import org.junit.runner.RunWith; @@ -116,24 +116,24 @@ private YarnConfiguration getConfiguration( return conf; } - @BeforeClass + @BeforeAll public static void setupUnitTests() throws Exception { TestDSTimelineV10.setupUnitTests(); } - @AfterClass + @AfterAll public static void tearDownUnitTests() throws Exception { TestDSTimelineV10.tearDownUnitTests(); } - @Before + @BeforeEach public void setup() throws Exception { distShellTest = new TestDSTimelineV10(); distShellTest.setupInternal(NUM_NMS, getConfiguration(multiNodePlacementEnabled)); } - @After + @AfterEach public void tearDown() throws Exception { if (dsClient != null) { dsClient.sendStopSignal(); @@ -211,7 +211,7 @@ public void testDSShellWithNodeLabelExpression() throws Exception { dsClient = new Client( new Configuration(distShellTest.getYarnClusterConfiguration())); - Assert.assertTrue(dsClient.init(args)); + Assertions.assertTrue(dsClient.init(args)); LOG.info("Running DS Client"); boolean result = dsClient.run(); LOG.info("Client run completed. Result={}", result); @@ -222,9 +222,9 @@ public void testDSShellWithNodeLabelExpression() throws Exception { int[] maxRunningContainersOnNMs = containerMonitorRunner.getMaxRunningContainersReport(); // Check no container allocated on NM[0] - Assert.assertEquals(0, maxRunningContainersOnNMs[0]); + Assertions.assertEquals(0, maxRunningContainersOnNMs[0]); // Check there are some containers allocated on NM[1] - Assert.assertTrue(maxRunningContainersOnNMs[1] > 0); + Assertions.assertTrue(maxRunningContainersOnNMs[1] > 0); } finally { if (containerMonitorRunner != null) { containerMonitorRunner.stopMonitoring(); @@ -253,7 +253,7 @@ public void testDistributedShellWithPlacementConstraint() dsClient = new Client( new Configuration(distShellTest.getYarnClusterConfiguration())); - Assert.assertTrue(dsClient.init(args)); + Assertions.assertTrue(dsClient.init(args)); LOG.info("Running DS Client"); boolean result = dsClient.run(); LOG.info("Client run completed. Result={}", result); @@ -276,8 +276,8 @@ public void testDistributedShellWithPlacementConstraint() int[] maxRunningContainersOnNMs = containerMonitorRunner.getMaxRunningContainersReport(); - Assert.assertEquals(expectedNMsCount[0], maxRunningContainersOnNMs[0]); - Assert.assertEquals(expectedNMsCount[1], maxRunningContainersOnNMs[1]); + Assertions.assertEquals(expectedNMsCount[0], maxRunningContainersOnNMs[0]); + Assertions.assertEquals(expectedNMsCount[1], maxRunningContainersOnNMs[1]); } finally { if (containerMonitorRunner != null) { containerMonitorRunner.stopMonitoring(); @@ -339,7 +339,7 @@ public void testDistributedShellWithAllocationTagNamespace() getNodeId(); NodeId nodeB = distShellTest.getNodeManager(1).getNMContext(). getNodeId(); - Assert.assertEquals(2, (expectedNMCounts[0] + expectedNMCounts[1])); + Assertions.assertEquals(2, (expectedNMCounts[0] + expectedNMCounts[1])); if (expectedNMCounts[0] != expectedNMCounts[1]) { taskContainerNodeIdA = masterContainerNodeIdARef.get(); } else { @@ -351,7 +351,7 @@ public void testDistributedShellWithAllocationTagNamespace() new Client( new Configuration(distShellTest.getYarnClusterConfiguration())); clientB.init(argsB); - Assert.assertTrue(clientB.run()); + Assertions.assertTrue(clientB.run()); containerMonitorRunner.stopMonitoring(); apps = distShellTest.getResourceManager().getRMContext().getRMApps(); Iterator it = apps.values().iterator(); @@ -379,8 +379,8 @@ public void testDistributedShellWithAllocationTagNamespace() } int[] maxRunningContainersOnNMs = containerMonitorRunner.getMaxRunningContainersReport(); - Assert.assertEquals(expectedNMCounts[0], maxRunningContainersOnNMs[0]); - Assert.assertEquals(expectedNMCounts[1], maxRunningContainersOnNMs[1]); + Assertions.assertEquals(expectedNMCounts[0], maxRunningContainersOnNMs[0]); + Assertions.assertEquals(expectedNMCounts[1], maxRunningContainersOnNMs[1]); try { yarnClient = YarnClient.createYarnClient(); From 67d2548e8aa19533ca5caf65b299ab840c17f4e0 Mon Sep 17 00:00:00 2001 From: fanshilun Date: Mon, 10 Feb 2025 11:23:03 +0800 Subject: [PATCH 2/4] YARN-11760. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-applications-distributedshell. --- .../DistributedShellBaseTest.java | 92 ++++++------- .../distributedshell/TestDSAppMaster.java | 66 +++++----- .../distributedshell/TestDSTimelineV10.java | 122 +++++++++++------- .../distributedshell/TestDSTimelineV15.java | 16 ++- .../distributedshell/TestDSTimelineV20.java | 80 ++++++------ .../TestDSWithMultipleNodeManager.java | 11 +- 6 files changed, 209 insertions(+), 178 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java index 408a14cca4c4e..7f325b0651d3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java @@ -28,7 +28,6 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.List; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -36,13 +35,11 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.Rule; -import org.junit.rules.TemporaryFolder; -import org.junit.rules.TestName; -import org.junit.rules.Timeout; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,9 +72,15 @@ import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + /** * Base class for testing DistributedShell features. */ +@Timeout(160) public abstract class DistributedShellBaseTest { protected static final int MIN_ALLOCATION_MB = 128; protected static final int NUM_DATA_NODES = 1; @@ -105,13 +108,9 @@ public abstract class DistributedShellBaseTest { private static MiniYARNCluster yarnCluster = null; private static String yarnSiteBackupPath = null; private static String yarnSitePath = null; - @Rule - public Timeout globalTimeout = new Timeout(TEST_TIME_OUT, - TimeUnit.MILLISECONDS); - @Rule - public TemporaryFolder tmpFolder = new TemporaryFolder(); - @Rule - public TestName name = new TestName(); + @TempDir + public java.nio.file.Path tmpFolder; + private Client dsClient; private YarnConfiguration conf = null; // location of the filesystem timeline writer for timeline service v.2 @@ -212,12 +211,12 @@ public String getTimelineV2StorageDir() { } public void setTimelineV2StorageDir() throws Exception { - timelineV2StorageDir = tmpFolder.newFolder().getAbsolutePath(); + timelineV2StorageDir = tmpFolder.toFile().getAbsolutePath(); } @BeforeEach - public void setup() throws Exception { - setupInternal(NUM_NMS, new YarnConfiguration()); + public void setup(TestInfo testInfo) throws Exception { + setupInternal(NUM_NMS, new YarnConfiguration(), getMethodName(testInfo)); } @AfterEach @@ -232,8 +231,8 @@ public void tearDown() throws IOException { shutdownHdfsCluster(); } - protected String[] createArgumentsWithAppName(String... args) { - return createArguments(() -> generateAppName(), args); + protected String[] createArgumentsWithAppName(String methodName, String... args) { + return createArguments(() -> generateAppName(methodName), args); } protected void waitForContainersLaunch(YarnClient client, int nContainers, @@ -307,9 +306,10 @@ protected Client setAndGetDSClient(String appMasterMainClass, return dsClient; } - protected void baseTestDSShell(boolean haveDomain, boolean defaultFlow) + protected void baseTestDSShell(String methodName, boolean haveDomain, boolean defaultFlow) throws Exception { String[] baseArgs = createArgumentsWithAppName( + methodName, "--num_containers", "2", "--shell_command", @@ -329,7 +329,7 @@ protected void baseTestDSShell(boolean haveDomain, boolean defaultFlow) YarnClient yarnClient; dsClient = setAndGetDSClient(new Configuration(yarnCluster.getConfig())); boolean initSuccess = dsClient.init(args); - Assertions.assertTrue(initSuccess); + assertTrue(initSuccess); LOG.info("Running DS Client"); final AtomicBoolean result = new AtomicBoolean(false); Thread t = new Thread(() -> { @@ -379,17 +379,17 @@ protected void baseTestDSShell(boolean haveDomain, boolean defaultFlow) t.join(); if (waitResult.get() == 2) { // Exception was raised - Assertions.fail("Exception in getting application report. Failed"); + fail("Exception in getting application report. Failed"); } if (waitResult.get() == 1) { - Assertions.assertEquals( + assertEquals( -1, appReportRef.get().getRpcPort(), "Failed waiting for expected rpc port to be -1."); } checkTimeline(appIdRef.get(), defaultFlow, haveDomain, appReportRef.get()); } - protected void baseTestDSShell(boolean haveDomain) throws Exception { - baseTestDSShell(haveDomain, true); + protected void baseTestDSShell(String methodName, boolean haveDomain) throws Exception { + baseTestDSShell(methodName, haveDomain, true); } protected void checkTimeline(ApplicationId appId, @@ -399,22 +399,22 @@ protected void checkTimeline(ApplicationId appId, if (haveDomain) { domain = yarnCluster.getApplicationHistoryServer() .getTimelineStore().getDomain("TEST_DOMAIN"); - Assertions.assertNotNull(domain); - Assertions.assertEquals("reader_user reader_group", domain.getReaders()); - Assertions.assertEquals("writer_user writer_group", domain.getWriters()); + assertNotNull(domain); + assertEquals("reader_user reader_group", domain.getReaders()); + assertEquals("writer_user writer_group", domain.getWriters()); } TimelineEntities entitiesAttempts = yarnCluster .getApplicationHistoryServer() .getTimelineStore() .getEntities(ApplicationMaster.DSEntity.DS_APP_ATTEMPT.toString(), null, null, null, null, null, null, null, null, null); - Assertions.assertNotNull(entitiesAttempts); - Assertions.assertEquals(1, entitiesAttempts.getEntities().size()); - Assertions.assertEquals(2, entitiesAttempts.getEntities().get(0).getEvents() + assertNotNull(entitiesAttempts); + assertEquals(1, entitiesAttempts.getEntities().size()); + assertEquals(2, entitiesAttempts.getEntities().get(0).getEvents() .size()); - Assertions.assertEquals(entitiesAttempts.getEntities().get(0).getEntityType(), + assertEquals(entitiesAttempts.getEntities().get(0).getEntityType(), ApplicationMaster.DSEntity.DS_APP_ATTEMPT.toString()); - Assertions.assertEquals(haveDomain ? domain.getId() : "DEFAULT", + assertEquals(haveDomain ? domain.getId() : "DEFAULT", entitiesAttempts.getEntities().get(0).getDomainId()); String currAttemptEntityId = entitiesAttempts.getEntities().get(0).getEntityId(); @@ -428,9 +428,9 @@ protected void checkTimeline(ApplicationId appId, .getTimelineStore() .getEntities(ApplicationMaster.DSEntity.DS_CONTAINER.toString(), null, null, null, null, null, primaryFilter, null, null, null); - Assertions.assertNotNull(entities); - Assertions.assertEquals(2, entities.getEntities().size()); - Assertions.assertEquals(entities.getEntities().get(0).getEntityType(), + assertNotNull(entities); + assertEquals(2, entities.getEntities().size()); + assertEquals(entities.getEntities().get(0).getEntityType(), ApplicationMaster.DSEntity.DS_CONTAINER.toString()); String entityId = entities.getEntities().get(0).getEntityId(); @@ -438,9 +438,9 @@ protected void checkTimeline(ApplicationId appId, yarnCluster.getApplicationHistoryServer().getTimelineStore() .getEntity(entityId, ApplicationMaster.DSEntity.DS_CONTAINER.toString(), null); - Assertions.assertNotNull(entity); - Assertions.assertEquals(entityId, entity.getEntityId()); - Assertions.assertEquals(haveDomain ? domain.getId() : "DEFAULT", + assertNotNull(entity); + assertEquals(entityId, entity.getEntityId()); + assertEquals(haveDomain ? domain.getId() : "DEFAULT", entities.getEntities().get(0).getDomainId()); } @@ -452,12 +452,12 @@ protected String[] createArgsWithPostFix(int index, String... args) { return res; } - protected String generateAppName() { - return generateAppName(null); + protected String generateAppName(String methodName) { + return generateAppName(methodName,null); } - protected String generateAppName(String postFix) { - return name.getMethodName().replaceFirst("test", "") + protected String generateAppName(String methodName, String postFix) { + return methodName.replaceFirst("test", "") .concat(postFix == null ? "" : "-" + postFix); } @@ -501,9 +501,9 @@ protected void setUpYarnCluster(int numNodeManagers, } protected void setupInternal(int numNodeManagers, - YarnConfiguration yarnConfig) throws Exception { + YarnConfiguration yarnConfig, String methodName) throws Exception { LOG.info("========== Setting UP UnitTest {}#{} ==========", - getClass().getCanonicalName(), name.getMethodName()); + getClass().getCanonicalName(), methodName); LOG.info("Starting up YARN cluster. Timeline version {}", getTimelineVersion()); conf = yarnConfig; @@ -604,4 +604,8 @@ private void waitForNMsToRegister() throws Exception { protected MiniDFSCluster getHDFSCluster() { return hdfsCluster; } + + public String getMethodName(TestInfo testInfo) { + return testInfo.getTestMethod().get().getName(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java index 8c3b8fc432b0b..81420465afb90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java @@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter; import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; @@ -48,6 +47,12 @@ import java.util.Arrays; import java.util.List; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + /** * A bunch of tests to make sure that the container allocations * and releases occur correctly. @@ -105,12 +110,12 @@ public void testDSAppMasterAllocateHandler() throws Exception { // first allocate a single container, everything should be fine handler.onContainersAllocated(containers); - Assertions.assertEquals(1 -, master.getAllocatedContainers(), "Wrong container allocation count"); - Assertions.assertEquals(1 -, master.threadsLaunched, "Incorrect number of threads launched"); - Assertions.assertEquals( - Arrays.asList("1"), master.yarnShellIds, "Incorrect YARN Shell IDs"); + assertEquals(1, master.getAllocatedContainers(), + "Wrong container allocation count"); + assertEquals(1, master.threadsLaunched, + "Incorrect number of threads launched"); + assertEquals(Arrays.asList("1"), master.yarnShellIds, + "Incorrect YARN Shell IDs"); // now send 3 extra containers containers.clear(); @@ -121,14 +126,12 @@ public void testDSAppMasterAllocateHandler() throws Exception { ContainerId id4 = BuilderUtils.newContainerId(1, 1, 1, 4); containers.add(generateContainer(id4)); handler.onContainersAllocated(containers); - Assertions.assertEquals(2 -, master.getAllocatedContainers(), "Wrong final container allocation count"); - - Assertions.assertEquals(2 -, master.threadsLaunched, "Incorrect number of threads launched"); - - Assertions.assertEquals( - Arrays.asList("1", "2"), master.yarnShellIds, "Incorrect YARN Shell IDs"); + assertEquals(2, master.getAllocatedContainers(), + "Wrong final container allocation count"); + assertEquals(2, master.threadsLaunched, + "Incorrect number of threads launched"); + assertEquals(Arrays.asList("1", "2"), master.yarnShellIds, + "Incorrect YARN Shell IDs"); // make sure we handle completion events correctly List status = new ArrayList<>(); status.add(generateContainerStatus(id1, ContainerExitStatus.SUCCESS)); @@ -137,25 +140,22 @@ public void testDSAppMasterAllocateHandler() throws Exception { status.add(generateContainerStatus(id4, ContainerExitStatus.ABORTED)); handler.onContainersCompleted(status); - Assertions.assertEquals( - targetContainers, master.getNumCompletedContainers(), "Unexpected number of completed containers"); - Assertions.assertTrue( - master.getDone(), "Master didn't finish containers as expected"); + assertEquals(targetContainers, master.getNumCompletedContainers(), + "Unexpected number of completed containers"); + assertTrue(master.getDone(), "Master didn't finish containers as expected"); // test for events from containers we know nothing about // these events should be ignored status = new ArrayList<>(); ContainerId id5 = BuilderUtils.newContainerId(1, 1, 1, 5); status.add(generateContainerStatus(id5, ContainerExitStatus.ABORTED)); - Assertions.assertEquals( - targetContainers, master.getNumCompletedContainers(), "Unexpected number of completed containers"); - Assertions.assertTrue( - master.getDone(), "Master didn't finish containers as expected"); + assertEquals(targetContainers, master.getNumCompletedContainers(), + "Unexpected number of completed containers"); + assertTrue(master.getDone(), "Master didn't finish containers as expected"); status.add(generateContainerStatus(id5, ContainerExitStatus.SUCCESS)); - Assertions.assertEquals( - targetContainers, master.getNumCompletedContainers(), "Unexpected number of completed containers"); - Assertions.assertTrue( - master.getDone(), "Master didn't finish containers as expected"); + assertEquals(targetContainers, master.getNumCompletedContainers(), + "Unexpected number of completed containers"); + assertTrue(master.getDone(), "Master didn't finish containers as expected"); } private Container generateContainer(ContainerId cid) { @@ -200,15 +200,15 @@ private void runTimelineClientInDSAppMaster(boolean v1Enabled, private void validateAppMasterTimelineService(boolean v1Enabled, boolean v2Enabled, ApplicationMaster appMaster) { if (v1Enabled) { - Assertions.assertEquals(appMaster.appSubmitterUgi, - ((TimelineClientImpl)appMaster.timelineClient).getUgi()); + assertEquals(appMaster.appSubmitterUgi, + ((TimelineClientImpl) appMaster.timelineClient).getUgi()); } else { - Assertions.assertNull(appMaster.timelineClient); + assertNull(appMaster.timelineClient); } if (v2Enabled) { - Assertions.assertNotNull(appMaster.timelineV2Client); + assertNotNull(appMaster.timelineV2Client); } else { - Assertions.assertNull(appMaster.timelineV2Client); + assertNull(appMaster.timelineV2Client); } } @@ -227,7 +227,7 @@ private ApplicationMaster createAppMasterWithStartedTimelineService( private Configuration getTimelineServiceConf(boolean v1Enabled, boolean v2Enabled) { Configuration conf = new YarnConfiguration(new Configuration(false)); - Assertions.assertFalse(YarnConfiguration.timelineServiceEnabled(conf)); + assertFalse(YarnConfiguration.timelineServiceEnabled(conf)); if (v1Enabled || v2Enabled) { conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV10.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV10.java index e7edcd9e4b106..cc4e784528bb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV10.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV10.java @@ -35,8 +35,8 @@ import java.util.concurrent.atomic.AtomicReference; import net.jodah.failsafe.RetryPolicy; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,7 +65,12 @@ import javax.ws.rs.ProcessingException; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -88,18 +93,20 @@ protected void cleanUpDFSClient() { } @Test - public void testDSShellWithDomain() throws Exception { - baseTestDSShell(true); + public void testDSShellWithDomain(TestInfo testInfo) throws Exception { + baseTestDSShell(getMethodName(testInfo), true); } @Test - public void testDSShellWithoutDomain() throws Exception { - baseTestDSShell(false); + public void testDSShellWithoutDomain(TestInfo testInfo) throws Exception { + baseTestDSShell(getMethodName(testInfo),false); } @Test - public void testDSRestartWithPreviousRunningContainers() throws Exception { + public void testDSRestartWithPreviousRunningContainers(TestInfo testInfo) + throws Exception { String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "1", "--shell_command", @@ -121,7 +128,7 @@ public void testDSRestartWithPreviousRunningContainers() throws Exception { boolean result = getDSClient().run(); LOG.info("Client run completed. Result={}", result); // application should succeed - Assertions.assertTrue(result); + assertTrue(result); } /* @@ -131,8 +138,9 @@ public void testDSRestartWithPreviousRunningContainers() throws Exception { * The application is expected to be successful. */ @Test - public void testDSAttemptFailuresValidityIntervalSuccess() throws Exception { + public void testDSAttemptFailuresValidityIntervalSuccess(TestInfo testInfo) throws Exception { String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "1", "--shell_command", @@ -158,7 +166,7 @@ public void testDSAttemptFailuresValidityIntervalSuccess() throws Exception { LOG.info("Client run completed. Result = {}.", result); // application should succeed - Assertions.assertTrue(result); + assertTrue(result); } /* @@ -168,8 +176,9 @@ public void testDSAttemptFailuresValidityIntervalSuccess() throws Exception { * The application is expected to be fail. */ @Test - public void testDSAttemptFailuresValidityIntervalFailed() throws Exception { + public void testDSAttemptFailuresValidityIntervalFailed(TestInfo testInfo) throws Exception { String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "1", "--shell_command", @@ -195,11 +204,11 @@ public void testDSAttemptFailuresValidityIntervalFailed() throws Exception { LOG.info("Client run completed. Result=" + result); // application should be failed - Assertions.assertFalse(result); + assertFalse(result); } @Test - public void testDSShellWithCustomLogPropertyFile() throws Exception { + public void testDSShellWithCustomLogPropertyFile(TestInfo testInfo) throws Exception { final File basedir = getBaseDirForTest(); final File tmpDir = new File(basedir, "tmpDir"); tmpDir.mkdirs(); @@ -208,13 +217,14 @@ public void testDSShellWithCustomLogPropertyFile() throws Exception { customLogProperty.delete(); } if (!customLogProperty.createNewFile()) { - Assertions.fail("Can not create custom log4j property file."); + fail("Can not create custom log4j property file."); } PrintWriter fileWriter = new PrintWriter(customLogProperty); // set the output to DEBUG level fileWriter.write("log4j.rootLogger=debug,stdout"); fileWriter.close(); String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "3", "--shell_command", @@ -236,39 +246,40 @@ public void testDSShellWithCustomLogPropertyFile() throws Exception { // Before run the DS, the default the log level is INFO final Logger LOG_Client = LoggerFactory.getLogger(Client.class); - Assertions.assertTrue(LOG_Client.isInfoEnabled()); - Assertions.assertFalse(LOG_Client.isDebugEnabled()); + assertTrue(LOG_Client.isInfoEnabled()); + assertFalse(LOG_Client.isDebugEnabled()); final Logger LOG_AM = LoggerFactory.getLogger(ApplicationMaster.class); - Assertions.assertTrue(LOG_AM.isInfoEnabled()); - Assertions.assertFalse(LOG_AM.isDebugEnabled()); + assertTrue(LOG_AM.isInfoEnabled()); + assertFalse(LOG_AM.isDebugEnabled()); LOG.info("Initializing DS Client"); setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); boolean initSuccess = getDSClient().init(args); - Assertions.assertTrue(initSuccess); + assertTrue(initSuccess); LOG.info("Running DS Client"); boolean result = getDSClient().run(); LOG.info("Client run completed. Result=" + result); - Assertions.assertTrue(verifyContainerLog(3, null, true, "DEBUG") > 10); + assertTrue(verifyContainerLog(3, null, true, "DEBUG") > 10); //After DS is finished, the log level should be DEBUG - Assertions.assertTrue(LOG_Client.isInfoEnabled()); - Assertions.assertTrue(LOG_Client.isDebugEnabled()); - Assertions.assertTrue(LOG_AM.isInfoEnabled()); - Assertions.assertTrue(LOG_AM.isDebugEnabled()); + assertTrue(LOG_Client.isInfoEnabled()); + assertTrue(LOG_Client.isDebugEnabled()); + assertTrue(LOG_AM.isInfoEnabled()); + assertTrue(LOG_AM.isDebugEnabled()); } @Test - public void testSpecifyingLogAggregationContext() throws Exception { + public void testSpecifyingLogAggregationContext(TestInfo testInfo) throws Exception { String regex = ".*(foo|bar)\\d"; String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--shell_command", "echo", "--rolling_log_pattern", regex ); setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); - Assertions.assertTrue(getDSClient().init(args)); + assertTrue(getDSClient().init(args)); ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class); @@ -279,8 +290,9 @@ public void testSpecifyingLogAggregationContext() throws Exception { } @Test - public void testDSShellWithMultipleArgs() throws Exception { + public void testDSShellWithMultipleArgs(TestInfo testInfo) throws Exception { String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "4", "--shell_command", @@ -300,7 +312,7 @@ public void testDSShellWithMultipleArgs() throws Exception { LOG.info("Initializing DS Client"); setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); boolean initSuccess = getDSClient().init(args); - Assertions.assertTrue(initSuccess); + assertTrue(initSuccess); LOG.info("Running DS Client"); boolean result = getDSClient().run(); @@ -311,7 +323,7 @@ public void testDSShellWithMultipleArgs() throws Exception { } @Test - public void testDSShellWithShellScript() throws Exception { + public void testDSShellWithShellScript(TestInfo testInfo) throws Exception { final File basedir = getBaseDirForTest(); final File tmpDir = new File(basedir, "tmpDir"); tmpDir.mkdirs(); @@ -320,7 +332,7 @@ public void testDSShellWithShellScript() throws Exception { customShellScript.delete(); } if (!customShellScript.createNewFile()) { - Assertions.fail("Can not create custom shell script file."); + fail("Can not create custom shell script file."); } PrintWriter fileWriter = new PrintWriter(customShellScript); // set the output to DEBUG level @@ -328,6 +340,7 @@ public void testDSShellWithShellScript() throws Exception { fileWriter.close(); LOG.info(customShellScript.getAbsolutePath()); String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "1", "--shell_script", @@ -344,7 +357,7 @@ public void testDSShellWithShellScript() throws Exception { LOG.info("Initializing DS Client"); setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); - Assertions.assertTrue(getDSClient().init(args)); + assertTrue(getDSClient().init(args)); LOG.info("Running DS Client"); assertTrue(getDSClient().run()); List expectedContent = new ArrayList<>(); @@ -562,8 +575,9 @@ protected TimelineWriter createTimelineWriter(Configuration conf, } @Test - public void testContainerLaunchFailureHandling() throws Exception { + public void testContainerLaunchFailureHandling(TestInfo testInfo) throws Exception { String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "2", "--shell_command", @@ -577,14 +591,15 @@ public void testContainerLaunchFailureHandling() throws Exception { LOG.info("Initializing DS Client"); setAndGetDSClient(ContainerLaunchFailAppMaster.class.getName(), new Configuration(getYarnClusterConfiguration())); - Assertions.assertTrue(getDSClient().init(args)); + assertTrue(getDSClient().init(args)); LOG.info("Running DS Client"); - Assertions.assertFalse(getDSClient().run()); + assertFalse(getDSClient().run()); } @Test - public void testDebugFlag() throws Exception { + public void testDebugFlag(TestInfo testInfo) throws Exception { String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "2", "--shell_command", @@ -602,9 +617,9 @@ public void testDebugFlag() throws Exception { LOG.info("Initializing DS Client"); setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); - Assertions.assertTrue(getDSClient().init(args)); + assertTrue(getDSClient().init(args)); LOG.info("Running DS Client"); - Assertions.assertTrue(getDSClient().run()); + assertTrue(getDSClient().run()); } private int verifyContainerLog(int containerNum, @@ -615,7 +630,7 @@ private int verifyContainerLog(int containerNum, YarnConfiguration.DEFAULT_NM_LOG_DIRS)); File[] listOfFiles = logFolder.listFiles(); - Assertions.assertNotNull(listOfFiles); + assertNotNull(listOfFiles); int currentContainerLogFileIndex = -1; for (int i = listOfFiles.length - 1; i >= 0; i--) { if (listOfFiles[i].listFiles().length == containerNum + 1) { @@ -623,7 +638,7 @@ private int verifyContainerLog(int containerNum, break; } } - Assertions.assertTrue(currentContainerLogFileIndex != -1); + assertTrue(currentContainerLogFileIndex != -1); File[] containerFiles = listOfFiles[currentContainerLogFileIndex].listFiles(); @@ -646,7 +661,7 @@ private int verifyContainerLog(int containerNum, } } else if (output.getName().trim().equals("stdout")) { if (!Shell.WINDOWS) { - Assertions.assertEquals( + assertEquals( expectedContent.get(numOffline), sCurrentLine.trim(), "The current is" + sCurrentLine); numOffline++; } else { @@ -662,7 +677,7 @@ private int verifyContainerLog(int containerNum, */ if (Shell.WINDOWS && !count && output.getName().trim().equals("stdout")) { - Assertions.assertTrue(stdOutContent.containsAll(expectedContent)); + assertTrue(stdOutContent.containsAll(expectedContent)); } } catch (IOException e) { LOG.error("Exception reading the buffer", e); @@ -694,7 +709,7 @@ public void testDistributedShellResourceProfiles() throws Exception { for (int i = 0; i < args.length; ++i) { LOG.info("Initializing DS Client[{}]", i); setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); - Assertions.assertTrue(getDSClient().init(args[i])); + assertTrue(getDSClient().init(args[i])); LOG.info("Running DS Client[{}]", i); LambdaTestUtils.intercept(Exception.class, () -> getDSClient().run()); @@ -702,10 +717,11 @@ public void testDistributedShellResourceProfiles() throws Exception { } @Test - public void testDSShellWithOpportunisticContainers() throws Exception { + public void testDSShellWithOpportunisticContainers(TestInfo testInfo) throws Exception { setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "2", "--master_memory", @@ -726,10 +742,11 @@ public void testDSShellWithOpportunisticContainers() throws Exception { } @Test - public void testDistributedShellAMResourcesWithUnknownResource() + public void testDistributedShellAMResourcesWithUnknownResource(TestInfo testInfo) throws Exception { assertThrows(ResourceNotFoundException.class, () -> { String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "1", "--shell_command", @@ -744,9 +761,10 @@ public void testDistributedShellAMResourcesWithUnknownResource() } @Test - public void testDistributedShellNonExistentQueue() throws Exception { + public void testDistributedShellNonExistentQueue(TestInfo testInfo) throws Exception { assertThrows(IllegalArgumentException.class, () -> { String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "1", "--shell_command", @@ -761,9 +779,10 @@ public void testDistributedShellNonExistentQueue() throws Exception { } @Test - public void testDistributedShellWithSingleFileLocalization() + public void testDistributedShellWithSingleFileLocalization(TestInfo testInfo) throws Exception { String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "1", "--shell_command", @@ -780,9 +799,10 @@ public void testDistributedShellWithSingleFileLocalization() } @Test - public void testDistributedShellWithMultiFileLocalization() + public void testDistributedShellWithMultiFileLocalization(TestInfo testInfo) throws Exception { String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "1", "--shell_command", @@ -799,10 +819,11 @@ public void testDistributedShellWithMultiFileLocalization() } @Test - public void testDistributedShellWithNonExistentFileLocalization() + public void testDistributedShellWithNonExistentFileLocalization(TestInfo testInfo) throws Exception { assertThrows(UncheckedIOException.class, () -> { String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "1", "--shell_command", @@ -819,9 +840,10 @@ public void testDistributedShellWithNonExistentFileLocalization() } @Test - public void testDistributedShellCleanup() + public void testDistributedShellCleanup(TestInfo testInfo) throws Exception { String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "1", "--shell_command", @@ -834,7 +856,7 @@ public void testDistributedShellCleanup() assertTrue(getDSClient().run()); ApplicationId appId = getDSClient().getAppId(); String relativePath = - ApplicationMaster.getRelativePath(generateAppName(), + ApplicationMaster.getRelativePath(generateAppName(getMethodName(testInfo)), appId.toString(), ""); FileSystem fs1 = FileSystem.get(config); Path path = new Path(fs1.getHomeDirectory(), relativePath); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV15.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV15.java index 3274a03af6ff9..8d0d1b1c54578 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV15.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV15.java @@ -20,8 +20,8 @@ import java.util.concurrent.atomic.AtomicReference; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +34,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.timeline.PluginStoreTestUtils; +import static org.junit.jupiter.api.Assertions.assertNull; + /** * Unit tests implementations for distributed shell on TimeLineV1.5. */ @@ -83,18 +85,18 @@ protected void checkTimeline(ApplicationId appId, return true; } }, scanInterval * 2, TEST_TIME_WINDOW_EXPIRE); - Assertions.assertNull( - exceptionRef.get(), "Exception in getting listing status"); + assertNull(exceptionRef.get(), + "Exception in getting listing status"); super.checkTimeline(appId, defaultFlow, haveDomain, appReport); } @Test - public void testDSShellWithDomain() throws Exception { - baseTestDSShell(true); + public void testDSShellWithDomain(TestInfo testInfo) throws Exception { + baseTestDSShell(getMethodName(testInfo), true); } @Test - public void testDSShellWithoutDomain() throws Exception { - baseTestDSShell(false); + public void testDSShellWithoutDomain(TestInfo testInfo) throws Exception { + baseTestDSShell(getMethodName(testInfo), false); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV20.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV20.java index 0b07ddd84c4a4..0e3b963732aa6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV20.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV20.java @@ -26,9 +26,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import org.junit.jupiter.api.Assertions; -import org.junit.Assume; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,6 +57,12 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + /** * Unit tests implementations for distributed shell on TimeLineV2. */ @@ -91,7 +96,7 @@ protected void customizeConfiguration( } @Test - public void testDSShellWithEnforceExecutionType() throws Exception { + public void testDSShellWithEnforceExecutionType(TestInfo testInfo) throws Exception { YarnClient yarnClient = null; AtomicReference thrownError = new AtomicReference<>(null); AtomicReference> containersListRef = @@ -101,6 +106,7 @@ public void testDSShellWithEnforceExecutionType() throws Exception { AtomicReference appAttemptReportRef = new AtomicReference<>(null); String[] args = createArgumentsWithAppName( + getMethodName(testInfo), "--num_containers", "2", "--master_memory", @@ -137,18 +143,18 @@ public void testDSShellWithEnforceExecutionType() throws Exception { waitForContainersLaunch(yarnClient, 3, appAttemptReportRef, containersListRef, appAttemptIdRef, thrownError); if (thrownError.get() != null) { - Assertions.fail(thrownError.get().getMessage()); + fail(thrownError.get().getMessage()); } ContainerId amContainerId = appAttemptReportRef.get().getAMContainerId(); for (ContainerReport container : containersListRef.get()) { if (!container.getContainerId().equals(amContainerId)) { - Assertions.assertEquals(container.getExecutionType(), + assertEquals(container.getExecutionType(), ExecutionType.OPPORTUNISTIC); } } } catch (Exception e) { LOG.error("Job execution with enforce execution type failed.", e); - Assertions.fail("Exception. " + e.getMessage()); + fail("Exception. " + e.getMessage()); } finally { if (yarnClient != null) { yarnClient.stop(); @@ -157,17 +163,18 @@ public void testDSShellWithEnforceExecutionType() throws Exception { } @Test - public void testDistributedShellWithResources() throws Exception { - doTestDistributedShellWithResources(false); + public void testDistributedShellWithResources(TestInfo testInfo) throws Exception { + doTestDistributedShellWithResources(getMethodName(testInfo), false); } @Test - public void testDistributedShellWithResourcesWithLargeContainers() - throws Exception { - doTestDistributedShellWithResources(true); + public void testDistributedShellWithResourcesWithLargeContainers(TestInfo testInfo) + throws Exception { + doTestDistributedShellWithResources(getMethodName(testInfo), true); } - private void doTestDistributedShellWithResources(boolean largeContainers) + private void doTestDistributedShellWithResources( + String methodName, boolean largeContainers) throws Exception { AtomicReference thrownExceptionRef = new AtomicReference<>(null); @@ -183,10 +190,10 @@ private void doTestDistributedShellWithResources(boolean largeContainers) String containerMemoryString = "512 Mi"; long[] memVars = {1024, 512}; YarnClient yarnClient = null; - Assume.assumeTrue("The cluster doesn't have enough memory for this test", - clusterResource.getMemorySize() >= memVars[0] + memVars[1]); - Assume.assumeTrue("The cluster doesn't have enough cores for this test", - clusterResource.getVirtualCores() >= 2); + assumeTrue(clusterResource.getMemorySize() >= memVars[0] + memVars[1], + "The cluster doesn't have enough memory for this test"); + assumeTrue(clusterResource.getVirtualCores() >= 2, + "The cluster doesn't have enough cores for this test"); if (largeContainers) { memVars[0] = clusterResource.getMemorySize() * 2 / 3; memVars[0] = memVars[0] - memVars[0] % MIN_ALLOCATION_MB; @@ -197,6 +204,7 @@ private void doTestDistributedShellWithResources(boolean largeContainers) } String[] args = createArgumentsWithAppName( + methodName, "--num_containers", "2", "--shell_command", @@ -209,7 +217,7 @@ private void doTestDistributedShellWithResources(boolean largeContainers) LOG.info("Initializing DS Client"); setAndGetDSClient(new Configuration(getYarnClusterConfiguration())); - Assertions.assertTrue(getDSClient().init(args)); + assertTrue(getDSClient().init(args)); LOG.info("Running DS Client"); final AtomicBoolean result = new AtomicBoolean(false); Thread dsClientRunner = new Thread(() -> { @@ -228,19 +236,19 @@ private void doTestDistributedShellWithResources(boolean largeContainers) waitForContainersLaunch(yarnClient, 2, appAttemptReportRef, containersListRef, appAttemptIdRef, thrownExceptionRef); if (thrownExceptionRef.get() != null) { - Assertions.fail(thrownExceptionRef.get().getMessage()); + fail(thrownExceptionRef.get().getMessage()); } ContainerId amContainerId = appAttemptReportRef.get().getAMContainerId(); ContainerReport report = yarnClient.getContainerReport(amContainerId); Resource masterResource = report.getAllocatedResource(); - Assertions.assertEquals(memVars[0], masterResource.getMemorySize()); - Assertions.assertEquals(1, masterResource.getVirtualCores()); + assertEquals(memVars[0], masterResource.getMemorySize()); + assertEquals(1, masterResource.getVirtualCores()); for (ContainerReport container : containersListRef.get()) { if (!container.getContainerId().equals(amContainerId)) { Resource containerResource = container.getAllocatedResource(); - Assertions.assertEquals(memVars[1], + assertEquals(memVars[1], containerResource.getMemorySize()); - Assertions.assertEquals(1, containerResource.getVirtualCores()); + assertEquals(1, containerResource.getVirtualCores()); } } } finally { @@ -253,18 +261,18 @@ private void doTestDistributedShellWithResources(boolean largeContainers) } @Test - public void testDSShellWithoutDomain() throws Exception { - baseTestDSShell(false); + public void testDSShellWithoutDomain(TestInfo testInfo) throws Exception { + baseTestDSShell(getMethodName(testInfo), false); } @Test - public void testDSShellWithoutDomainDefaultFlow() throws Exception { - baseTestDSShell(false, true); + public void testDSShellWithoutDomainDefaultFlow(TestInfo testInfo) throws Exception { + baseTestDSShell(getMethodName(testInfo), false, true); } @Test - public void testDSShellWithoutDomainCustomizedFlow() throws Exception { - baseTestDSShell(false, false); + public void testDSShellWithoutDomainCustomizedFlow(TestInfo testInfo) throws Exception { + baseTestDSShell(getMethodName(testInfo), false, false); } @Override @@ -294,7 +302,7 @@ protected void checkTimeline(ApplicationId appId, boolean defaultFlow, File tmpRootFolder = new File(tmpRoot); try { - Assertions.assertTrue(tmpRootFolder.isDirectory()); + assertTrue(tmpRootFolder.isDirectory()); String basePath = tmpRoot + YarnConfiguration.DEFAULT_RM_CLUSTER_ID + File.separator + UserGroupInformation.getCurrentUser().getShortUserName() + @@ -436,19 +444,19 @@ private void verifyEntityForTimeline(File entityFile, String expectedEvent, TimelineEntity entity = FileSystemTimelineReaderImpl. getTimelineRecordFromJSON(entityLine, TimelineEntity.class); TimelineEvent event = entity.getEvents().pollFirst(); - Assertions.assertNotNull(event); - Assertions.assertTrue( + assertNotNull(event); + assertTrue( event.getInfo().containsKey(ApplicationMaster.DIAGNOSTICS), "diagnostics"); } if (checkIdPrefix) { TimelineEntity entity = FileSystemTimelineReaderImpl. getTimelineRecordFromJSON(entityLine, TimelineEntity.class); - Assertions.assertTrue( + assertTrue( entity.getIdPrefix() > 0, "Entity ID prefix expected to be > 0"); if (idPrefix == -1) { idPrefix = entity.getIdPrefix(); } else { - Assertions.assertEquals( + assertEquals( idPrefix, entity.getIdPrefix(), "Entity ID prefix should be same across each publish of " + "same entity"); } @@ -463,7 +471,7 @@ private void verifyEntityForTimeline(File entityFile, String expectedEvent, }, sleepTime, (checkTimes + 1) * sleepTime); if (thrownExceptionRef.get() != null) { - Assertions.fail("verifyEntityForTimeline failed " + fail("verifyEntityForTimeline failed " + thrownExceptionRef.get().getMessage()); } } @@ -475,10 +483,10 @@ private File verifyEntityTypeFileExists(String basePath, String entityType, LOG.info("verifyEntityTypeFileExists output path for entityType {}: {}", entityType, outputDirPathForEntity); File outputDirForEntity = new File(outputDirPathForEntity); - Assertions.assertTrue(outputDirForEntity.isDirectory()); + assertTrue(outputDirForEntity.isDirectory()); String entityFilePath = outputDirPathForEntity + entityFileName; File entityFile = new File(entityFilePath); - Assertions.assertTrue(entityFile.exists()); + assertTrue(entityFile.exists()); return entityFile; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java index 43136e4c1569e..467df6cdbcd1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java @@ -28,13 +28,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.*; import org.junit.Rule; -import org.junit.jupiter.api.Test; import org.junit.rules.TestName; import org.junit.rules.Timeout; import org.junit.runner.RunWith; @@ -127,10 +122,10 @@ public static void tearDownUnitTests() throws Exception { } @BeforeEach - public void setup() throws Exception { + public void setup(TestInfo testInfo) throws Exception { distShellTest = new TestDSTimelineV10(); distShellTest.setupInternal(NUM_NMS, - getConfiguration(multiNodePlacementEnabled)); + getConfiguration(multiNodePlacementEnabled), testInfo.getDisplayName()); } @AfterEach From 25979d4ac04b3397365ae547f952d10287dc102b Mon Sep 17 00:00:00 2001 From: fanshilun Date: Mon, 10 Feb 2025 14:05:14 +0800 Subject: [PATCH 3/4] HDFS-11760. Fix Junit Test & CheckStyle. --- .../DistributedShellBaseTest.java | 8 +- .../distributedshell/TestDSTimelineV10.java | 6 +- .../distributedshell/TestDSTimelineV20.java | 13 ++- .../TestDSWithMultipleNodeManager.java | 91 ++++++++++--------- 4 files changed, 63 insertions(+), 55 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java index 7f325b0651d3b..b77b50a0592ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java @@ -109,7 +109,7 @@ public abstract class DistributedShellBaseTest { private static String yarnSiteBackupPath = null; private static String yarnSitePath = null; @TempDir - public java.nio.file.Path tmpFolder; + private java.nio.file.Path tmpFolder; private Client dsClient; private YarnConfiguration conf = null; @@ -382,8 +382,8 @@ protected void baseTestDSShell(String methodName, boolean haveDomain, boolean de fail("Exception in getting application report. Failed"); } if (waitResult.get() == 1) { - assertEquals( - -1, appReportRef.get().getRpcPort(), "Failed waiting for expected rpc port to be -1."); + assertEquals(-1, appReportRef.get().getRpcPort(), + "Failed waiting for expected rpc port to be -1."); } checkTimeline(appIdRef.get(), defaultFlow, haveDomain, appReportRef.get()); } @@ -453,7 +453,7 @@ protected String[] createArgsWithPostFix(int index, String... args) { } protected String generateAppName(String methodName) { - return generateAppName(methodName,null); + return generateAppName(methodName, null); } protected String generateAppName(String methodName, String postFix) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV10.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV10.java index cc4e784528bb3..5d65219b7dfe3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV10.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV10.java @@ -99,7 +99,7 @@ public void testDSShellWithDomain(TestInfo testInfo) throws Exception { @Test public void testDSShellWithoutDomain(TestInfo testInfo) throws Exception { - baseTestDSShell(getMethodName(testInfo),false); + baseTestDSShell(getMethodName(testInfo), false); } @Test @@ -661,8 +661,8 @@ private int verifyContainerLog(int containerNum, } } else if (output.getName().trim().equals("stdout")) { if (!Shell.WINDOWS) { - assertEquals( - expectedContent.get(numOffline), sCurrentLine.trim(), "The current is" + sCurrentLine); + assertEquals(expectedContent.get(numOffline), sCurrentLine.trim(), + "The current is" + sCurrentLine); numOffline++; } else { stdOutContent.add(sCurrentLine.trim()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV20.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV20.java index 0e3b963732aa6..6ef26ed1cce72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV20.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSTimelineV20.java @@ -445,20 +445,19 @@ private void verifyEntityForTimeline(File entityFile, String expectedEvent, getTimelineRecordFromJSON(entityLine, TimelineEntity.class); TimelineEvent event = entity.getEvents().pollFirst(); assertNotNull(event); - assertTrue( - event.getInfo().containsKey(ApplicationMaster.DIAGNOSTICS), "diagnostics"); + assertTrue(event.getInfo().containsKey(ApplicationMaster.DIAGNOSTICS), + "diagnostics"); } if (checkIdPrefix) { TimelineEntity entity = FileSystemTimelineReaderImpl. getTimelineRecordFromJSON(entityLine, TimelineEntity.class); - assertTrue( - entity.getIdPrefix() > 0, "Entity ID prefix expected to be > 0"); + assertTrue(entity.getIdPrefix() > 0, + "Entity ID prefix expected to be > 0"); if (idPrefix == -1) { idPrefix = entity.getIdPrefix(); } else { - assertEquals( - idPrefix, entity.getIdPrefix(), "Entity ID prefix should be same across each publish of " - + "same entity"); + assertEquals(idPrefix, entity.getIdPrefix(), + "Entity ID prefix should be same across each publish of same entity"); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java index 467df6cdbcd1a..7ebc747ed2ea1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSWithMultipleNodeManager.java @@ -25,15 +25,15 @@ import java.util.Iterator; import java.util.Set; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import org.junit.jupiter.api.*; -import org.junit.Rule; -import org.junit.rules.TestName; -import org.junit.rules.Timeout; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,13 +56,16 @@ import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Test for Distributed Shell With Multiple Node Managers. * Parameter 0 tests with Single Node Placement and * parameter 1 tests with Multiple Node Placement. */ -@RunWith(value = Parameterized.class) + +@Timeout(160) public class TestDSWithMultipleNodeManager { private static final Logger LOG = LoggerFactory.getLogger(TestDSWithMultipleNodeManager.class); @@ -70,21 +73,17 @@ public class TestDSWithMultipleNodeManager { private static final int NUM_NMS = 2; private static final String POLICY_CLASS_NAME = ResourceUsageMultiNodeLookupPolicy.class.getName(); - private final Boolean multiNodePlacementEnabled; - @Rule - public TestName name = new TestName(); - @Rule - public Timeout globalTimeout = - new Timeout(DistributedShellBaseTest.TEST_TIME_OUT, - TimeUnit.MILLISECONDS); + private Boolean multiNodePlacementEnabled; + private DistributedShellBaseTest distShellTest; private Client dsClient; - public TestDSWithMultipleNodeManager(Boolean multiNodePlacementEnabled) { - this.multiNodePlacementEnabled = multiNodePlacementEnabled; + public void initTestDSWithMultipleNodeManager( + Boolean pMultiNodePlacementEnabled, TestInfo testInfo) throws Exception { + this.multiNodePlacementEnabled = pMultiNodePlacementEnabled; + setup(testInfo); } - @Parameterized.Parameters public static Collection getParams() { return Arrays.asList(false, true); } @@ -121,7 +120,6 @@ public static void tearDownUnitTests() throws Exception { TestDSTimelineV10.tearDownUnitTests(); } - @BeforeEach public void setup(TestInfo testInfo) throws Exception { distShellTest = new TestDSTimelineV10(); distShellTest.setupInternal(NUM_NMS, @@ -171,8 +169,11 @@ private void initializeNodeLabels() throws IOException { labelsMgr.addLabelsToNode(ImmutableMap.of(nodeIds[1], labels)); } - @Test - public void testDSShellWithNodeLabelExpression() throws Exception { + @MethodSource("getParams") + @ParameterizedTest + public void testDSShellWithNodeLabelExpression(Boolean pMultiNodePlacementEnabled, + TestInfo testInfo) throws Exception { + initTestDSWithMultipleNodeManager(pMultiNodePlacementEnabled, testInfo); NMContainerMonitor containerMonitorRunner = null; initializeNodeLabels(); @@ -183,7 +184,8 @@ public void testDSShellWithNodeLabelExpression() throws Exception { // Submit a job which will sleep for 60 sec String[] args = - DistributedShellBaseTest.createArguments(() -> generateAppName(), + DistributedShellBaseTest.createArguments(() -> + generateAppName(distShellTest.getMethodName(testInfo)), "--num_containers", "4", "--shell_command", @@ -206,7 +208,7 @@ public void testDSShellWithNodeLabelExpression() throws Exception { dsClient = new Client( new Configuration(distShellTest.getYarnClusterConfiguration())); - Assertions.assertTrue(dsClient.init(args)); + assertTrue(dsClient.init(args)); LOG.info("Running DS Client"); boolean result = dsClient.run(); LOG.info("Client run completed. Result={}", result); @@ -217,9 +219,9 @@ public void testDSShellWithNodeLabelExpression() throws Exception { int[] maxRunningContainersOnNMs = containerMonitorRunner.getMaxRunningContainersReport(); // Check no container allocated on NM[0] - Assertions.assertEquals(0, maxRunningContainersOnNMs[0]); + assertEquals(0, maxRunningContainersOnNMs[0]); // Check there are some containers allocated on NM[1] - Assertions.assertTrue(maxRunningContainersOnNMs[1] > 0); + assertTrue(maxRunningContainersOnNMs[1] > 0); } finally { if (containerMonitorRunner != null) { containerMonitorRunner.stopMonitoring(); @@ -228,12 +230,16 @@ public void testDSShellWithNodeLabelExpression() throws Exception { } } - @Test - public void testDistributedShellWithPlacementConstraint() + @MethodSource("getParams") + @ParameterizedTest + public void testDistributedShellWithPlacementConstraint( + Boolean pMultiNodePlacementEnabled, TestInfo testInfo) throws Exception { + initTestDSWithMultipleNodeManager(pMultiNodePlacementEnabled, testInfo); NMContainerMonitor containerMonitorRunner = null; String[] args = - DistributedShellBaseTest.createArguments(() -> generateAppName(), + DistributedShellBaseTest.createArguments(() -> + generateAppName(distShellTest.getMethodName(testInfo)), "1", "--shell_command", DistributedShellBaseTest.getSleepCommand(15), @@ -248,7 +254,7 @@ public void testDistributedShellWithPlacementConstraint() dsClient = new Client( new Configuration(distShellTest.getYarnClusterConfiguration())); - Assertions.assertTrue(dsClient.init(args)); + assertTrue(dsClient.init(args)); LOG.info("Running DS Client"); boolean result = dsClient.run(); LOG.info("Client run completed. Result={}", result); @@ -271,8 +277,8 @@ public void testDistributedShellWithPlacementConstraint() int[] maxRunningContainersOnNMs = containerMonitorRunner.getMaxRunningContainersReport(); - Assertions.assertEquals(expectedNMsCount[0], maxRunningContainersOnNMs[0]); - Assertions.assertEquals(expectedNMsCount[1], maxRunningContainersOnNMs[1]); + assertEquals(expectedNMsCount[0], maxRunningContainersOnNMs[0]); + assertEquals(expectedNMsCount[1], maxRunningContainersOnNMs[1]); } finally { if (containerMonitorRunner != null) { containerMonitorRunner.stopMonitoring(); @@ -281,9 +287,12 @@ public void testDistributedShellWithPlacementConstraint() } } - @Test - public void testDistributedShellWithAllocationTagNamespace() + @MethodSource("getParams") + @ParameterizedTest + public void testDistributedShellWithAllocationTagNamespace( + Boolean pMultiNodePlacementEnabled, TestInfo testInfo) throws Exception { + initTestDSWithMultipleNodeManager(pMultiNodePlacementEnabled, testInfo); NMContainerMonitor containerMonitorRunner = null; Client clientB = null; YarnClient yarnClient = null; @@ -334,7 +343,7 @@ public void testDistributedShellWithAllocationTagNamespace() getNodeId(); NodeId nodeB = distShellTest.getNodeManager(1).getNMContext(). getNodeId(); - Assertions.assertEquals(2, (expectedNMCounts[0] + expectedNMCounts[1])); + assertEquals(2, (expectedNMCounts[0] + expectedNMCounts[1])); if (expectedNMCounts[0] != expectedNMCounts[1]) { taskContainerNodeIdA = masterContainerNodeIdARef.get(); } else { @@ -346,7 +355,7 @@ public void testDistributedShellWithAllocationTagNamespace() new Client( new Configuration(distShellTest.getYarnClusterConfiguration())); clientB.init(argsB); - Assertions.assertTrue(clientB.run()); + assertTrue(clientB.run()); containerMonitorRunner.stopMonitoring(); apps = distShellTest.getResourceManager().getRMContext().getRMApps(); Iterator it = apps.values().iterator(); @@ -374,8 +383,8 @@ public void testDistributedShellWithAllocationTagNamespace() } int[] maxRunningContainersOnNMs = containerMonitorRunner.getMaxRunningContainersReport(); - Assertions.assertEquals(expectedNMCounts[0], maxRunningContainersOnNMs[0]); - Assertions.assertEquals(expectedNMCounts[1], maxRunningContainersOnNMs[1]); + assertEquals(expectedNMCounts[0], maxRunningContainersOnNMs[0]); + assertEquals(expectedNMCounts[1], maxRunningContainersOnNMs[1]); try { yarnClient = YarnClient.createYarnClient(); @@ -401,12 +410,12 @@ public void testDistributedShellWithAllocationTagNamespace() } } - protected String generateAppName() { - return generateAppName(null); + protected String generateAppName(String name) { + return generateAppName(name, null); } - protected String generateAppName(String postFix) { - return name.getMethodName().replaceFirst("test", "") + protected String generateAppName(String name, String postFix) { + return name.replaceFirst("test", "") .concat(postFix == null ? "" : "-" + postFix); } From 024de52333e61f2046bd1c9127f1cf382a253937 Mon Sep 17 00:00:00 2001 From: fanshilun Date: Wed, 12 Feb 2025 01:21:59 +0800 Subject: [PATCH 4/4] YARN-11760. Fix CheckStyle. --- .../applications/distributedshell/DistributedShellBaseTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java index b77b50a0592ca..fd9e2db61119f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java @@ -309,7 +309,7 @@ protected Client setAndGetDSClient(String appMasterMainClass, protected void baseTestDSShell(String methodName, boolean haveDomain, boolean defaultFlow) throws Exception { String[] baseArgs = createArgumentsWithAppName( - methodName, + methodName, "--num_containers", "2", "--shell_command",