Skip to content

Commit

Permalink
YARN-11760. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-applicat…
Browse files Browse the repository at this point in the history 
…ions-distributedshell. (#7373)

Co-authored-by: Chris Nauroth <[email protected]>
Reviewed-by: Chris Nauroth <[email protected]>
Signed-off-by: Shilun Fan <[email protected]>
  • Loading branch information
@slfan1989 @cnauroth
slfan1989 and cnauroth authored Feb 12, 2025
1 parent b24996c commit b4168c3
Show file tree
Hide file tree
Showing 7 changed files with 327 additions and 286 deletions.
110 changes: 57 additions & 53 deletions ...t/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,21 +28,18 @@
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;

import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.rules.TemporaryFolder;
import org.junit.rules.TestName;
import org.junit.rules.Timeout;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -75,9 +72,15 @@
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;

/**
* Base class for testing DistributedShell features.
*/
@Timeout(160)
public abstract class DistributedShellBaseTest {
protected static final int MIN_ALLOCATION_MB = 128;
protected static final int NUM_DATA_NODES = 1;
Expand Down Expand Up @@ -105,19 +108,15 @@ public abstract class DistributedShellBaseTest {
private static MiniYARNCluster yarnCluster = null;
private static String yarnSiteBackupPath = null;
private static String yarnSitePath = null;
@Rule
public Timeout globalTimeout = new Timeout(TEST_TIME_OUT,
TimeUnit.MILLISECONDS);
@Rule
public TemporaryFolder tmpFolder = new TemporaryFolder();
@Rule
public TestName name = new TestName();
@TempDir
private java.nio.file.Path tmpFolder;

private Client dsClient;
private YarnConfiguration conf = null;
// location of the filesystem timeline writer for timeline service v.2
private String timelineV2StorageDir = null;

@BeforeClass
@BeforeAll
public static void setupUnitTests() throws Exception {
URL url = Thread.currentThread().getContextClassLoader().getResource(
"yarn-site.xml");
Expand All @@ -134,7 +133,7 @@ public static void setupUnitTests() throws Exception {
StandardCopyOption.REPLACE_EXISTING);
}

@AfterClass
@AfterAll
public static void tearDownUnitTests() throws Exception {
// shutdown the clusters.
shutdownYarnCluster();
Expand Down Expand Up @@ -212,15 +211,15 @@ public String getTimelineV2StorageDir() {
}

public void setTimelineV2StorageDir() throws Exception {
timelineV2StorageDir = tmpFolder.newFolder().getAbsolutePath();
timelineV2StorageDir = tmpFolder.toFile().getAbsolutePath();
}

@Before
public void setup() throws Exception {
setupInternal(NUM_NMS, new YarnConfiguration());
@BeforeEach
public void setup(TestInfo testInfo) throws Exception {
setupInternal(NUM_NMS, new YarnConfiguration(), getMethodName(testInfo));
}

@After
@AfterEach
public void tearDown() throws IOException {
cleanUpDFSClient();
FileContext fsContext = FileContext.getLocalFSFileContext();
Expand All @@ -232,8 +231,8 @@ public void tearDown() throws IOException {
shutdownHdfsCluster();
}

protected String[] createArgumentsWithAppName(String... args) {
return createArguments(() -> generateAppName(), args);
protected String[] createArgumentsWithAppName(String methodName, String... args) {
return createArguments(() -> generateAppName(methodName), args);
}

protected void waitForContainersLaunch(YarnClient client, int nContainers,
Expand Down Expand Up @@ -307,9 +306,10 @@ protected Client setAndGetDSClient(String appMasterMainClass,
return dsClient;
}

protected void baseTestDSShell(boolean haveDomain, boolean defaultFlow)
protected void baseTestDSShell(String methodName, boolean haveDomain, boolean defaultFlow)
throws Exception {
String[] baseArgs = createArgumentsWithAppName(
methodName,
"--num_containers",
"2",
"--shell_command",
Expand All @@ -329,7 +329,7 @@ protected void baseTestDSShell(boolean haveDomain, boolean defaultFlow)
YarnClient yarnClient;
dsClient = setAndGetDSClient(new Configuration(yarnCluster.getConfig()));
boolean initSuccess = dsClient.init(args);
Assert.assertTrue(initSuccess);
assertTrue(initSuccess);
LOG.info("Running DS Client");
final AtomicBoolean result = new AtomicBoolean(false);
Thread t = new Thread(() -> {
Expand Down Expand Up @@ -379,17 +379,17 @@ protected void baseTestDSShell(boolean haveDomain, boolean defaultFlow)
t.join();
if (waitResult.get() == 2) {
// Exception was raised
Assert.fail("Exception in getting application report. Failed");
fail("Exception in getting application report. Failed");
}
if (waitResult.get() == 1) {
Assert.assertEquals("Failed waiting for expected rpc port to be -1.",
-1, appReportRef.get().getRpcPort());
assertEquals(-1, appReportRef.get().getRpcPort(),
"Failed waiting for expected rpc port to be -1.");
}
checkTimeline(appIdRef.get(), defaultFlow, haveDomain, appReportRef.get());
}

protected void baseTestDSShell(boolean haveDomain) throws Exception {
baseTestDSShell(haveDomain, true);
protected void baseTestDSShell(String methodName, boolean haveDomain) throws Exception {
baseTestDSShell(methodName, haveDomain, true);
}

protected void checkTimeline(ApplicationId appId,
Expand All @@ -399,22 +399,22 @@ protected void checkTimeline(ApplicationId appId,
if (haveDomain) {
domain = yarnCluster.getApplicationHistoryServer()
.getTimelineStore().getDomain("TEST_DOMAIN");
Assert.assertNotNull(domain);
Assert.assertEquals("reader_user reader_group", domain.getReaders());
Assert.assertEquals("writer_user writer_group", domain.getWriters());
assertNotNull(domain);
assertEquals("reader_user reader_group", domain.getReaders());
assertEquals("writer_user writer_group", domain.getWriters());
}
TimelineEntities entitiesAttempts = yarnCluster
.getApplicationHistoryServer()
.getTimelineStore()
.getEntities(ApplicationMaster.DSEntity.DS_APP_ATTEMPT.toString(),
null, null, null, null, null, null, null, null, null);
Assert.assertNotNull(entitiesAttempts);
Assert.assertEquals(1, entitiesAttempts.getEntities().size());
Assert.assertEquals(2, entitiesAttempts.getEntities().get(0).getEvents()
assertNotNull(entitiesAttempts);
assertEquals(1, entitiesAttempts.getEntities().size());
assertEquals(2, entitiesAttempts.getEntities().get(0).getEvents()
.size());
Assert.assertEquals(entitiesAttempts.getEntities().get(0).getEntityType(),
assertEquals(entitiesAttempts.getEntities().get(0).getEntityType(),
ApplicationMaster.DSEntity.DS_APP_ATTEMPT.toString());
Assert.assertEquals(haveDomain ? domain.getId() : "DEFAULT",
assertEquals(haveDomain ? domain.getId() : "DEFAULT",
entitiesAttempts.getEntities().get(0).getDomainId());
String currAttemptEntityId =
entitiesAttempts.getEntities().get(0).getEntityId();
Expand All @@ -428,19 +428,19 @@ protected void checkTimeline(ApplicationId appId,
.getTimelineStore()
.getEntities(ApplicationMaster.DSEntity.DS_CONTAINER.toString(), null,
null, null, null, null, primaryFilter, null, null, null);
Assert.assertNotNull(entities);
Assert.assertEquals(2, entities.getEntities().size());
Assert.assertEquals(entities.getEntities().get(0).getEntityType(),
assertNotNull(entities);
assertEquals(2, entities.getEntities().size());
assertEquals(entities.getEntities().get(0).getEntityType(),
ApplicationMaster.DSEntity.DS_CONTAINER.toString());

String entityId = entities.getEntities().get(0).getEntityId();
TimelineEntity entity =
yarnCluster.getApplicationHistoryServer().getTimelineStore()
.getEntity(entityId,
ApplicationMaster.DSEntity.DS_CONTAINER.toString(), null);
Assert.assertNotNull(entity);
Assert.assertEquals(entityId, entity.getEntityId());
Assert.assertEquals(haveDomain ? domain.getId() : "DEFAULT",
assertNotNull(entity);
assertEquals(entityId, entity.getEntityId());
assertEquals(haveDomain ? domain.getId() : "DEFAULT",
entities.getEntities().get(0).getDomainId());
}

Expand All @@ -452,12 +452,12 @@ protected String[] createArgsWithPostFix(int index, String... args) {
return res;
}

protected String generateAppName() {
return generateAppName(null);
protected String generateAppName(String methodName) {
return generateAppName(methodName, null);
}

protected String generateAppName(String postFix) {
return name.getMethodName().replaceFirst("test", "")
protected String generateAppName(String methodName, String postFix) {
return methodName.replaceFirst("test", "")
.concat(postFix == null ? "" : "-" + postFix);
}

Expand Down Expand Up @@ -501,9 +501,9 @@ protected void setUpYarnCluster(int numNodeManagers,
}

protected void setupInternal(int numNodeManagers,
YarnConfiguration yarnConfig) throws Exception {
YarnConfiguration yarnConfig, String methodName) throws Exception {
LOG.info("========== Setting UP UnitTest {}#{} ==========",
getClass().getCanonicalName(), name.getMethodName());
getClass().getCanonicalName(), methodName);
LOG.info("Starting up YARN cluster. Timeline version {}",
getTimelineVersion());
conf = yarnConfig;
Expand Down Expand Up @@ -604,4 +604,8 @@ private void waitForNMsToRegister() throws Exception {
protected MiniDFSCluster getHDFSCluster() {
return hdfsCluster;
}

public String getMethodName(TestInfo testInfo) {
return testInfo.getTestMethod().get().getName();
}
}
4 changes: 2 additions & 2 deletions ...dshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@

package org.apache.hadoop.yarn.applications.distributedshell;

import org.junit.Test;
import org.junit.jupiter.api.Test;

import java.util.HashMap;
import java.util.Map;

import static org.junit.Assert.assertEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;

public class TestClient {
@Test
Expand Down
68 changes: 34 additions & 34 deletions ...l/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,7 @@
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentMatchers;
import org.mockito.Mockito;

Expand All @@ -48,6 +47,12 @@
import java.util.Arrays;
import java.util.List;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;

/**
* A bunch of tests to make sure that the container allocations
* and releases occur correctly.
Expand Down Expand Up @@ -105,12 +110,12 @@ public void testDSAppMasterAllocateHandler() throws Exception {

// first allocate a single container, everything should be fine
handler.onContainersAllocated(containers);
Assert.assertEquals("Wrong container allocation count", 1,
master.getAllocatedContainers());
Assert.assertEquals("Incorrect number of threads launched", 1,
master.threadsLaunched);
Assert.assertEquals("Incorrect YARN Shell IDs",
Arrays.asList("1"), master.yarnShellIds);
assertEquals(1, master.getAllocatedContainers(),
"Wrong container allocation count");
assertEquals(1, master.threadsLaunched,
"Incorrect number of threads launched");
assertEquals(Arrays.asList("1"), master.yarnShellIds,
"Incorrect YARN Shell IDs");

// now send 3 extra containers
containers.clear();
Expand All @@ -121,14 +126,12 @@ public void testDSAppMasterAllocateHandler() throws Exception {
ContainerId id4 = BuilderUtils.newContainerId(1, 1, 1, 4);
containers.add(generateContainer(id4));
handler.onContainersAllocated(containers);
Assert.assertEquals("Wrong final container allocation count", 2,
master.getAllocatedContainers());

Assert.assertEquals("Incorrect number of threads launched", 2,
master.threadsLaunched);

Assert.assertEquals("Incorrect YARN Shell IDs",
Arrays.asList("1", "2"), master.yarnShellIds);
assertEquals(2, master.getAllocatedContainers(),
"Wrong final container allocation count");
assertEquals(2, master.threadsLaunched,
"Incorrect number of threads launched");
assertEquals(Arrays.asList("1", "2"), master.yarnShellIds,
"Incorrect YARN Shell IDs");
// make sure we handle completion events correctly
List<ContainerStatus> status = new ArrayList<>();
status.add(generateContainerStatus(id1, ContainerExitStatus.SUCCESS));
Expand All @@ -137,25 +140,22 @@ public void testDSAppMasterAllocateHandler() throws Exception {
status.add(generateContainerStatus(id4, ContainerExitStatus.ABORTED));
handler.onContainersCompleted(status);

Assert.assertEquals("Unexpected number of completed containers",
targetContainers, master.getNumCompletedContainers());
Assert.assertTrue("Master didn't finish containers as expected",
master.getDone());
assertEquals(targetContainers, master.getNumCompletedContainers(),
"Unexpected number of completed containers");
assertTrue(master.getDone(), "Master didn't finish containers as expected");

// test for events from containers we know nothing about
// these events should be ignored
status = new ArrayList<>();
ContainerId id5 = BuilderUtils.newContainerId(1, 1, 1, 5);
status.add(generateContainerStatus(id5, ContainerExitStatus.ABORTED));
Assert.assertEquals("Unexpected number of completed containers",
targetContainers, master.getNumCompletedContainers());
Assert.assertTrue("Master didn't finish containers as expected",
master.getDone());
assertEquals(targetContainers, master.getNumCompletedContainers(),
"Unexpected number of completed containers");
assertTrue(master.getDone(), "Master didn't finish containers as expected");
status.add(generateContainerStatus(id5, ContainerExitStatus.SUCCESS));
Assert.assertEquals("Unexpected number of completed containers",
targetContainers, master.getNumCompletedContainers());
Assert.assertTrue("Master didn't finish containers as expected",
master.getDone());
assertEquals(targetContainers, master.getNumCompletedContainers(),
"Unexpected number of completed containers");
assertTrue(master.getDone(), "Master didn't finish containers as expected");
}

private Container generateContainer(ContainerId cid) {
Expand Down Expand Up @@ -200,15 +200,15 @@ private void runTimelineClientInDSAppMaster(boolean v1Enabled,
private void validateAppMasterTimelineService(boolean v1Enabled,
boolean v2Enabled, ApplicationMaster appMaster) {
if (v1Enabled) {
Assert.assertEquals(appMaster.appSubmitterUgi,
((TimelineClientImpl)appMaster.timelineClient).getUgi());
assertEquals(appMaster.appSubmitterUgi,
((TimelineClientImpl) appMaster.timelineClient).getUgi());
} else {
Assert.assertNull(appMaster.timelineClient);
assertNull(appMaster.timelineClient);
}
if (v2Enabled) {
Assert.assertNotNull(appMaster.timelineV2Client);
assertNotNull(appMaster.timelineV2Client);
} else {
Assert.assertNull(appMaster.timelineV2Client);
assertNull(appMaster.timelineV2Client);
}
}

Expand All @@ -227,7 +227,7 @@ private ApplicationMaster createAppMasterWithStartedTimelineService(
private Configuration getTimelineServiceConf(boolean v1Enabled,
boolean v2Enabled) {
Configuration conf = new YarnConfiguration(new Configuration(false));
Assert.assertFalse(YarnConfiguration.timelineServiceEnabled(conf));
assertFalse(YarnConfiguration.timelineServiceEnabled(conf));

if (v1Enabled || v2Enabled) {
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
Expand Down
Loading

0 comments on commit b4168c3

Please sign in to comment.