Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

YARN-11760. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-applications-distributedshell. #7373

Merged
merged 4 commits into from
Feb 12, 2025
Merged

YARN-11760. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-applications-distributedshell. #7373

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b5ddf3b
YARN-11760. Upgrade JUnit from 4 to 5 in hadoop-yarn-applications-dis…
Feb 9, 2025
67d2548
YARN-11760. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-applicat…
Feb 10, 2025
25979d4
HDFS-11760. Fix Junit Test & CheckStyle.
Feb 10, 2025
024de52
YARN-11760. Fix CheckStyle.
Feb 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 57 additions & 53 deletions ...t/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellBaseTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,21 +28,18 @@
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;

import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.rules.TemporaryFolder;
import org.junit.rules.TestName;
import org.junit.rules.Timeout;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -75,9 +72,15 @@
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;

/**
* Base class for testing DistributedShell features.
*/
@Timeout(160)
public abstract class DistributedShellBaseTest {
protected static final int MIN_ALLOCATION_MB = 128;
protected static final int NUM_DATA_NODES = 1;
Expand Down Expand Up @@ -105,19 +108,15 @@ public abstract class DistributedShellBaseTest {
private static MiniYARNCluster yarnCluster = null;
private static String yarnSiteBackupPath = null;
private static String yarnSitePath = null;
@Rule
public Timeout globalTimeout = new Timeout(TEST_TIME_OUT,
TimeUnit.MILLISECONDS);
@Rule
public TemporaryFolder tmpFolder = new TemporaryFolder();
@Rule
public TestName name = new TestName();
@TempDir
private java.nio.file.Path tmpFolder;

private Client dsClient;
private YarnConfiguration conf = null;
// location of the filesystem timeline writer for timeline service v.2
private String timelineV2StorageDir = null;

@BeforeClass
@BeforeAll
public static void setupUnitTests() throws Exception {
URL url = Thread.currentThread().getContextClassLoader().getResource(
"yarn-site.xml");
Expand All @@ -134,7 +133,7 @@ public static void setupUnitTests() throws Exception {
StandardCopyOption.REPLACE_EXISTING);
}

@AfterClass
@AfterAll
public static void tearDownUnitTests() throws Exception {
// shutdown the clusters.
shutdownYarnCluster();
Expand Down Expand Up @@ -212,15 +211,15 @@ public String getTimelineV2StorageDir() {
}

public void setTimelineV2StorageDir() throws Exception {
timelineV2StorageDir = tmpFolder.newFolder().getAbsolutePath();
timelineV2StorageDir = tmpFolder.toFile().getAbsolutePath();
}

@Before
public void setup() throws Exception {
setupInternal(NUM_NMS, new YarnConfiguration());
@BeforeEach
public void setup(TestInfo testInfo) throws Exception {
setupInternal(NUM_NMS, new YarnConfiguration(), getMethodName(testInfo));
}

@After
@AfterEach
public void tearDown() throws IOException {
cleanUpDFSClient();
FileContext fsContext = FileContext.getLocalFSFileContext();
Expand All @@ -232,8 +231,8 @@ public void tearDown() throws IOException {
shutdownHdfsCluster();
}

protected String[] createArgumentsWithAppName(String... args) {
return createArguments(() -> generateAppName(), args);
protected String[] createArgumentsWithAppName(String methodName, String... args) {
return createArguments(() -> generateAppName(methodName), args);
}

protected void waitForContainersLaunch(YarnClient client, int nContainers,
Expand Down Expand Up @@ -307,9 +306,10 @@ protected Client setAndGetDSClient(String appMasterMainClass,
return dsClient;
}

protected void baseTestDSShell(boolean haveDomain, boolean defaultFlow)
protected void baseTestDSShell(String methodName, boolean haveDomain, boolean defaultFlow)
throws Exception {
String[] baseArgs = createArgumentsWithAppName(
methodName,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there an extra space in the indentation? If so, can you please remove it before committing?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for your message. I will fix this issue.

"--num_containers",
"2",
"--shell_command",
Expand All @@ -329,7 +329,7 @@ protected void baseTestDSShell(boolean haveDomain, boolean defaultFlow)
YarnClient yarnClient;
dsClient = setAndGetDSClient(new Configuration(yarnCluster.getConfig()));
boolean initSuccess = dsClient.init(args);
Assert.assertTrue(initSuccess);
assertTrue(initSuccess);
LOG.info("Running DS Client");
final AtomicBoolean result = new AtomicBoolean(false);
Thread t = new Thread(() -> {
Expand Down Expand Up @@ -379,17 +379,17 @@ protected void baseTestDSShell(boolean haveDomain, boolean defaultFlow)
t.join();
if (waitResult.get() == 2) {
// Exception was raised
Assert.fail("Exception in getting application report. Failed");
fail("Exception in getting application report. Failed");
}
if (waitResult.get() == 1) {
Assert.assertEquals("Failed waiting for expected rpc port to be -1.",
-1, appReportRef.get().getRpcPort());
assertEquals(-1, appReportRef.get().getRpcPort(),
"Failed waiting for expected rpc port to be -1.");
}
checkTimeline(appIdRef.get(), defaultFlow, haveDomain, appReportRef.get());
}

protected void baseTestDSShell(boolean haveDomain) throws Exception {
baseTestDSShell(haveDomain, true);
protected void baseTestDSShell(String methodName, boolean haveDomain) throws Exception {
baseTestDSShell(methodName, haveDomain, true);
}

protected void checkTimeline(ApplicationId appId,
Expand All @@ -399,22 +399,22 @@ protected void checkTimeline(ApplicationId appId,
if (haveDomain) {
domain = yarnCluster.getApplicationHistoryServer()
.getTimelineStore().getDomain("TEST_DOMAIN");
Assert.assertNotNull(domain);
Assert.assertEquals("reader_user reader_group", domain.getReaders());
Assert.assertEquals("writer_user writer_group", domain.getWriters());
assertNotNull(domain);
assertEquals("reader_user reader_group", domain.getReaders());
assertEquals("writer_user writer_group", domain.getWriters());
}
TimelineEntities entitiesAttempts = yarnCluster
.getApplicationHistoryServer()
.getTimelineStore()
.getEntities(ApplicationMaster.DSEntity.DS_APP_ATTEMPT.toString(),
null, null, null, null, null, null, null, null, null);
Assert.assertNotNull(entitiesAttempts);
Assert.assertEquals(1, entitiesAttempts.getEntities().size());
Assert.assertEquals(2, entitiesAttempts.getEntities().get(0).getEvents()
assertNotNull(entitiesAttempts);
assertEquals(1, entitiesAttempts.getEntities().size());
assertEquals(2, entitiesAttempts.getEntities().get(0).getEvents()
.size());
Assert.assertEquals(entitiesAttempts.getEntities().get(0).getEntityType(),
assertEquals(entitiesAttempts.getEntities().get(0).getEntityType(),
ApplicationMaster.DSEntity.DS_APP_ATTEMPT.toString());
Assert.assertEquals(haveDomain ? domain.getId() : "DEFAULT",
assertEquals(haveDomain ? domain.getId() : "DEFAULT",
entitiesAttempts.getEntities().get(0).getDomainId());
String currAttemptEntityId =
entitiesAttempts.getEntities().get(0).getEntityId();
Expand All @@ -428,19 +428,19 @@ protected void checkTimeline(ApplicationId appId,
.getTimelineStore()
.getEntities(ApplicationMaster.DSEntity.DS_CONTAINER.toString(), null,
null, null, null, null, primaryFilter, null, null, null);
Assert.assertNotNull(entities);
Assert.assertEquals(2, entities.getEntities().size());
Assert.assertEquals(entities.getEntities().get(0).getEntityType(),
assertNotNull(entities);
assertEquals(2, entities.getEntities().size());
assertEquals(entities.getEntities().get(0).getEntityType(),
ApplicationMaster.DSEntity.DS_CONTAINER.toString());

String entityId = entities.getEntities().get(0).getEntityId();
TimelineEntity entity =
yarnCluster.getApplicationHistoryServer().getTimelineStore()
.getEntity(entityId,
ApplicationMaster.DSEntity.DS_CONTAINER.toString(), null);
Assert.assertNotNull(entity);
Assert.assertEquals(entityId, entity.getEntityId());
Assert.assertEquals(haveDomain ? domain.getId() : "DEFAULT",
assertNotNull(entity);
assertEquals(entityId, entity.getEntityId());
assertEquals(haveDomain ? domain.getId() : "DEFAULT",
entities.getEntities().get(0).getDomainId());
}

Expand All @@ -452,12 +452,12 @@ protected String[] createArgsWithPostFix(int index, String... args) {
return res;
}

protected String generateAppName() {
return generateAppName(null);
protected String generateAppName(String methodName) {
return generateAppName(methodName, null);
}

protected String generateAppName(String postFix) {
return name.getMethodName().replaceFirst("test", "")
protected String generateAppName(String methodName, String postFix) {
return methodName.replaceFirst("test", "")
.concat(postFix == null ? "" : "-" + postFix);
}

Expand Down Expand Up @@ -501,9 +501,9 @@ protected void setUpYarnCluster(int numNodeManagers,
}

protected void setupInternal(int numNodeManagers,
YarnConfiguration yarnConfig) throws Exception {
YarnConfiguration yarnConfig, String methodName) throws Exception {
LOG.info("========== Setting UP UnitTest {}#{} ==========",
getClass().getCanonicalName(), name.getMethodName());
getClass().getCanonicalName(), methodName);
LOG.info("Starting up YARN cluster. Timeline version {}",
getTimelineVersion());
conf = yarnConfig;
Expand Down Expand Up @@ -604,4 +604,8 @@ private void waitForNMsToRegister() throws Exception {
protected MiniDFSCluster getHDFSCluster() {
return hdfsCluster;
}

public String getMethodName(TestInfo testInfo) {
return testInfo.getTestMethod().get().getName();
}
}
4 changes: 2 additions & 2 deletions ...dshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@

package org.apache.hadoop.yarn.applications.distributedshell;

import org.junit.Test;
import org.junit.jupiter.api.Test;

import java.util.HashMap;
import java.util.Map;

import static org.junit.Assert.assertEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;

public class TestClient {
@Test
Expand Down
68 changes: 34 additions & 34 deletions ...l/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,7 @@
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentMatchers;
import org.mockito.Mockito;

Expand All @@ -48,6 +47,12 @@
import java.util.Arrays;
import java.util.List;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;

/**
* A bunch of tests to make sure that the container allocations
* and releases occur correctly.
Expand Down Expand Up @@ -105,12 +110,12 @@ public void testDSAppMasterAllocateHandler() throws Exception {

// first allocate a single container, everything should be fine
handler.onContainersAllocated(containers);
Assert.assertEquals("Wrong container allocation count", 1,
master.getAllocatedContainers());
Assert.assertEquals("Incorrect number of threads launched", 1,
master.threadsLaunched);
Assert.assertEquals("Incorrect YARN Shell IDs",
Arrays.asList("1"), master.yarnShellIds);
assertEquals(1, master.getAllocatedContainers(),
"Wrong container allocation count");
assertEquals(1, master.threadsLaunched,
"Incorrect number of threads launched");
assertEquals(Arrays.asList("1"), master.yarnShellIds,
"Incorrect YARN Shell IDs");

// now send 3 extra containers
containers.clear();
Expand All @@ -121,14 +126,12 @@ public void testDSAppMasterAllocateHandler() throws Exception {
ContainerId id4 = BuilderUtils.newContainerId(1, 1, 1, 4);
containers.add(generateContainer(id4));
handler.onContainersAllocated(containers);
Assert.assertEquals("Wrong final container allocation count", 2,
master.getAllocatedContainers());

Assert.assertEquals("Incorrect number of threads launched", 2,
master.threadsLaunched);

Assert.assertEquals("Incorrect YARN Shell IDs",
Arrays.asList("1", "2"), master.yarnShellIds);
assertEquals(2, master.getAllocatedContainers(),
"Wrong final container allocation count");
assertEquals(2, master.threadsLaunched,
"Incorrect number of threads launched");
assertEquals(Arrays.asList("1", "2"), master.yarnShellIds,
"Incorrect YARN Shell IDs");
// make sure we handle completion events correctly
List<ContainerStatus> status = new ArrayList<>();
status.add(generateContainerStatus(id1, ContainerExitStatus.SUCCESS));
Expand All @@ -137,25 +140,22 @@ public void testDSAppMasterAllocateHandler() throws Exception {
status.add(generateContainerStatus(id4, ContainerExitStatus.ABORTED));
handler.onContainersCompleted(status);

Assert.assertEquals("Unexpected number of completed containers",
targetContainers, master.getNumCompletedContainers());
Assert.assertTrue("Master didn't finish containers as expected",
master.getDone());
assertEquals(targetContainers, master.getNumCompletedContainers(),
"Unexpected number of completed containers");
assertTrue(master.getDone(), "Master didn't finish containers as expected");

// test for events from containers we know nothing about
// these events should be ignored
status = new ArrayList<>();
ContainerId id5 = BuilderUtils.newContainerId(1, 1, 1, 5);
status.add(generateContainerStatus(id5, ContainerExitStatus.ABORTED));
Assert.assertEquals("Unexpected number of completed containers",
targetContainers, master.getNumCompletedContainers());
Assert.assertTrue("Master didn't finish containers as expected",
master.getDone());
assertEquals(targetContainers, master.getNumCompletedContainers(),
"Unexpected number of completed containers");
assertTrue(master.getDone(), "Master didn't finish containers as expected");
status.add(generateContainerStatus(id5, ContainerExitStatus.SUCCESS));
Assert.assertEquals("Unexpected number of completed containers",
targetContainers, master.getNumCompletedContainers());
Assert.assertTrue("Master didn't finish containers as expected",
master.getDone());
assertEquals(targetContainers, master.getNumCompletedContainers(),
"Unexpected number of completed containers");
assertTrue(master.getDone(), "Master didn't finish containers as expected");
}

private Container generateContainer(ContainerId cid) {
Expand Down Expand Up @@ -200,15 +200,15 @@ private void runTimelineClientInDSAppMaster(boolean v1Enabled,
private void validateAppMasterTimelineService(boolean v1Enabled,
boolean v2Enabled, ApplicationMaster appMaster) {
if (v1Enabled) {
Assert.assertEquals(appMaster.appSubmitterUgi,
((TimelineClientImpl)appMaster.timelineClient).getUgi());
assertEquals(appMaster.appSubmitterUgi,
((TimelineClientImpl) appMaster.timelineClient).getUgi());
} else {
Assert.assertNull(appMaster.timelineClient);
assertNull(appMaster.timelineClient);
}
if (v2Enabled) {
Assert.assertNotNull(appMaster.timelineV2Client);
assertNotNull(appMaster.timelineV2Client);
} else {
Assert.assertNull(appMaster.timelineV2Client);
assertNull(appMaster.timelineV2Client);
}
}

Expand All @@ -227,7 +227,7 @@ private ApplicationMaster createAppMasterWithStartedTimelineService(
private Configuration getTimelineServiceConf(boolean v1Enabled,
boolean v2Enabled) {
Configuration conf = new YarnConfiguration(new Configuration(false));
Assert.assertFalse(YarnConfiguration.timelineServiceEnabled(conf));
assertFalse(YarnConfiguration.timelineServiceEnabled(conf));

if (v1Enabled || v2Enabled) {
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
Expand Down
Loading