how to create yarn application #cmdevio2017
TRANSCRIPT
YarnClient client = YarnClient.createYarnClient();Configuration conf = new Configuration();client.init(conf);client.start();
YarnClientApplication app = client.createApplication();ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();ApplicationId appId = appContext.getApplicationId();
// appContext.setApplicationName(appName);
// AM memory, vcore Resource resource = Records.newRecord(Resource.class);resource.setMemory(amMemory);resource.setVirtualCores(amVcore);appContext.setResource(resource);
Map<String, LocalResource> localResources = new HashMap<>();ContainerLaunchContext container = Records.newRecord(ContainerLaunchContext.class);
FileSystem fs = FileSystem.get(conf);
// HDFS Jarfs.copyFromLocalFile(new Path(src), new Path(distPath));
// JarFileStatus status = fs.getFileStatus(distPath);LocalResource localResource = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(dist.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, status.getLen(), status.getModificationTime());localResources.put(distJar, localResource);container.setLocalResources(localResources);
Map<String, String> env = new HashMap<>();LocalResource appJar = localResources.get(distJar);Path jarPath= new Path(distPath);
env.put(“jar_path”, distPath);env.put(“jar_length”, Long.toString(status.getLen()));env.put(“jar_timestamp”, Long.toString(status.getModificationTime()));
StringBuilder classPath = new StringBuilder(Environment.CLASSPATH.$$()) .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");
// ※ Hadoop classpath
env.put(“classpath”, classPath.toString());container.setEnviroment(env);
StringBuilder sb = new StringBuilder();sb.appned(Environment.JAVA_HOME.$$() + “/bin/java ”) .append("-Xmx" + amMemory + "m ") .append("jp.classmethod.yarn_app.MyApplicationMaster ") .append("…") // .append("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout ") .append("2> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout ");List<String> cmd = new ArrayList<>();cmd.append(sb.toString());container.setCommands(cmd);appContext.setAMContainerSpec(container);
client.submitApplication();
while(true) { ApplicationReport report = client.getApplicationReport(appId); YarnApplicationState appState = report.getYarnApplicationState(); FinalApplicationStatus finStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == appState && FinalApplicationStatus.SUCCESS == finStatus) { // } else { // } Thread.sleep(1000);}
Configuration conf = new Configuration();
// (ex. attempId )ContainerId containerId = ConverterUtils.toContainerId( envs.get(ApplicationConstants.Environment.CONTAINER_ID.name()));ApplicationAttemptId attemptId = containerId.getApplicationAttemptId();
AMRMClient<ContainerRequest> client = AMRMClient.createAMRMClient();client.init(conf);client.start();
// ApplicationMasterclient.registerApplicationMaster("", 0, "");
// for (i = 0; i < totalContainerSize; i++) { ContainerRequest containerReq = new ContainerRequest(capability, null, null, priority); client.addContainerRequest(containerReq);}
NMClient nmClient = NMClient.createNMClient();nmClient.init(conf);nmClient.start();
// //int allocatedContainers = 0;int complitedConttainers = 0;
while(completedContainers < totalContainerSize) { AllocateResponse response = amRMClient.allocate(completedContainers / totalContainerSize);
for (Container container : response.getAllocatedContainers()) { allocatedContainers++; ContainerLaunchContext appContainer = … // ※ container nmClient.startContainer(container, appContainer);
}
completedContainers += response.getCompletedContainersStatuses().size();
Thread.sleep(100);}
ContainerLaunchContext appContainer = Records.newRecord(ContainerLaunchContext.class);
// jarappContainer.setLocalRecources( Collections.singletonMap("appJar", appJar);appContainer.setEnviroment(env);
// Container javaappContainer.setCommands("$JAVA_HOME/bin/java" + "…");
// client.unregisterApplicationMaster( FinalApplicationStatus.SUCCEEDED, "", "");