Unable to run MapReduce Word Count Example on Hadoop 2.6

[Amit Hora]

Hi All,

I have installed Hadoop 2.6 on two VM's each running CentOS6.5 ,64 bit while the host machine hosting Vm's is Windows 8.1,64 bit 

While trying to run the following code by submitting the jar to yarn by doing yarn jar ..... it runs fine ,but when i am running it as Java Application from Eclipse it is giving below exception 

java.lang.NullPointerException

at java.lang.ProcessBuilder.start(ProcessBuilder.java:1011)

at org.apache.hadoop.util.Shell.runCommand(Shell.java:482)

at org.apache.hadoop.util.Shell.run(Shell.java:455)

at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:715)

at org.apache.hadoop.util.Shell.execCommand(Shell.java:808)

at org.apache.hadoop.util.Shell.execCommand(Shell.java:791)

at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:656)

at org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:444)

at org.apache.hadoop.fs.FilterFileSystem.mkdirs(FilterFileSystem.java:293)

at org.apache.hadoop.mapreduce.JobSubmissionFiles.getStagingDir(JobSubmissionFiles.java:133)

at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:437)

at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1296)

at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1293)

at java.security.AccessController.doPrivileged(Native Method)

at javax.security.auth.Subject.doAs(Subject.java:415)

at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1628)

at org.apache.hadoop.mapreduce.Job.submit(Job.java:1293)

at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1314)

at com.mapr.example.WordCount2.main(WordCount2.java:104)

PFB Code

package com.mapr.example;

import java.io.IOException;

import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.JobACL;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.security.UserGroupInformation;

public class WordCount2 {

  public static class TokenizerMapper

       extends Mapper<Object, Text, Text, IntWritable>{

    private final static IntWritable one = new IntWritable(1);

    private Text word = new Text();

    public void map(Object key, Text value, Context context

                    ) throws IOException, InterruptedException {

      StringTokenizer itr = new StringTokenizer(value.toString());

      while (itr.hasMoreTokens()) {

        word.set(itr.nextToken());

        context.write(word, one);

      }

    }

  }

  public static class IntSumReducer

       extends Reducer<Text,IntWritable,Text,IntWritable> {

    private IntWritable result = new IntWritable();

    public void reduce(Text key, Iterable<IntWritable> values,

                       Context context

                       ) throws IOException, InterruptedException {

      int sum = 0;

      for (IntWritable val : values) {

        sum += val.get();

      }

      result.set(sum);

      context.write(key, result);

    }

  }

  public static void main(String[] args) throws Exception {

 System.setProperty("HADOOP_USER_NAME", "root");

 

    Configuration conf = new Configuration();

    conf.set("fs.hdfs.impl",org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()

        );

        conf.set("fs.defaultFS", "hdfs://ip:9000");

    

    UserGroupInformation ugi = UserGroupInformation.createProxyUser("root", UserGroupInformation.getLoginUser());  

    conf.set("hadoop.job.ugi", "root");

   

//    conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());

    

//    FileSystem fs = FileSystem.get(conf); 

//    fs.setPermission(p, permission);

    

//    boolean success = fs.mkdirs(new Path("/user/root/testdirectory9"));

//    System.out.println(success +" created ");

    

    Job job = Job.getInstance(conf, "word count");

    

System.out.println(job.getCluster()+" The cluster");

job.setJarByClass(WordCount2.class);

    job.setMapperClass(TokenizerMapper.class);

    job.setCombinerClass(IntSumReducer.class);

    job.setReducerClass(IntSumReducer.class);

    job.setOutputKeyClass(Text.class);

    job.setOutputValueClass(IntWritable.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));

    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    try{

    job.waitForCompletion(true);

    }catch(Exception e){

    System.out.println(e.getMessage());

    e.printStackTrace();

    }

//    System.exit(job.waitForCompletion(true) ? 0 : 1);

  }

} 

It is trying to create sataging directory as /tmp/hadoop-<my windows user name host machines >/root../staging 

[veera krishna]

better to change platform, linux is the best.