Created
August 2, 2012 22:37
-
-
Save harit-sunrun/3241462 to your computer and use it in GitHub Desktop.
This is basic template you would usually need to write a Map reduce program. Most part would remain same, except Job related inputs and input/output type classes
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import org.apache.hadoop.conf.Configured; | |
import org.apache.hadoop.fs.Path; | |
import org.apache.hadoop.io.LongWritable; | |
import org.apache.hadoop.io.Text; | |
import org.apache.hadoop.mapreduce.Job; | |
import org.apache.hadoop.mapreduce.Mapper; | |
import org.apache.hadoop.mapreduce.Reducer; | |
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; | |
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; | |
import org.apache.hadoop.util.Tool; | |
import org.apache.hadoop.util.ToolRunner; | |
import java.io.IOException; | |
public class MapReduceClass extends Configured implements Tool { | |
public static class MapClass extends Mapper<LongWritable, Text, Text, LongWritable> { | |
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { | |
// your map code goes here | |
} | |
} | |
public static class Reduce extends Reducer<Text, LongWritable, Text, Text> { | |
public void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException { | |
// your reduce function goes here | |
} | |
public int run(String args[]) throws Exception { | |
Job job = new Job(); | |
job.setJarByClass(MapReduceClass.class); | |
job.setMapperClass(MapClass.class); | |
job.setReducerClass(Reduce.class); | |
FileInputFormat.setInputPaths(job, new Path(args[0])); | |
FileOutputFormat.setOutputPath(job, new Path(args[1])); | |
// if your map output key/value classes are different than your input key/value classes | |
// see - http://stackoverflow.com/questions/11761135/hadoop-job-fails-while-reducing-java-io-ioexception-type-mismatch-in-value-fro | |
job.setMapOutputKeyClass(Text.class); | |
job.setMapOutputValueClass(LongWritable.class); | |
job.setJobName("MapReduceClass"); | |
// uncomment the following if you just want to run Mapper and not Reducer | |
// job.setNumReduceTasks(0); | |
boolean success = job.waitForCompletion(true); | |
return success ? 0 : 1; | |
} | |
public static void main(String args[]) throws Exception { | |
int ret = ToolRunner.run(new MapReduceClass(), args); | |
System.exit(ret); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Just missing a brace between 29 and 30.