7. 常见MR标准测试场景样例

操作场景

以下针对hadoop的标准benchmark的用例测试描述。

操作步骤

用例	描述
MR bench	hadoop jar hadoop-mapreduce-client-jobclient-**-tests.jar mrbench \ -Dmapreduce.reduce.merge.memtomem.enabled=true -Dmapreduce.job.reduce.slowstart.completedmaps=0.0f \ -numRuns 50 -inputLines 8000000 -maps 100 -reduces 50
teragen	hadoop jar hadoop-mapreduce-examples.jar teragen \ -Ddfs.block.size=268435456 \ -Dmapreduce.job.maps=1000 10000000000
terasort	hadoop jar hadoop-mapreduce-examples.jar terasort \ -Dmapreduce.job.reduce.slowstart.completedmaps=0.1 -Dmapreduce.job.reduces=100 \ -Ddfs.client-write-packet-size=262144 -Dmapreduce.map.output.compress=true -Dmapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.SnappyCodec \ /terasort-input /terasort-output
wordcount	hadoop jar hadoop-mapreduce-examples.jar wordcount -Dmapreduce.job.reduce.slowstart.completedmaps=0.05f \ -Ddfs.client-write-packet-size=262144 -Dmapreduce.map.output.compress=true -Dmapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.SnappyCodec \ -Dmapreduce.reduce.merge.memtomem.enabled=true \ -Ddfs.datanode.max.transfer.threads=8192 \ -Dmapreduce.job.reduces=70 /wordcount_input /wordcount_output
randomtextwriter	hadoop jar hadoop-mapreduce-examples.jar randomtextwriter \ -Dmapreduce.randomtextwriter.totalbytes=1099511627776 \ /wordcount_input