Created
June 25, 2014 17:14
-
-
Save drewlanenga/0140a1e5f6e946776710 to your computer and use it in GitHub Desktop.
Shuffle OOM
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 14/06/25 16:40:07 INFO mapreduce.Job: map 97% reduce 0% | |
| 14/06/25 16:40:24 INFO mapreduce.Job: map 97% reduce 3% | |
| 14/06/25 16:40:27 INFO mapreduce.Job: map 97% reduce 5% | |
| 14/06/25 16:40:27 INFO mapreduce.Job: Task Id : attempt_1402677141678_0008_r_000000_0, Status : FAILED | |
| Error: org.apache.hadoop.mapreduce.task.reduce.Shuffle$ShuffleError: error in shuffle in fetcher#1 | |
| at org.apache.hadoop.mapreduce.task.reduce.Shuffle.run(Shuffle.java:121) | |
| at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:380) | |
| at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:162) | |
| at java.security.AccessController.doPrivileged(Native Method) | |
| at javax.security.auth.Subject.doAs(Subject.java:396) | |
| at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1491) | |
| at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:157) | |
| Caused by: java.lang.OutOfMemoryError: Java heap space | |
| at org.apache.hadoop.io.BoundedByteArrayOutputStream.<init>(BoundedByteArrayOutputStream.java:56) | |
| at org.apache.hadoop.io.BoundedByteArrayOutputStream.<init>(BoundedByteArrayOutputStream.java:46) | |
| at org.apache.hadoop.mapreduce.task.reduce.InMemoryMapOutput.<init>(InMemoryMapOutput.java:63) | |
| at org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl.unconditionalReserve(MergeManagerImpl.java:297) | |
| at org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl.reserve(MergeManagerImpl.java:287) | |
| at org.apache.hadoop.mapreduce.task.reduce.Fetcher.copyMapOutput(Fetcher.java:411) | |
| at org.apache.hadoop.mapreduce.task.reduce.Fetcher.copyFromHost(Fetcher.java:341) | |
| at org.apache.hadoop.mapreduce.task.reduce.Fetcher.run(Fetcher.java:165) | |
| 14/06/25 16:40:28 INFO mapreduce.Job: map 97% reduce 0% | |
| 14/06/25 16:40:33 INFO mapreduce.Job: map 98% reduce 0% | |
| 14/06/25 16:40:52 INFO mapreduce.Job: map 98% reduce 5% | |
| 14/06/25 16:40:52 INFO mapreduce.Job: Task Id : attempt_1402677141678_0008_r_000000_1, Status : FAILED | |
| Error: org.apache.hadoop.mapreduce.task.reduce.Shuffle$ShuffleError: error in shuffle in fetcher#1 | |
| at org.apache.hadoop.mapreduce.task.reduce.Shuffle.run(Shuffle.java:121) | |
| at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:380) | |
| at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:162) | |
| at java.security.AccessController.doPrivileged(Native Method) | |
| at javax.security.auth.Subject.doAs(Subject.java:396) | |
| at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1491) | |
| at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:157) | |
| Caused by: java.lang.OutOfMemoryError: Java heap space | |
| at org.apache.hadoop.io.BoundedByteArrayOutputStream.<init>(BoundedByteArrayOutputStream.java:56) | |
| at org.apache.hadoop.io.BoundedByteArrayOutputStream.<init>(BoundedByteArrayOutputStream.java:46) | |
| at org.apache.hadoop.mapreduce.task.reduce.InMemoryMapOutput.<init>(InMemoryMapOutput.java:63) | |
| at org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl.unconditionalReserve(MergeManagerImpl.java:297) | |
| at org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl.reserve(MergeManagerImpl.java:287) | |
| at org.apache.hadoop.mapreduce.task.reduce.Fetcher.copyMapOutput(Fetcher.java:411) | |
| at org.apache.hadoop.mapreduce.task.reduce.Fetcher.copyFromHost(Fetcher.java:341) | |
| at org.apache.hadoop.mapreduce.task.reduce.Fetcher.run(Fetcher.java:165) | |
| 14/06/25 16:40:53 INFO mapreduce.Job: map 98% reduce 0% | |
| 14/06/25 16:41:08 INFO mapreduce.Job: map 98% reduce 2% | |
| 14/06/25 16:41:12 INFO mapreduce.Job: map 98% reduce 3% | |
| 14/06/25 16:41:16 INFO mapreduce.Job: map 98% reduce 5% | |
| 14/06/25 16:41:19 INFO mapreduce.Job: map 98% reduce 6% | |
| 14/06/25 16:41:34 INFO mapreduce.Job: map 98% reduce 8% | |
| 14/06/25 16:41:37 INFO mapreduce.Job: map 98% reduce 10% | |
| 14/06/25 16:41:39 INFO mapreduce.Job: map 100% reduce 10% | |
| 14/06/25 16:41:40 INFO mapreduce.Job: map 100% reduce 11% | |
| 14/06/25 16:41:53 INFO mapreduce.Job: map 100% reduce 13% | |
| 14/06/25 16:41:56 INFO mapreduce.Job: map 100% reduce 14% | |
| 14/06/25 16:41:58 INFO mapreduce.Job: map 100% reduce 16% | |
| 14/06/25 16:42:11 INFO mapreduce.Job: map 100% reduce 17% | |
| 14/06/25 16:42:15 INFO mapreduce.Job: map 100% reduce 21% | |
| 14/06/25 16:42:21 INFO mapreduce.Job: map 100% reduce 22% | |
| 14/06/25 16:42:24 INFO mapreduce.Job: map 100% reduce 24% | |
| 14/06/25 16:42:36 INFO mapreduce.Job: map 100% reduce 25% | |
| 14/06/25 16:42:42 INFO mapreduce.Job: map 100% reduce 27% | |
| 14/06/25 16:42:45 INFO mapreduce.Job: map 100% reduce 33% | |
| 14/06/25 16:42:48 INFO mapreduce.Job: map 100% reduce 67% | |
| 14/06/25 16:44:35 INFO mapreduce.Job: map 100% reduce 68% | |
| 14/06/25 16:45:45 INFO mapreduce.Job: map 100% reduce 69% | |
| 14/06/25 16:47:06 INFO mapreduce.Job: map 100% reduce 70% | |
| 14/06/25 16:48:29 INFO mapreduce.Job: map 100% reduce 71% | |
| 14/06/25 16:49:32 INFO mapreduce.Job: map 100% reduce 72% | |
| 14/06/25 16:50:39 INFO mapreduce.Job: map 100% reduce 73% | |
| 14/06/25 16:51:52 INFO mapreduce.Job: map 100% reduce 74% | |
| 14/06/25 16:52:57 INFO mapreduce.Job: map 100% reduce 75% | |
| 14/06/25 16:54:03 INFO mapreduce.Job: map 100% reduce 76% |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment