Created
July 11, 2016 14:26
-
-
Save fac2003/1f89e21862d6c8e46ee5564fc52a7f37 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| java -Xmx10g -cp target/dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar org.deeplearning4j.examples.rnn.GravesLSTMCharModellingExample 2>&1 |tee GPU-benchmark-0.4.0-3.txt | |
| o.n.n.NativeOps - Number of threads used for linear algebra 32 | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [1] to device [0], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [27] to device [0], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [28] to device [0], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [29] to device [0], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [30] to device [0], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [31] to device [0], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [32] to device [0], out of [8] devices... | |
| o.n.j.h.i.CudaZeroHandler - Creating bucketID: 5 | |
| o.n.j.h.i.CudaZeroHandler - Creating bucketID: 0 | |
| o.n.j.h.i.CudaZeroHandler - Creating bucketID: 3 | |
| o.n.j.h.i.CudaZeroHandler - Creating bucketID: 4 | |
| o.n.j.h.i.CudaZeroHandler - Creating bucketID: 1 | |
| o.n.j.h.i.CudaZeroHandler - Creating bucketID: 2 | |
| Using existing text file at /tmp/Shakespeare.txt | |
| o.n.j.c.CudaAffinityManager - Mapping thread [144] to device [1], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [146] to device [2], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [123] to device [3], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [125] to device [4], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [134] to device [5], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [137] to device [6], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [119] to device [7], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [124] to device [0], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [145] to device [1], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [140] to device [2], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [127] to device [3], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [149] to device [4], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [147] to device [5], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [143] to device [6], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [141] to device [7], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [136] to device [0], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [138] to device [1], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [120] to device [2], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [122] to device [3], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [126] to device [4], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [128] to device [5], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [129] to device [6], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [133] to device [7], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [148] to device [0], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [139] to device [1], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [131] to device [2], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [132] to device [3], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [142] to device [4], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [121] to device [5], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [135] to device [6], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [118] to device [7], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Mapping thread [130] to device [0], out of [8] devices... | |
| o.d.s.i.p.ParameterAveragingTrainingMaster - Starting training of split 1 of 1. workerMiniBatchSize=8, averagingFreq=3, Configured for 32 workers | |
| [Stage 4:=======> (4 + 28) / 32]o.n.j.c.CudaAffinityManager - Manually mapping thread [398] to device [5], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [397] to device [1], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [396] to device [4], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [399] to device [5], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [400] to device [6], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [402] to device [7], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [403] to device [3], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [404] to device [7], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [405] to device [2], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [401] to device [0], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [406] to device [4], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [407] to device [5], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [408] to device [3], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [409] to device [1], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [410] to device [6], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [411] to device [4], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [412] to device [6], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [413] to device [5], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [414] to device [1], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [415] to device [6], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [416] to device [0], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [417] to device [2], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [419] to device [0], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [418] to device [7], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [421] to device [1], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [422] to device [7], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [433] to device [0], out of [8] devices... | |
| o.n.j.c.CudaAffinityManager - Manually mapping thread [434] to device [4], out of [8] devices... | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:2831 code=77(<unknown>) "cudaStreamSynchronize(*stream)" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" | |
| Failed on [140503481981904] -> [35868684800], size: [307692], direction: [0], result: [77] | |
| Failed on [140505362010848] -> [35639787520], size: [307692], direction: [0], result: [77] | |
| Failed on [140504420758064] -> [35642192384], size: [307692], direction: [0], result: [77] | |
| Failed on [140505159116224] -> [35870781952], size: [307692], direction: [0], result: [77] | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" | |
| Failed on [35918708736] -> [180751826944], size: [2239508], direction: [1], result: [77] | |
| o.a.s.e.Executor - Exception in task 12.0 in stage 4.0 (TID 140) | |
| java.lang.IllegalStateException: MemcpyAsync relocate H2D failed: [35918708736] -> [180751826944] | |
| at org.nd4j.jita.handler.impl.CudaZeroHandler.relocate(CudaZeroHandler.java:366) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.jita.handler.impl.CudaZeroHandler.getDevicePointer(CudaZeroHandler.java:733) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.jita.allocator.impl.AtomicAllocator.getPointer(AtomicAllocator.java:256) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.jcublas.ops.executioner.JCudaExecutioner.invoke(JCudaExecutioner.java:1001) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.jcublas.ops.executioner.JCudaExecutioner.exec(JCudaExecutioner.java:552) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| o.a.s.s.TaskSetManager - Task 12 in stage 4.0 failed 1 times; aborting job | |
| Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 12 in stage 4.0 failed 1 times, most recent failure: Lost task 12.0 in stage 4.0 (TID 140, localhost): java.lang.IllegalStateException: MemcpyAsync relocate H2D failed: [35918708736] -> [180751826944] | |
| at org.nd4j.jita.handler.impl.CudaZeroHandler.relocate(CudaZeroHandler.java:366) | |
| at org.nd4j.jita.handler.impl.CudaZeroHandler.getDevicePointer(CudaZeroHandler.java:733) | |
| at org.nd4j.jita.allocator.impl.AtomicAllocator.getPointer(AtomicAllocator.java:256) | |
| at org.nd4j.linalg.jcublas.ops.executioner.JCudaExecutioner.invoke(JCudaExecutioner.java:1001) | |
| at org.nd4j.linalg.jcublas.ops.executioner.JCudaExecutioner.exec(JCudaExecutioner.java:552) | |
| at org.nd4j.linalg.api.ndarray.BaseNDArray.assign(BaseNDArray.java:2943) | |
| at org.nd4j.linalg.factory.BaseNDArrayFactory.ones(BaseNDArrayFactory.java:1247) | |
| at org.nd4j.linalg.factory.BaseNDArrayFactory.ones(BaseNDArrayFactory.java:972) | |
| at org.nd4j.linalg.factory.Nd4j.ones(Nd4j.java:4504) | |
| at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.initMask(MultiLayerNetwork.java:1879) | |
| at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.init(MultiLayerNetwork.java:416) | |
| at org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingWorker.getInitialModel(ParameterAveragingTrainingWorker.java:44) | |
| at org.deeplearning4j.spark.api.worker.ExecuteWorkerFlatMap.call(ExecuteWorkerFlatMap.java:68) | |
| at org.deeplearning4j.spark.api.worker.ExecuteWorkerFlatMap.call(ExecuteWorkerFlatMap.java:26) | |
| at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$4$1.apply(JavaRDDLike.scala:156) | |
| at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$4$1.apply(JavaRDDLike.scala:156) | |
| at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:706) | |
| at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:706) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:264) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:88) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:745) | |
| Driver stacktrace: | |
| at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1280) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1268) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1267) | |
| at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) | |
| at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) | |
| at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1267) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) | |
| at scala.Option.foreach(Option.scala:257) | |
| at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:697) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1493) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1455) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1444) | |
| at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) | |
| at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:1813) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:1933)CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:2831 code=77(<unknown>) "cudaStreamSynchronize(*stream)" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" | |
| at org.apache.spark.rdd.RDD$$anonfun$aggregate$1.apply(RDD.scala:1078) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:147)Failed on [140504555050896] -> [35534496768], size: [307692], direction: [0], result: [77] | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:108) | |
| at org.apache.spark.rdd.RDD.withScope(RDD.scala:306) | |
| at org.apache.spark.rdd.RDD.aggregate(RDD.scala:1071) | |
| at org.apache.spark.api.java.JavaRDDLike$class.aggregate(JavaRDDLike.scala:416) | |
| at org.apache.spark.api.java.AbstractJavaRDDLike.aggregate(JavaRDDLike.scala:47) | |
| at org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster.processResults(ParameterAveragingTrainingMaster.java:407) | |
| at org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster.doIteration(ParameterAveragingTrainingMaster.java:339) | |
| at org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster.executeTraining(ParameterAveragingTrainingMaster.java:181) | |
| at org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer.fit(SparkDl4jMultiLayer.java:201) | |
| at org.deeplearning4j.examples.rnn.GravesLSTMCharModellingExample.main(GravesLSTMCharModellingExample.java:115) | |
| Caused by: java.lang.IllegalStateException: MemcpyAsync relocate H2D failed: [35918708736] -> [180751826944] | |
| at org.nd4j.jita.handler.impl.CudaZeroHandler.relocate(CudaZeroHandler.java:366) | |
| at org.nd4j.jita.handler.impl.CudaZeroHandler.getDevicePointer(CudaZeroHandler.java:733) | |
| at org.nd4j.jita.allocator.impl.AtomicAllocator.getPointer(AtomicAllocator.java:256) | |
| at org.nd4j.linalg.jcublas.ops.executioner.JCudaExecutioner.invoke(JCudaExecutioner.java:1001) | |
| at org.nd4j.linalg.jcublas.ops.executioner.JCudaExecutioner.exec(JCudaExecutioner.java:552) | |
| at org.nd4j.linalg.api.ndarray.BaseNDArray.assign(BaseNDArray.java:2943) | |
| at org.nd4j.linalg.factory.BaseNDArrayFactory.ones(BaseNDArrayFactory.java:1247) | |
| at org.nd4j.linalg.factory.BaseNDArrayFactory.ones(BaseNDArrayFactory.java:972) | |
| at org.nd4j.linalg.factory.Nd4j.ones(Nd4j.java:4504) | |
| o.n.j.h.i.CudaZeroHandler - Out of [DEVICE] memory, host memory will be used instead: deviceId: [1], requested bytes: [2239508] | |
| at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.initMask(MultiLayerNetwork.java:1879) | |
| at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.init(MultiLayerNetwork.java:416) | |
| at org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingWorker.getInitialModel(ParameterAveragingTrainingWorker.java:44) | |
| at org.deeplearning4j.spark.api.worker.ExecuteWorkerFlatMap.call(ExecuteWorkerFlatMap.java:68) | |
| at org.deeplearning4j.spark.api.worker.ExecuteWorkerFlatMap.call(ExecuteWorkerFlatMap.java:26) | |
| at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$4$1.apply(JavaRDDLike.scala:156) | |
| at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$4$1.apply(JavaRDDLike.scala:156) | |
| at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:706) | |
| at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:706) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) | |
| o.n.j.h.i.CudaZeroHandler - Out of [DEVICE] memory, host memory will be used instead: deviceId: [1], requested bytes: [2239508] | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:264) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:88) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| at java.lang.Thread.run(Thread.java:745) | |
| o.n.j.h.i.CudaZeroHandler - Out of [DEVICE] memory, host memory will be used instead: deviceId: [5], requested bytes: [2239508] | |
| o.n.j.h.i.CudaZeroHandler - Out of [DEVICE] memory, host memory will be used instead: deviceId: [5], requested bytes: [2239508] | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" | |
| Failed on [140504352848288] -> [35692831744], size: [307692], direction: [0], result: [77] | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:2831 code=77(<unknown>) "cudaStreamSynchronize(*stream)" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:2831 code=77(<unknown>) "cudaStreamSynchronize(*stream)" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:2831 code=77(<unknown>) "cudaStreamSynchronize(*stream)" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:2831 code=77(<unknown>) "cudaStreamSynchronize(*stream)" | |
| o.a.s.e.Executor - Exception in task 26.0 in stage 4.0 (TID 154) | |
| java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140504352848288] -> [35692831744] | |
| at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.readObject(BaseNDArrayProxy.java:68) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source) ~[na:na] | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_91] | |
| at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_91] | |
| at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1058) ~[na:1.8.0_91] | |
| Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140504352848288] -> [35692831744] | |
| at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.read(BaseCudaDataBuffer.java:737) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.api.buffer.BaseDataBuffer.read(BaseDataBuffer.java:1129) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.read(BaseNDArrayProxy.java:75) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.readObject(BaseNDArrayProxy.java:66) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source) ~[na:na] | |
| Caused by: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140504352848288] -> [35692831744] | |
| at org.nd4j.jita.handler.impl.CudaZeroHandler.memcpyAsync(CudaZeroHandler.java:510) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.jita.allocator.impl.AtomicAllocator.memcpyAsync(AtomicAllocator.java:774) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.set(BaseCudaDataBuffer.java:324) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.setData(BaseCudaDataBuffer.java:371) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.read(BaseCudaDataBuffer.java:728) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| o.a.s.e.Executor - Exception in task 22.0 in stage 4.0 (TID 150) | |
| java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140505362010848] -> [35639787520] | |
| at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.readObject(BaseNDArrayProxy.java:68) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source) ~[na:na] | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_91] | |
| at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_91] | |
| at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1058) ~[na:1.8.0_91] | |
| Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140505362010848] -> [35639787520] | |
| at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.read(BaseCudaDataBuffer.java:737) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.api.buffer.BaseDataBuffer.read(BaseDataBuffer.java:1129) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.read(BaseNDArrayProxy.java:75) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.readObject(BaseNDArrayProxy.java:66) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source) ~[na:na] | |
| Caused by: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140505362010848] -> [35639787520] | |
| at org.nd4j.jita.handler.impl.CudaZeroHandler.memcpyAsync(CudaZeroHandler.java:510) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.jita.allocator.impl.AtomicAllocator.memcpyAsync(AtomicAllocator.java:774) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.set(BaseCudaDataBuffer.java:324) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.setData(BaseCudaDataBuffer.java:371) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.read(BaseCudaDataBuffer.java:728) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| o.a.s.e.Executor - Exception in task 17.0 in stage 4.0 (TID 145) | |
| java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140504420758064] -> [35642192384] | |
| at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.readObject(BaseNDArrayProxy.java:68) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source) ~[na:na] | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_91] | |
| at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_91] | |
| at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1058) ~[na:1.8.0_91] | |
| Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140504420758064] -> [35642192384] | |
| at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.read(BaseCudaDataBuffer.java:737) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.api.buffer.BaseDataBuffer.read(BaseDataBuffer.java:1129) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.read(BaseNDArrayProxy.java:75) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.readObject(BaseNDArrayProxy.java:66) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source) ~[na:na] | |
| Caused by: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140504420758064] -> [35642192384] | |
| at org.nd4j.jita.handler.impl.CudaZeroHandler.memcpyAsync(CudaZeroHandler.java:510) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.jita.allocator.impl.AtomicAllocator.memcpyAsync(AtomicAllocator.java:774) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.set(BaseCudaDataBuffer.java:324) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.setData(BaseCudaDataBuffer.java:371) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.read(BaseCudaDataBuffer.java:728) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na] | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" | |
| CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment