Created
November 15, 2017 15:52
-
-
Save greebie/668142378301893f4a35fc16d518a007 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
[ryandeschamps@desktop2-ia ~]$ ssh altiscale | |
ssh: Could not resolve hostname altiscale: Name or service not known | |
[ryandeschamps@desktop2-ia ~]$ cat .bash_profile | |
# .bash_profile | |
# Get the aliases and functions | |
if [ -f ~/.bashrc ]; then | |
. ~/.bashrc | |
fi | |
# User specific environment and startup programs | |
PATH=$PATH:/opt/spark-beta/bin | |
export PATH | |
export PATH | |
export SCALA_HOME=/opt/scala/ | |
export YARN_CONF_DIR=/etc/hadoop/ | |
export SPARK_HOME=/opt/spark-beta | |
SPARK_CONF_DIR=/etc/spark-beta | |
# Update Python path for PySpark | |
export PYTHONPATH=$SPARK_HOME/python/:$PYTHONPATH | |
export PYTHONPATH=$SPARK_HOME/python/lib/py4j-0.10.4-src.zip:$PYTHONPATH | |
#cd $SPARK_HOME/test_spark && ./init_spark.sh | |
#cd | |
[ryandeschamps@desktop2-ia ~]$ echo $SCALA_HOME | |
/opt/scala/ | |
[ryandeschamps@desktop2-ia ~]$ pyspark | |
Python 2.6.6 (r266:84292, Jan 22 2014, 09:42:36) | |
[GCC 4.4.7 20120313 (Red Hat 4.4.7-4)] on linux2 | |
Type "help", "copyright", "credits" or "license" for more information. | |
Setting default log level to "WARN". | |
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel). | |
2017-11-15 15:48:47,490 WARN org.apache.spark.util.Utils (Logging.scala:logWarning(66)) - Service 'sparkDriver' could not bind on port 45055. Attempting port 45056. | |
2017-11-15 15:48:48,111 WARN org.spark_project.jetty.util.component.AbstractLifeCycle (AbstractLifeCycle.java:setFailed(212)) - FAILED Spark@20cb1201{HTTP/1.1}{0.0.0.0:45100}: java.net.BindException: Address already in use | |
java.net.BindException: Address already in use | |
at sun.nio.ch.Net.bind0(Native Method) | |
at sun.nio.ch.Net.bind(Net.java:433) | |
at sun.nio.ch.Net.bind(Net.java:425) | |
at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223) | |
at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74) | |
at org.spark_project.jetty.server.ServerConnector.open(ServerConnector.java:321) | |
at org.spark_project.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80) | |
at org.spark_project.jetty.server.ServerConnector.doStart(ServerConnector.java:236) | |
at org.spark_project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68) | |
at org.spark_project.jetty.server.Server.doStart(Server.java:366) | |
at org.spark_project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68) | |
at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:365) | |
at org.apache.spark.ui.JettyUtils$$anonfun$5.apply(JettyUtils.scala:375) | |
at org.apache.spark.ui.JettyUtils$$anonfun$5.apply(JettyUtils.scala:375) | |
at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:2213) | |
at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:160) | |
at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:2204) | |
at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:375) | |
at org.apache.spark.ui.WebUI.bind(WebUI.scala:130) | |
at org.apache.spark.SparkContext$$anonfun$10.apply(SparkContext.scala:460) | |
at org.apache.spark.SparkContext$$anonfun$10.apply(SparkContext.scala:460) | |
at scala.Option.foreach(Option.scala:257) | |
at org.apache.spark.SparkContext.<init>(SparkContext.scala:460) | |
at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58) | |
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) | |
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) | |
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) | |
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) | |
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247) | |
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
at py4j.Gateway.invoke(Gateway.java:236) | |
at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80) | |
at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69) | |
at py4j.GatewayConnection.run(GatewayConnection.java:214) | |
at java.lang.Thread.run(Thread.java:748) | |
2017-11-15 15:48:48,127 WARN org.spark_project.jetty.util.component.AbstractLifeCycle (AbstractLifeCycle.java:setFailed(212)) - FAILED org.spark_project.jetty.server.Server@2f2cc2e3: java.net.BindException: Address already in use | |
java.net.BindException: Address already in use | |
at sun.nio.ch.Net.bind0(Native Method) | |
at sun.nio.ch.Net.bind(Net.java:433) | |
at sun.nio.ch.Net.bind(Net.java:425) | |
at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223) | |
at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74) | |
at org.spark_project.jetty.server.ServerConnector.open(ServerConnector.java:321) | |
at org.spark_project.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80) | |
at org.spark_project.jetty.server.ServerConnector.doStart(ServerConnector.java:236) | |
at org.spark_project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68) | |
at org.spark_project.jetty.server.Server.doStart(Server.java:366) | |
at org.spark_project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68) | |
at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:365) | |
at org.apache.spark.ui.JettyUtils$$anonfun$5.apply(JettyUtils.scala:375) | |
at org.apache.spark.ui.JettyUtils$$anonfun$5.apply(JettyUtils.scala:375) | |
at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:2213) | |
at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:160) | |
at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:2204) | |
at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:375) | |
at org.apache.spark.ui.WebUI.bind(WebUI.scala:130) | |
at org.apache.spark.SparkContext$$anonfun$10.apply(SparkContext.scala:460) | |
at org.apache.spark.SparkContext$$anonfun$10.apply(SparkContext.scala:460) | |
at scala.Option.foreach(Option.scala:257) | |
at org.apache.spark.SparkContext.<init>(SparkContext.scala:460) | |
at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58) | |
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) | |
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) | |
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) | |
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) | |
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247) | |
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
at py4j.Gateway.invoke(Gateway.java:236) | |
at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80) | |
at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69) | |
at py4j.GatewayConnection.run(GatewayConnection.java:214) | |
at java.lang.Thread.run(Thread.java:748) | |
2017-11-15 15:48:48,144 WARN org.apache.spark.util.Utils (Logging.scala:logWarning(66)) - Service 'SparkUI' could not bind on port 45100. Attempting port 45101. | |
2017-11-15 15:48:53,470 WARN org.apache.spark.deploy.yarn.Client (Logging.scala:logWarning(66)) - Neither spark.yarn.jars nor spark.yarn.archive is set, falling back to uploading libraries under SPARK_HOME. | |
2017-11-15 15:49:05,921 ERROR org.apache.spark.SparkContext (Logging.scala:logError(91)) - Error initializing SparkContext. | |
java.io.FileNotFoundException: File does not exist: hdfs://nn-ia.s3s.altiscale.com:8020/apps/spark/2.1.1/external/hive/1.2.1/archive | |
at org.apache.hadoop.fs.Hdfs.getFileStatus(Hdfs.java:134) | |
at org.apache.hadoop.fs.AbstractFileSystem.resolvePath(AbstractFileSystem.java:467) | |
at org.apache.hadoop.fs.FileContext$25.next(FileContext.java:2193) | |
at org.apache.hadoop.fs.FileContext$25.next(FileContext.java:2189) | |
at org.apache.hadoop.fs.FSLinkResolver.resolve(FSLinkResolver.java:90) | |
at org.apache.hadoop.fs.FileContext.resolve(FileContext.java:2189) | |
at org.apache.hadoop.fs.FileContext.resolvePath(FileContext.java:601) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$8.apply(Client.scala:383) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$8.apply(Client.scala:381) | |
at scala.collection.mutable.MapLike$class.getOrElseUpdate(MapLike.scala:194) | |
at scala.collection.mutable.AbstractMap.getOrElseUpdate(Map.scala:80) | |
at org.apache.spark.deploy.yarn.Client.copyFileToRemote(Client.scala:381) | |
at org.apache.spark.deploy.yarn.Client.org$apache$spark$deploy$yarn$Client$$distribute$1(Client.scala:490) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11$$anonfun$apply$8.apply(Client.scala:611) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11$$anonfun$apply$8.apply(Client.scala:610) | |
at scala.collection.mutable.ArraySeq.foreach(ArraySeq.scala:74) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11.apply(Client.scala:610) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11.apply(Client.scala:609) | |
at scala.collection.immutable.List.foreach(List.scala:381) | |
at org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:609) | |
at org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scala:882) | |
at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:171) | |
at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:56) | |
at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:156) | |
at org.apache.spark.SparkContext.<init>(SparkContext.scala:509) | |
at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58) | |
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) | |
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) | |
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) | |
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) | |
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247) | |
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
at py4j.Gateway.invoke(Gateway.java:236) | |
at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80) | |
at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69) | |
at py4j.GatewayConnection.run(GatewayConnection.java:214) | |
at java.lang.Thread.run(Thread.java:748) | |
2017-11-15 15:49:05,952 WARN org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnSchedulerEndpoint (Logging.scala:logWarning(66)) - Attempted to request executors before the AM has registered! | |
2017-11-15 15:49:06,030 WARN org.apache.spark.metrics.MetricsSystem (Logging.scala:logWarning(66)) - Stopping a MetricsSystem that is not running | |
2017-11-15 15:49:06,260 WARN org.apache.spark.SparkContext (Logging.scala:logWarning(66)) - Another SparkContext is being constructed (or threw an exception in its constructor). This may indicate an error, since only one SparkContext may be running in this JVM (see SPARK-2243). The other SparkContext was created at: | |
org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58) | |
sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) | |
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) | |
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) | |
java.lang.reflect.Constructor.newInstance(Constructor.java:423) | |
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247) | |
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
py4j.Gateway.invoke(Gateway.java:236) | |
py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80) | |
py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69) | |
py4j.GatewayConnection.run(GatewayConnection.java:214) | |
java.lang.Thread.run(Thread.java:748) | |
2017-11-15 15:49:06,296 WARN org.apache.spark.util.Utils (Logging.scala:logWarning(66)) - Service 'sparkDriver' could not bind on port 45055. Attempting port 45056. | |
2017-11-15 15:49:06,632 WARN org.spark_project.jetty.util.component.AbstractLifeCycle (AbstractLifeCycle.java:setFailed(212)) - FAILED Spark@105a70da{HTTP/1.1}{0.0.0.0:45100}: java.net.BindException: Address already in use | |
java.net.BindException: Address already in use | |
at sun.nio.ch.Net.bind0(Native Method) | |
at sun.nio.ch.Net.bind(Net.java:433) | |
at sun.nio.ch.Net.bind(Net.java:425) | |
at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223) | |
at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74) | |
at org.spark_project.jetty.server.ServerConnector.open(ServerConnector.java:321) | |
at org.spark_project.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80) | |
at org.spark_project.jetty.server.ServerConnector.doStart(ServerConnector.java:236) | |
at org.spark_project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68) | |
at org.spark_project.jetty.server.Server.doStart(Server.java:366) | |
at org.spark_project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68) | |
at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:365) | |
at org.apache.spark.ui.JettyUtils$$anonfun$5.apply(JettyUtils.scala:375) | |
at org.apache.spark.ui.JettyUtils$$anonfun$5.apply(JettyUtils.scala:375) | |
at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:2213) | |
at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:160) | |
at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:2204) | |
at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:375) | |
at org.apache.spark.ui.WebUI.bind(WebUI.scala:130) | |
at org.apache.spark.SparkContext$$anonfun$10.apply(SparkContext.scala:460) | |
at org.apache.spark.SparkContext$$anonfun$10.apply(SparkContext.scala:460) | |
at scala.Option.foreach(Option.scala:257) | |
at org.apache.spark.SparkContext.<init>(SparkContext.scala:460) | |
at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58) | |
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) | |
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) | |
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) | |
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) | |
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247) | |
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
at py4j.Gateway.invoke(Gateway.java:236) | |
at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80) | |
at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69) | |
at py4j.GatewayConnection.run(GatewayConnection.java:214) | |
at java.lang.Thread.run(Thread.java:748) | |
2017-11-15 15:49:06,633 WARN org.spark_project.jetty.util.component.AbstractLifeCycle (AbstractLifeCycle.java:setFailed(212)) - FAILED org.spark_project.jetty.server.Server@3f08e119: java.net.BindException: Address already in use | |
java.net.BindException: Address already in use | |
at sun.nio.ch.Net.bind0(Native Method) | |
at sun.nio.ch.Net.bind(Net.java:433) | |
at sun.nio.ch.Net.bind(Net.java:425) | |
at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223) | |
at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74) | |
at org.spark_project.jetty.server.ServerConnector.open(ServerConnector.java:321) | |
at org.spark_project.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80) | |
at org.spark_project.jetty.server.ServerConnector.doStart(ServerConnector.java:236) | |
at org.spark_project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68) | |
at org.spark_project.jetty.server.Server.doStart(Server.java:366) | |
at org.spark_project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68) | |
at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:365) | |
at org.apache.spark.ui.JettyUtils$$anonfun$5.apply(JettyUtils.scala:375) | |
at org.apache.spark.ui.JettyUtils$$anonfun$5.apply(JettyUtils.scala:375) | |
at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:2213) | |
at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:160) | |
at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:2204) | |
at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:375) | |
at org.apache.spark.ui.WebUI.bind(WebUI.scala:130) | |
at org.apache.spark.SparkContext$$anonfun$10.apply(SparkContext.scala:460) | |
at org.apache.spark.SparkContext$$anonfun$10.apply(SparkContext.scala:460) | |
at scala.Option.foreach(Option.scala:257) | |
at org.apache.spark.SparkContext.<init>(SparkContext.scala:460) | |
at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58) | |
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) | |
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) | |
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) | |
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) | |
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247) | |
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
at py4j.Gateway.invoke(Gateway.java:236) | |
at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80) | |
at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69) | |
at py4j.GatewayConnection.run(GatewayConnection.java:214) | |
at java.lang.Thread.run(Thread.java:748) | |
2017-11-15 15:49:06,649 WARN org.apache.spark.util.Utils (Logging.scala:logWarning(66)) - Service 'SparkUI' could not bind on port 45100. Attempting port 45101. | |
2017-11-15 15:49:07,528 WARN org.apache.spark.deploy.yarn.Client (Logging.scala:logWarning(66)) - Neither spark.yarn.jars nor spark.yarn.archive is set, falling back to uploading libraries under SPARK_HOME. | |
2017-11-15 15:49:16,778 ERROR org.apache.spark.SparkContext (Logging.scala:logError(91)) - Error initializing SparkContext. | |
java.io.FileNotFoundException: File does not exist: hdfs://nn-ia.s3s.altiscale.com:8020/apps/spark/2.1.1/external/hive/1.2.1/archive | |
at org.apache.hadoop.fs.Hdfs.getFileStatus(Hdfs.java:134) | |
at org.apache.hadoop.fs.AbstractFileSystem.resolvePath(AbstractFileSystem.java:467) | |
at org.apache.hadoop.fs.FileContext$25.next(FileContext.java:2193) | |
at org.apache.hadoop.fs.FileContext$25.next(FileContext.java:2189) | |
at org.apache.hadoop.fs.FSLinkResolver.resolve(FSLinkResolver.java:90) | |
at org.apache.hadoop.fs.FileContext.resolve(FileContext.java:2189) | |
at org.apache.hadoop.fs.FileContext.resolvePath(FileContext.java:601) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$8.apply(Client.scala:383) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$8.apply(Client.scala:381) | |
at scala.collection.mutable.MapLike$class.getOrElseUpdate(MapLike.scala:194) | |
at scala.collection.mutable.AbstractMap.getOrElseUpdate(Map.scala:80) | |
at org.apache.spark.deploy.yarn.Client.copyFileToRemote(Client.scala:381) | |
at org.apache.spark.deploy.yarn.Client.org$apache$spark$deploy$yarn$Client$$distribute$1(Client.scala:490) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11$$anonfun$apply$8.apply(Client.scala:611) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11$$anonfun$apply$8.apply(Client.scala:610) | |
at scala.collection.mutable.ArraySeq.foreach(ArraySeq.scala:74) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11.apply(Client.scala:610) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11.apply(Client.scala:609) | |
at scala.collection.immutable.List.foreach(List.scala:381) | |
at org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:609) | |
at org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scala:882) | |
at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:171) | |
at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:56) | |
at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:156) | |
at org.apache.spark.SparkContext.<init>(SparkContext.scala:509) | |
at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58) | |
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) | |
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) | |
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) | |
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) | |
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247) | |
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
at py4j.Gateway.invoke(Gateway.java:236) | |
at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80) | |
at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69) | |
at py4j.GatewayConnection.run(GatewayConnection.java:214) | |
at java.lang.Thread.run(Thread.java:748) | |
2017-11-15 15:49:16,786 WARN org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnSchedulerEndpoint (Logging.scala:logWarning(66)) - Attempted to request executors before the AM has registered! | |
2017-11-15 15:49:16,792 WARN org.apache.spark.metrics.MetricsSystem (Logging.scala:logWarning(66)) - Stopping a MetricsSystem that is not running | |
Traceback (most recent call last): | |
File "/opt/spark-beta/python/pyspark/shell.py", line 47, in <module> | |
spark = SparkSession.builder.getOrCreate() | |
File "/opt/spark-beta/python/pyspark/sql/session.py", line 169, in getOrCreate | |
sc = SparkContext.getOrCreate(sparkConf) | |
File "/opt/spark-beta/python/pyspark/context.py", line 310, in getOrCreate | |
SparkContext(conf=conf or SparkConf()) | |
File "/opt/spark-beta/python/pyspark/context.py", line 118, in __init__ | |
conf, jsc, profiler_cls) | |
File "/opt/spark-beta/python/pyspark/context.py", line 182, in _do_init | |
self._jsc = jsc or self._initialize_context(self._conf._jconf) | |
File "/opt/spark-beta/python/pyspark/context.py", line 249, in _initialize_context | |
return self._jvm.JavaSparkContext(jconf) | |
File "/opt/spark-beta/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py", line 1401, in __call__ | |
File "/opt/spark-beta/python/lib/py4j-0.10.4-src.zip/py4j/protocol.py", line 319, in get_return_value | |
py4j.protocol.Py4JJavaError: An error occurred while calling None.org.apache.spark.api.java.JavaSparkContext. | |
: java.io.FileNotFoundException: File does not exist: hdfs://nn-ia.s3s.altiscale.com:8020/apps/spark/2.1.1/external/hive/1.2.1/archive | |
at org.apache.hadoop.fs.Hdfs.getFileStatus(Hdfs.java:134) | |
at org.apache.hadoop.fs.AbstractFileSystem.resolvePath(AbstractFileSystem.java:467) | |
at org.apache.hadoop.fs.FileContext$25.next(FileContext.java:2193) | |
at org.apache.hadoop.fs.FileContext$25.next(FileContext.java:2189) | |
at org.apache.hadoop.fs.FSLinkResolver.resolve(FSLinkResolver.java:90) | |
at org.apache.hadoop.fs.FileContext.resolve(FileContext.java:2189) | |
at org.apache.hadoop.fs.FileContext.resolvePath(FileContext.java:601) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$8.apply(Client.scala:383) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$8.apply(Client.scala:381) | |
at scala.collection.mutable.MapLike$class.getOrElseUpdate(MapLike.scala:194) | |
at scala.collection.mutable.AbstractMap.getOrElseUpdate(Map.scala:80) | |
at org.apache.spark.deploy.yarn.Client.copyFileToRemote(Client.scala:381) | |
at org.apache.spark.deploy.yarn.Client.org$apache$spark$deploy$yarn$Client$$distribute$1(Client.scala:490) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11$$anonfun$apply$8.apply(Client.scala:611) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11$$anonfun$apply$8.apply(Client.scala:610) | |
at scala.collection.mutable.ArraySeq.foreach(ArraySeq.scala:74) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11.apply(Client.scala:610) | |
at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11.apply(Client.scala:609) | |
at scala.collection.immutable.List.foreach(List.scala:381) | |
at org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:609) | |
at org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scala:882) | |
at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:171) | |
at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:56) | |
at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:156) | |
at org.apache.spark.SparkContext.<init>(SparkContext.scala:509) | |
at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58) | |
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) | |
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) | |
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) | |
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) | |
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247) | |
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
at py4j.Gateway.invoke(Gateway.java:236) | |
at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80) | |
at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69) | |
at py4j.GatewayConnection.run(GatewayConnection.java:214) | |
at java.lang.Thread.run(Thread.java:748) | |
>>> x = sc |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment