Skip to content

Instantly share code, notes, and snippets.

@laserson

laserson/cmds.scala

Created August 14, 2014 18:30
Show Gist options
  • Save laserson/ec67216462926aeae535 to your computer and use it in GitHub Desktop.
Save laserson/ec67216462926aeae535 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
cmds.scala
import org.bdgenomics.formats.avro.AlignmentRecord
import org.bdgenomics.adam.rdd.ADAMContext._
val bamFile = "/Users/laserson/repos/adam/adam-core/src/test/resources/artificial.sam"
val reads = sc.adamLoad[AlignmentRecord, Nothing](bamFile)
reads.first()
/*
java.lang.IncompatibleClassChangeError: Implementing class
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)
at java.net.URLClassLoader.access$100(URLClassLoader.java:71)
at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:190)
at hbparquet.hadoop.util.ContextUtil.<clinit>(ContextUtil.java:68)
at fi.tkk.ics.hadoop.bam.AnySAMInputFormat.getSplits(AnySAMInputFormat.java:217)
at org.apache.spark.rdd.NewHadoopRDD.getPartitions(NewHadoopRDD.scala:90)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:202)
at org.apache.spark.rdd.MappedRDD.getPartitions(MappedRDD.scala:28)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
*/
reads.take(5)
/*
java.lang.NoClassDefFoundError: Could not initialize class hbparquet.hadoop.util.ContextUtil
at fi.tkk.ics.hadoop.bam.AnySAMInputFormat.getSplits(AnySAMInputFormat.java:217)
at org.apache.spark.rdd.NewHadoopRDD.getPartitions(NewHadoopRDD.scala:90)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:202)
at org.apache.spark.rdd.MappedRDD.getPartitions(MappedRDD.scala:28)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
at scala.Option.getOrElse(Option.scala:120)
*/
Raw
start_spark.bash
export REPO=/Users/laserson/repos/adam/adam-cli/target/appassembler/repo
/Users/laserson/repos/spark-copy/bin/spark-shell \
--master local[2] \
--jars $REPO/org/scoverage/scalac-scoverage-plugin_2.10/0.99.2/scalac-scoverage-plugin_2.10-0.99.2.jar,$REPO/commons-io/commons-io/1.3.2/commons-io-1.3.2.jar,$REPO/org/bdgenomics/bdg-formats/bdg-formats/0.2.0/bdg-formats-0.2.0.jar,$REPO/org/apache/avro/avro/1.7.6/avro-1.7.6.jar,$REPO/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar,$REPO/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar,$REPO/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar,$REPO/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar,$REPO/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar,$REPO/org/tukaani/xz/1.0/xz-1.0.jar,$REPO/org/bdgenomics/adam/adam-core/0.12.2-SNAPSHOT/adam-core-0.12.2-SNAPSHOT.jar,$REPO/com/esotericsoftware/kryo/kryo/2.21/kryo-2.21.jar,$REPO/com/esotericsoftware/reflectasm/reflectasm/1.07/reflectasm-1.07-shaded.jar,$REPO/org/ow2/asm/asm/4.0/asm-4.0.jar,$REPO/com/esotericsoftware/minlog/minlog/1.2/minlog-1.2.jar,$REPO/org/objenesis/objenesis/1.2/objenesis-1.2.jar,$REPO/org/apache/hadoop/hadoop-client/2.2.0/hadoop-client-2.2.0.jar,$REPO/org/apache/hadoop/hadoop-common/2.2.0/hadoop-common-2.2.0.jar,$REPO/commons-cli/commons-cli/1.2/commons-cli-1.2.jar,$REPO/org/apache/commons/commons-math/2.1/commons-math-2.1.jar,$REPO/xmlenc/xmlenc/0.52/xmlenc-0.52.jar,$REPO/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar,$REPO/commons-lang/commons-lang/2.5/commons-lang-2.5.jar,$REPO/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar,$REPO/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar,$REPO/commons-digester/commons-digester/1.8/commons-digester-1.8.jar,$REPO/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar,$REPO/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar,$REPO/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar,$REPO/org/apache/hadoop/hadoop-auth/2.2.0/hadoop-auth-2.2.0.jar,$REPO/org/apache/zookeeper/zookeeper/3.4.5/zookeeper-3.4.5.jar,$REPO/org/apache/hadoop/hadoop-hdfs/2.2.0/hadoop-hdfs-2.2.0.jar,$REPO/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar,$REPO/org/apache/hadoop/hadoop-mapreduce-client-app/2.2.0/hadoop-mapreduce-client-app-2.2.0.jar,$REPO/org/apache/hadoop/hadoop-mapreduce-client-common/2.2.0/hadoop-mapreduce-client-common-2.2.0.jar,$REPO/org/apache/hadoop/hadoop-yarn-client/2.2.0/hadoop-yarn-client-2.2.0.jar,$REPO/com/google/inject/guice/3.0/guice-3.0.jar,$REPO/javax/inject/javax.inject/1/javax.inject-1.jar,$REPO/aopalliance/aopalliance/1.0/aopalliance-1.0.jar,$REPO/com/sun/jersey/jersey-test-framework/jersey-test-framework-grizzly2/1.9/jersey-test-framework-grizzly2-1.9.jar,$REPO/com/sun/jersey/jersey-test-framework/jersey-test-framework-core/1.9/jersey-test-framework-core-1.9.jar,$REPO/javax/servlet/javax.servlet-api/3.0.1/javax.servlet-api-3.0.1.jar,$REPO/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar,$REPO/com/sun/jersey/jersey-grizzly2/1.9/jersey-grizzly2-1.9.jar,$REPO/org/glassfish/grizzly/grizzly-http/2.1.2/grizzly-http-2.1.2.jar,$REPO/org/glassfish/grizzly/grizzly-framework/2.1.2/grizzly-framework-2.1.2.jar,$REPO/org/glassfish/gmbal/gmbal-api-only/3.0.0-b023/gmbal-api-only-3.0.0-b023.jar,$REPO/org/glassfish/external/management-api/3.0.0-b012/management-api-3.0.0-b012.jar,$REPO/org/glassfish/grizzly/grizzly-http-server/2.1.2/grizzly-http-server-2.1.2.jar,$REPO/org/glassfish/grizzly/grizzly-rcm/2.1.2/grizzly-rcm-2.1.2.jar,$REPO/org/glassfish/grizzly/grizzly-http-servlet/2.1.2/grizzly-http-servlet-2.1.2.jar,$REPO/org/glassfish/javax.servlet/3.1/javax.servlet-3.1.jar,$REPO/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar,$REPO/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar,$REPO/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar,$REPO/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar,$REPO/stax/stax-api/1.0.1/stax-api-1.0.1.jar,$REPO/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar,$REPO/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar,$REPO/javax/activation/activation/1.1/activation-1.1.jar,$REPO/com/sun/jersey/contribs/jersey-guice/1.9/jersey-guice-1.9.jar,$REPO/org/apache/hadoop/hadoop-yarn-server-common/2.2.0/hadoop-yarn-server-common-2.2.0.jar,$REPO/org/apache/hadoop/hadoop-mapreduce-client-shuffle/2.2.0/hadoop-mapreduce-client-shuffle-2.2.0.jar,$REPO/org/apache/hadoop/hadoop-yarn-api/2.2.0/hadoop-yarn-api-2.2.0.jar,$REPO/org/apache/hadoop/hadoop-mapreduce-client-core/2.2.0/hadoop-mapreduce-client-core-2.2.0.jar,$REPO/org/apache/hadoop/hadoop-yarn-common/2.2.0/hadoop-yarn-common-2.2.0.jar,$REPO/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.2.0/hadoop-mapreduce-client-jobclient-2.2.0.jar,$REPO/org/apache/hadoop/hadoop-annotations/2.2.0/hadoop-annotations-2.2.0.jar,$REPO/org/apache/spark/spark-core_2.10/1.0.1/spark-core_2.10-1.0.1.jar,$REPO/net/java/dev/jets3t/jets3t/0.7.1/jets3t-0.7.1.jar,$REPO/org/apache/curator/curator-recipes/2.4.0/curator-recipes-2.4.0.jar,$REPO/org/apache/curator/curator-framework/2.4.0/curator-framework-2.4.0.jar,$REPO/org/apache/curator/curator-client/2.4.0/curator-client-2.4.0.jar,$REPO/org/eclipse/jetty/jetty-plus/8.1.14.v20131031/jetty-plus-8.1.14.v20131031.jar,$REPO/org/eclipse/jetty/orbit/javax.transaction/1.1.1.v201105210645/javax.transaction-1.1.1.v201105210645.jar,$REPO/org/eclipse/jetty/jetty-webapp/8.1.14.v20131031/jetty-webapp-8.1.14.v20131031.jar,$REPO/org/eclipse/jetty/jetty-xml/8.1.14.v20131031/jetty-xml-8.1.14.v20131031.jar,$REPO/org/eclipse/jetty/jetty-servlet/8.1.14.v20131031/jetty-servlet-8.1.14.v20131031.jar,$REPO/org/eclipse/jetty/jetty-jndi/8.1.14.v20131031/jetty-jndi-8.1.14.v20131031.jar,$REPO/org/eclipse/jetty/orbit/javax.mail.glassfish/1.4.1.v201005082020/javax.mail.glassfish-1.4.1.v201005082020.jar,$REPO/org/eclipse/jetty/orbit/javax.activation/1.1.0.v201105071233/javax.activation-1.1.0.v201105071233.jar,$REPO/org/eclipse/jetty/jetty-security/8.1.14.v20131031/jetty-security-8.1.14.v20131031.jar,$REPO/org/eclipse/jetty/jetty-util/8.1.14.v20131031/jetty-util-8.1.14.v20131031.jar,$REPO/org/eclipse/jetty/jetty-server/8.1.14.v20131031/jetty-server-8.1.14.v20131031.jar,$REPO/org/eclipse/jetty/orbit/javax.servlet/3.0.0.v201112011016/javax.servlet-3.0.0.v201112011016.jar,$REPO/org/eclipse/jetty/jetty-continuation/8.1.14.v20131031/jetty-continuation-8.1.14.v20131031.jar,$REPO/org/eclipse/jetty/jetty-http/8.1.14.v20131031/jetty-http-8.1.14.v20131031.jar,$REPO/org/eclipse/jetty/jetty-io/8.1.14.v20131031/jetty-io-8.1.14.v20131031.jar,$REPO/com/google/guava/guava/14.0.1/guava-14.0.1.jar,$REPO/org/apache/commons/commons-lang3/3.3.2/commons-lang3-3.3.2.jar,$REPO/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar,$REPO/org/slf4j/jul-to-slf4j/1.7.5/jul-to-slf4j-1.7.5.jar,$REPO/org/slf4j/jcl-over-slf4j/1.7.5/jcl-over-slf4j-1.7.5.jar,$REPO/com/ning/compress-lzf/1.0.0/compress-lzf-1.0.0.jar,$REPO/com/twitter/chill_2.10/0.3.6/chill_2.10-0.3.6.jar,$REPO/com/twitter/chill-java/0.3.6/chill-java-0.3.6.jar,$REPO/commons-net/commons-net/2.2/commons-net-2.2.jar,$REPO/org/spark-project/akka/akka-remote_2.10/2.2.3-shaded-protobuf/akka-remote_2.10-2.2.3-shaded-protobuf.jar,$REPO/org/spark-project/akka/akka-actor_2.10/2.2.3-shaded-protobuf/akka-actor_2.10-2.2.3-shaded-protobuf.jar,$REPO/com/typesafe/config/1.0.2/config-1.0.2.jar,$REPO/io/netty/netty/3.6.6.Final/netty-3.6.6.Final.jar,$REPO/org/spark-project/protobuf/protobuf-java/2.4.1-shaded/protobuf-java-2.4.1-shaded.jar,$REPO/org/uncommons/maths/uncommons-maths/1.2.2a/uncommons-maths-1.2.2a.jar,$REPO/org/spark-project/akka/akka-slf4j_2.10/2.2.3-shaded-protobuf/akka-slf4j_2.10-2.2.3-shaded-protobuf.jar,$REPO/org/json4s/json4s-jackson_2.10/3.2.6/json4s-jackson_2.10-3.2.6.jar,$REPO/com/fasterxml/jackson/core/jackson-databind/2.3.0/jackson-databind-2.3.0.jar,$REPO/com/fasterxml/jackson/core/jackson-annotations/2.3.0/jackson-annotations-2.3.0.jar,$REPO/com/fasterxml/jackson/core/jackson-core/2.3.0/jackson-core-2.3.0.jar,$REPO/colt/colt/1.2.0/colt-1.2.0.jar,$REPO/concurrent/concurrent/1.3.4/concurrent-1.3.4.jar,$REPO/org/apache/mesos/mesos/0.18.1/mesos-0.18.1-shaded-protobuf.jar,$REPO/io/netty/netty-all/4.0.17.Final/netty-all-4.0.17.Final.jar,$REPO/com/clearspring/analytics/stream/2.5.1/stream-2.5.1.jar,$REPO/com/codahale/metrics/metrics-core/3.0.0/metrics-core-3.0.0.jar,$REPO/com/codahale/metrics/metrics-jvm/3.0.0/metrics-jvm-3.0.0.jar,$REPO/com/codahale/metrics/metrics-json/3.0.0/metrics-json-3.0.0.jar,$REPO/com/codahale/metrics/metrics-graphite/3.0.0/metrics-graphite-3.0.0.jar,$REPO/org/tachyonproject/tachyon/0.4.1-thrift/tachyon-0.4.1-thrift.jar,$REPO/org/apache/ant/ant/1.9.0/ant-1.9.0.jar,$REPO/org/apache/ant/ant-launcher/1.9.0/ant-launcher-1.9.0.jar,$REPO/org/spark-project/pyrolite/2.0.1/pyrolite-2.0.1.jar,$REPO/net/sf/py4j/py4j/0.8.1/py4j-0.8.1.jar,$REPO/it/unimi/dsi/fastutil/6.4.4/fastutil-6.4.4.jar,$REPO/com/twitter/parquet-avro/1.4.3/parquet-avro-1.4.3.jar,$REPO/com/twitter/parquet-column/1.4.3/parquet-column-1.4.3.jar,$REPO/com/twitter/parquet-common/1.4.3/parquet-common-1.4.3.jar,$REPO/com/twitter/parquet-encoding/1.4.3/parquet-encoding-1.4.3.jar,$REPO/com/twitter/parquet-generator/1.4.3/parquet-generator-1.4.3.jar,$REPO/com/twitter/parquet-hadoop/1.4.3/parquet-hadoop-1.4.3.jar,$REPO/com/twitter/parquet-jackson/1.4.3/parquet-jackson-1.4.3.jar,$REPO/com/twitter/parquet-format/2.0.0/parquet-format-2.0.0.jar,$REPO/fi/tkk/ics/hadoop/bam/hadoop-bam/6.2/hadoop-bam-6.2.jar,$REPO/picard/picard/1.107/picard-1.107.jar,$REPO/cofoja/cofoja/1.0/cofoja-1.0.jar,$REPO/tribble/tribble/1.107/tribble-1.107.jar,$REPO/variant/variant/1.107/variant-1.107.jar,$REPO/samtools/samtools/1.107/samtools-1.107.jar,$REPO/org/apache/httpcomponents/httpclient/4.2.6/httpclient-4.2.6.jar,$REPO/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar,$REPO/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar,$REPO/commons-codec/commons-codec/1.6/commons-codec-1.6.jar,$REPO/com/netflix/servo/servo-core/0.5.5/servo-core-0.5.5.jar,$REPO/com/google/code/findbugs/annotations/2.0.0/annotations-2.0.0.jar,$REPO/org/scala-lang/scala-library/2.10.3/scala-library-2.10.3.jar,$REPO/org/slf4j/slf4j-log4j12/1.7.5/slf4j-log4j12-1.7.5.jar,$REPO/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar,$REPO/log4j/log4j/1.2.17/log4j-1.2.17.jar,$REPO/args4j/args4j/2.0.23/args4j-2.0.23.jar,$REPO/org/fusesource/scalate/scalate-core_2.10/1.6.1/scalate-core_2.10-1.6.1.jar,$REPO/org/fusesource/scalate/scalate-util_2.10/1.6.1/scalate-util_2.10-1.6.1.jar,$REPO/org/scala-lang/scala-compiler/2.10.0/scala-compiler-2.10.0.jar,$REPO/org/scala-lang/scala-reflect/2.10.0/scala-reflect-2.10.0.jar,$REPO/org/scalatra/scalatra-json_2.10/2.3.0/scalatra-json_2.10-2.3.0.jar,$REPO/org/json4s/json4s-core_2.10/3.2.10/json4s-core_2.10-3.2.10.jar,$REPO/org/json4s/json4s-ast_2.10/3.2.10/json4s-ast_2.10-3.2.10.jar,$REPO/org/scala-lang/scalap/2.10.0/scalap-2.10.0.jar,$REPO/org/scalatra/scalatra_2.10/2.3.0/scalatra_2.10-2.3.0.jar,$REPO/org/scalatra/scalatra-common_2.10/2.3.0/scalatra-common_2.10-2.3.0.jar,$REPO/org/clapper/grizzled-slf4j_2.10/1.0.2/grizzled-slf4j_2.10-1.0.2.jar,$REPO/org/scalatra/rl/rl_2.10/0.4.10/rl_2.10-0.4.10.jar,$REPO/com/googlecode/juniversalchardet/juniversalchardet/1.0.3/juniversalchardet-1.0.3.jar,$REPO/eu/medsea/mimeutil/mime-util/2.1.3/mime-util-2.1.3.jar,$REPO/joda-time/joda-time/2.3/joda-time-2.3.jar,$REPO/org/joda/joda-convert/1.6/joda-convert-1.6.jar,$REPO/org/bdgenomics/adam/adam-cli/0.12.2-SNAPSHOT/adam-cli-0.12.2-SNAPSHOT.jar \
--properties-file /Users/laserson/repos/adam/examples/adam-examples.conf
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment