-
Notifications
You must be signed in to change notification settings - Fork 22
Description
reproduction steps
Running spark application in standalone mode,when I initial the SparkContext,it will produce NullPointerException
using Scala (2.12.10),
Caused by: java.lang.NullPointerException
at scala.collection.mutable.ArrayOps$ofRef$.length$extension(ArrayOps.scala:204)
at scala.collection.mutable.ArrayOps$ofRef.length(ArrayOps.scala:204)
at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:35)
at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
at scala.collection.TraversableOnce.addString(TraversableOnce.scala:362)
at scala.collection.TraversableOnce.addString$(TraversableOnce.scala:358)
at scala.collection.mutable.ArrayOps$ofRef.addString(ArrayOps.scala:198)
at scala.collection.TraversableOnce.mkString(TraversableOnce.scala:328)
at scala.collection.TraversableOnce.mkString$(TraversableOnce.scala:327)
at scala.collection.mutable.ArrayOps$ofRef.mkString(ArrayOps.scala:198)
at scala.collection.TraversableOnce.mkString(TraversableOnce.scala:330)
at scala.collection.TraversableOnce.mkString$(TraversableOnce.scala:330)
at scala.collection.mutable.ArrayOps$ofRef.mkString(ArrayOps.scala:198)
at scala.reflect.runtime.ReflectionUtils$.inferClasspath$1(ReflectionUtils.scala:51)
at scala.reflect.runtime.ReflectionUtils$.show(ReflectionUtils.scala:62)
at scala.reflect.runtime.ReflectionUtils$.show(ReflectionUtils.scala:62)
at scala.reflect.runtime.ReflectionUtils$.show(ReflectionUtils.scala:62)
at scala.reflect.runtime.ReflectionUtils$.show(ReflectionUtils.scala:62)
at scala.reflect.api.JavaUniverse$JavaMirror.toString(JavaUniverse.scala:60)
at scala.reflect.api.JavaUniverse$JavaMirror.toString$(JavaUniverse.scala:60)
at scala.reflect.runtime.JavaMirrors$JavaMirror.toString(JavaMirrors.scala:68)
at java.lang.String.valueOf(String.java:3466)
at java.lang.StringBuilder.append(StringBuilder.java:537)
at scala.reflect.internal.Mirrors$RootsBase.$anonfun$getModuleOrClass$6(Mirrors.scala:65)
at scala.reflect.internal.Mirrors$RootsBase.getRequiredClass(Mirrors.scala:65)
at scala.reflect.internal.Mirrors$RootsBase.$anonfun$getClassIfDefined$3(Mirrors.scala:137)
at scala.reflect.internal.Mirrors$RootsBase.getClassIfDefined(Mirrors.scala:243)
at scala.reflect.internal.Mirrors$RootsBase.getClassIfDefined(Mirrors.scala:129)
at scala.reflect.internal.Definitions$DefinitionsClass.VarHandleClass$lzycompute(Definitions.scala:576)
at scala.reflect.internal.Definitions$DefinitionsClass.VarHandleClass(Definitions.scala:576)
at scala.reflect.runtime.JavaUniverseForce.force(JavaUniverseForce.scala:345)
at scala.reflect.runtime.JavaUniverseForce.force$(JavaUniverseForce.scala:18)
at scala.reflect.runtime.JavaUniverse.force(JavaUniverse.scala:30)
at scala.reflect.runtime.JavaUniverse.init(JavaUniverse.scala:162)
at scala.reflect.runtime.JavaUniverse.(JavaUniverse.scala:93)
at scala.reflect.runtime.package$.universe$lzycompute(package.scala:29)
at scala.reflect.runtime.package$.universe(package.scala:29)
at scala.reflect.runtime.package$.universe(package.scala:29)
at org.apache.spark.sql.catalyst.ScalaReflection$.(ScalaReflection.scala:50)
at org.apache.spark.sql.catalyst.ScalaReflection$.(ScalaReflection.scala)
at org.apache.spark.sql.catalyst.encoders.RowEncoder$.serializerFor(RowEncoder.scala:77)
at org.apache.spark.sql.catalyst.encoders.RowEncoder$.apply(RowEncoder.scala:66)
at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:99)
at org.apache.spark.sql.Dataset$$$Lambda$1092/0x0000000015495d70.apply(Unknown Source)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772)
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:96)
at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:615)
at org.apache.spark.sql.SparkSession$$Lambda$996/0x000000001547e8f0.apply(Unknown Source)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772)
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:610)
problem
After debugging the code ,we found that the root cause is in ReflectionUtils.inferClasspath method.
def inferClasspath(cl: ClassLoader): String = cl match {
case cl: java.net.URLClassLoader =>
(cl.getURLs mkString ",")
case cl if cl != null && isAbstractFileClassLoader(cl.getClass) =>
cl.asInstanceOf[{val root: scala.reflect.io.AbstractFile}].root.canonicalPath
case null =>
val loadBootCp = (flavor: String) => scala.util.Properties.propOrNone(flavor + ".boot.class.path")
loadBootCp("sun") orElse loadBootCp("java") getOrElse "<unknown>"
case _ =>
"<unknown>"
}
when we run the app in WebSphere ,the class loader WebSphere used is :
com.ibm.ws.bootstrap.ExtClassLoader ,it extends URLClassloader,but Override the getURLs() method,
public URL[] getURLs() {
return null;
}
it will return null, this will cause the NullPointerExcetion here;
And we have found the similar problem in guava issues here:
the solution is guava itself to judge weather the getURLs method
return null.
So we think maybe it is necessary for Scala to handle the problem