Fix EMR Spark support by using Seq instead of ArrayBuffer for externalAccums

aboitreaud · aboitreaud · commit 0cb068786f1d · 2026-03-27T17:53:09.000+01:00
diff --git a/dd-java-agent/instrumentation/spark/spark_2.12/src/main/java/datadog/trace/instrumentation/spark/DatadogSpark212Listener.java b/dd-java-agent/instrumentation/spark/spark_2.12/src/main/java/datadog/trace/instrumentation/spark/DatadogSpark212Listener.java
@@ -14,6 +14,7 @@
 import org.apache.spark.util.AccumulatorV2;
 import scala.Function1;
 import scala.collection.JavaConverters;
+import scala.collection.Seq;
 import scala.collection.mutable.ArrayBuffer;
 
 /**
@@ -91,7 +92,9 @@ protected List<AccumulatorV2> getExternalAccumulators(TaskMetrics metrics) {
     }
 
     // withExternalAccums didn't work, try the legacy method
-    ArrayBuffer<AccumulatorV2> accumulators = methodLoader.invoke(externalAccums, metrics);
+    // Use Seq (not ArrayBuffer) since some Spark distributions (e.g. Amazon EMR) return a
+    // JListWrapper which implements Seq but cannot be cast to ArrayBuffer
+    Seq<AccumulatorV2> accumulators = methodLoader.invoke(externalAccums, metrics);
     if (accumulators != null && !accumulators.isEmpty()) {
       return JavaConverters.seqAsJavaList(accumulators);
     }
diff --git a/dd-java-agent/instrumentation/spark/spark_2.13/src/main/java/datadog/trace/instrumentation/spark/DatadogSpark213Listener.java b/dd-java-agent/instrumentation/spark/spark_2.13/src/main/java/datadog/trace/instrumentation/spark/DatadogSpark213Listener.java
@@ -13,6 +13,7 @@
 import org.apache.spark.sql.execution.metric.SQLMetricInfo;
 import org.apache.spark.util.AccumulatorV2;
 import scala.Function1;
+import scala.collection.Seq;
 import scala.collection.mutable.ArrayBuffer;
 import scala.jdk.javaapi.CollectionConverters;
 
@@ -91,7 +92,9 @@ protected List<AccumulatorV2> getExternalAccumulators(TaskMetrics metrics) {
     }
 
     // withExternalAccums didn't work, try the legacy method
-    ArrayBuffer<AccumulatorV2> accumulators = methodLoader.invoke(externalAccums, metrics);
+    // Use Seq (not ArrayBuffer) since some Spark distributions (e.g. Amazon EMR) return a
+    // JListWrapper which implements Seq but cannot be cast to ArrayBuffer
+    Seq<AccumulatorV2> accumulators = methodLoader.invoke(externalAccums, metrics);
     if (accumulators != null && !accumulators.isEmpty()) {
       return CollectionConverters.asJava(accumulators);
     }
