Spark context Web UI available at http://

val df = spark.sql("select type, value from t")
df.rdd.toDebugString
df.explain(true)
df.show

val df = spark.sql("select type, count(*) from t group by type")
df.rdd.toDebugString
df.explain(true)
df.show

scala> df.rdd.toDebugString
res0: String =
(2) MapPartitionsRDD[4] at rdd at <console>:26 []
 |  MapPartitionsRDD[3] at rdd at <console>:26 []
 |  MapPartitionsRDD[2] at rdd at <console>:26 []
 |  FileScanRDD[1] at rdd at <console>:26 []

scala> df.rdd.toDebugString
res3: String =
(400) MapPartitionsRDD[14] at rdd at <console>:26 []
  |   MapPartitionsRDD[13] at rdd at <console>:26 []
  |   MapPartitionsRDD[12] at rdd at <console>:26 []
  |   ShuffledRowRDD[11] at rdd at <console>:26 []
  +-(2) MapPartitionsRDD[10] at rdd at <console>:26 []
     |  MapPartitionsRDD[9] at rdd at <console>:26 []
     |  FileScanRDD[8] at rdd at <console>:26 []

scala> df.explain(true)
== Parsed Logical Plan ==
'Project ['type, 'value]
+- 'UnresolvedRelation `t`

== Analyzed Logical Plan ==
type: string, value: int
Project [type#20, value#21]
+- SubqueryAlias t
   +- Relation[position#18,id#19,type#20,value#21] parquet

== Optimized Logical Plan ==
Project [type#20, value#21]
+- Relation[position#18,id#19,type#20,value#21] parquet

== Physical Plan ==
*Project [type#20, value#21]
+- *FileScan parquet t[type#20,value#21] Batched: true, Format: Parquet, Location: InMemoryFileIndex[hdfs://.../t],
PartitionFilters: [], PushedFilters: [], ReadSchema: struct<type:string,value:int>

import org.apache.spark.sql.execution.debug._
df.debugCodegen

Container killed by YARN for exceeding memory limits.

Container killed on request. Exit code is 143

create table t_big as
select * from
(with t10 as
(
select 0 id union all select 1 union all select 2 union all select 3 union all select 4
union all select 5 id union all select 6 union all select 7 union all select 8 union all select 9
),
t1000 as
(select t1.id * 1e0 + t2.id * 1e1 + t3.id * 1e2 id from t10 t1, t10 t2, t10 t3)
select t1.id + t2.id * 1e3 id, lpad('x', 1e3, 'x') padding
  from t1000 t1, t1000 t2) t
order by 1;

spark.sql("""
select *
from
(select id, padding, rn
  from (select tt.*, row_number() over (order by id asc) rn from t_big tt) x) a
cross join  
(select id, padding, rn
  from (select tt.*, row_number() over (order by id asc) rn from t_big tt) x) b
""").show(1)

spark-shell --driver-memory 600M --executor-memory 600M

ERROR cluster.YarnScheduler: Lost executor nnn on qqq.com: Container marked as failed: container_eee on host: qqq.com. Exit status: 143.
Diagnostics: Container killed on request. Exit code is 143
Container exited with a non-zero exit code 143
Killed by external signal

spark-shell --driver-memory 10G --executor-memory 10G

+---+--------------------+---+---+--------------------+---+
| id|             padding| rn| id|             padding| rn|
+---+--------------------+---+---+--------------------+---+
|  0|xxxxxxxxxxxxxxxxx...|  1|  0|xxxxxxxxxxxxxxxxx...|  1|
+---+--------------------+---+---+--------------------+---+
only showing top 1 row