更新时间:2023-10-20 13:47:46
您可以在 Spark 2.0 中尝试以下操作:
You can try the following in Spark 2.0:
def main(args: Array[String]) {
val conf = new SparkConf()
val sparkSession = SparkSession
.builder()
.appName("TestApp")
.config(conf)
.enableHiveSupport()
.getOrCreate()
val tableName = args(0) // passed as an argument
val tableData = sparkSession
.table(tableName)
.select($"IdNum", $"Name")
.filter($"IdNum" =!= "")
.map( x => (x.getString(0), x.getString(1)) ).collectAsMap()
................
...............
}`