SparkConf conf = new SparkConf().setAppName("DataFrameOps").setMaster("local");
JavaSparkContext sc = new JavaSparkContext(conf);
/**
* 若想使用SparkSQL必须创建SQLContext 必须是传入SparkContext 不能是SparkConf
*/
SQLContext sqlContext = new SQLContext(sc);
/**
* 创建一个本地的集合 集合中元素的格式 json 类型String
*/
List<String> nameList = Arrays.asList(
"{'name':'zhangsan', 'age':55}",
"{'name':'lisi', 'age':30}",
"{'name':'lisisi', 'age':30}",
"{'name':'wangwu', 'age':19}");
List<String> scoreList = Arrays.asList(
"{'name':'zhangsan','score':100}",
"{'name':'lisi','score':99}" );
/**
* 并行化成一个rdd 现在rdd中元素格式 json格式
*/
JavaRDD<String> nameRDD = sc.parallelize(nameList);
JavaRDD<String> scoreRDD = sc.parallelize(scoreList);
DataFrame nameDF = sqlContext.read().json(nameRDD);
DataFrame scoreDF = sqlContext.read().json(scoreRDD);
/**
* SELECT nameTable.name,nameTable.age,scoreTable.score
* FROM nameTable JOIN nameTable ON (nameTable.name == scoreTable.name)
*/
nameDF.join(scoreDF, nameDF.col("name").$eq$eq$eq(scoreDF.col("name")))
.select(nameDF.col("name"),nameDF.col("age"),scoreDF.col("score")).show();
nameDF.registerTempTable("name");
scoreDF.registerTempTable("score");
String sql = "SELECT name.name,name.age,score.score "
+ "FROM name join score ON (name.name = score.name)";
sqlContext.sql(sql).show();