center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

feat(backend): spark

Browse Source
This commit is contained in:
junjie 2021-03-29 18:21:22 +08:00
parent 2e8e9cc8e9
commit 5fcc97565d
1 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/src/main/java/io/dataease/service/spark/SparkCalc.java
View File

@ -16,7 +16,10 @@ import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function; import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.*; import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.types.StructType;
@ -28,8 +31,6 @@ import java.util.ArrayList;
import java.util.Base64; import java.util.Base64;
import java.util.List; import java.util.List;
import static org.reflections8.Reflections.collect;
/** /**
* @Author gin * @Author gin
* @Date 2021/3/26 3:49 下午 * @Date 2021/3/26 3:49 下午
@ -98,8 +99,7 @@ public class SparkCalc {
List<String[]> data = new ArrayList<>(); List<String[]> data = new ArrayList<>();
// transform // transform
JavaRDD<Row> rowJavaRDD = sql.javaRDD(); List<Row> list = sql.javaRDD().collect();
List<Row> list = rowJavaRDD.collect();
for (Row row : list) { for (Row row : list) {
String[] r = new String[row.length()]; String[] r = new String[row.length()];
for (int i = 0; i < row.length(); i++) { for (int i = 0; i < row.length(); i++) {