-
Notifications
You must be signed in to change notification settings - Fork 0
/
HelloSpark.py
28 lines (20 loc) · 639 Bytes
/
HelloSpark.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
# python
import findspark
from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession
# [opt] Create a variable with the absolute path to the text file
logFile = "/home/workspace/Test.txt"
# create a Spark Session
spark = SparkSession.builder \
.master("local") \
.appName("Spark") \
.config("spark.sql.repl.eagerEval.enabled", True) \
.getOrCreate()
# set the log level to WARN
spark.sparkContext.setLogLevel("WARN")
logData = spark.read.text(logFile).cache()
# write functions
df = spark.createDataFrame([('1', 100), ('2', 200), ('3', 300)]).toDF("id", "value")
df
# stop Spark
spark.stop()