Spark case - wordcount

local Local mode

add to pom rely on

<properties>
    <scala.version>2.12.0</scala.version>
  </properties>

<dependencies>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
      <version>${scala.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.12</artifactId>
      <version>3.0.0</version>
    </dependency>
  </dependencies>

Prepare a file by yourself word.txt

// establish spark Run configuration object 
    val sparkConf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCount")

    // establish spark Context object ( Connection object )
    val sc: SparkContext = new SparkContext(sparkConf)

    // Read the file 
    val file: RDD[String] = sc.textFile("src/main/inputfile/word.txt")
	// If the document is in hdfs On 
	// val file: RDD[String] = sc.textFile("hdfs://ip Address :9820/study/sparktest/word.txt")

    // Calculation 
    file.flatMap(_.split(" "))
      .map((_,1))
      .reduceByKey(_+_)
      .collect()
      .foreach(println)

    sc.stop()

If you want to save the results

 sc.textFile("src/main/inputfile/word.txt")
      .flatMap(_.split(" "))
      .map((_, 1))
      .reduceByKey(_ + _)
      .repartition(1)
        .saveAsTextFile(" Address ")