Spark small case - RDD, broadcast

RDD Small cases

object RDD01 {
    
  def main(args: Array[String]): Unit = {
    
    val sparkConf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("RDD01")
    val sc: SparkContext = new SparkContext(sparkConf)

    // Create... From the collection RDD,spark Two methods are provided ,parallelize and makeRDD,makeRDD Is to achieve parallelize
    sc.parallelize(
      List(1,2,3,4)
    ).filter(_>3)
      .collect()
      .foreach(println)

    sc.stop()
  }
}

broadcast Small cases

/** * broadCast A simple case  *  Broadcast variables are used to efficiently distribute large objects . Send a read-only value to all working nodes , *  For one or more Spark Operation and use . such as , If your application needs to send a large read-only query table to all nodes , *  Broadcast variables are easy to use . Using the same variable in multiple parallel operations , however  Spark Each task will be sent separately . */
object BroadCast01 {
    
  def main(args: Array[String]): Unit = {
    
    val sparkConf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("broadcast01")
    val sc: SparkContext = new SparkContext(sparkConf)

    val rdd1= sc.makeRDD(List(("a", 1), ("b", 2), ("c", 3), ("d", 4)))
    val list= List(("a", 5), ("b", 6), ("c",7))
    // Declare broadcast variables 
    val broadcast: Broadcast[List[(String,Int)]] = sc.broadcast(list)

    rdd1.map {
    
      case (key, num) => {
    
        var num2 = 0
        // Use broadcast variables 
        for ((k, v) <- broadcast.value) {
    
          if (key == k) {
    
            num2 =  num * v
          }
        }
        (key, num2)
      }
    }.filter(x=>x._2>0)  // Screening value>0 Of 
      .foreach(x=>{
    println(x._1+":"+x._2)})

    sc.stop()
  }
}