分别用BeautifulSoup和scrapy爬取某一城市天气预报

分别用BeautifulSoup和scrapy爬取某一城市天气预报

爬取网站：中国天气网 http://www.weather.com.cn
此次我们以北京为例。
1、首先我们搜索进入到北京页面:
http://www.weather.com.cn/weather/101010100.shtml?from=cityListCmp
然后分析页面源代码构造
BeautifulSoup

from urllib import request
from bs4 import BeautifulSoup
from bs4 import UnicodeDammit
url = "http://www.weather.com.cn/weather/101010100.shtml"
try:
    headers = {
    "User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"}
    req = request.Request(url,headers=headers)
    data = request.urlopen(req)
    data = data.read()   #爬取该网页全部内容
    #print(data)
    dammit = UnicodeDammit(data,["Utf-8","gbk"])
    data = dammit.unicode_markup
    soup = BeautifulSoup(data,"lxml")
    lis = soup.select("ul[class='t clearfix'] li")     # [tagName][attName[=value]]
    # print(lis) # 查找到li所有内容
    for li in lis:
        try:
            data1 = li.select('h1')[0].text     #日期
            weather = li.select("p[class='wea']")[0].text       #天气
            tem = li.select("p[class='tem' i]")[0].text     #温度
            print(data1+" "+weather+" "+tem+"\n")
        except Exception as e1:
            print(e1)

except Exception as e2:
    print(e2)

scrapy
(其他步骤省略)

# -*- coding: utf-8 -*-
import scrapy
from ..items import TqpcItem
from scrapy.http import Request

class TqSpider(scrapy.Spider):
    name = 'tq'
    allowed_domains = ['weather.com']
    #start_urls = ['http://weather.com/']
    header = {
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"}
    def start_requests(self):
        url = "http://www.weather.com.cn/weather/101010100.shtml"
        yield Request(url, callback=self.parse)

    def parse(self, response):
        for i in range(0,8):
            item = TqpcItem()
            item["day"] = response.xpath("//ul[@class='t clearfix']/li/h1/text()")[i].extract()
            item["w1"] = response.xpath("//ul[@class='t clearfix']/li/p[@class='wea']/text()")[i].extract()
            item["w2"] = response.xpath("//ul[@class='t clearfix']/li/p[@class='tem']/span/text()")[i].extract()
            print(item["day"]+" "+item["w1"]+" "+item["w2"])
            yield item