Reptile efficiency improvement method

coroutines ： In function ( Special functions ) When defining , Use async modification , After function call , Internal statements do not execute immediately , It will return a process object
Task object ： Task object = Advanced collaboration objects ( Further encapsulation )= Special functions , The task object must be registered in the time cycle object , Bind callback to task object : Crawler data analysis
The event loop ： As a container for loading task objects , When the event loop object is started , The task object stored inside will execute asynchronously

First flask service

from flask import Flask
import time

app = Flask(__name__)

@app.route('/ Zhang San ')
def index_bobo():
  time.sleep(2)
  return 'hello  Zhang San !'

@app.route('/ Li Si ')
def index_jay():
  time.sleep(2)
  return 'hello  Li Si !'

@app.route('/ Wang Wu ')
def index_tom():
  time.sleep(2)
  return 'hello  Wang Wu !'

if __name__ == '__main__':
  app.run(threaded=True)

One ,aiohttp modular + Single thread multitask asynchronous coroutine

import asyncio
import aiohttp
import requests
import time

start = time.time()
async def get_page(url):
  # page_text = requests.get(url=url).text
  # print(page_text)
  # return page_text
  async with aiohttp.ClientSession() as s: # Generate a session object 
    async with await s.get(url=url) as response:
      page_text = await response.text()
      print(page_text)
  return page_text

urls = [
 'http://127.0.0.1:5000/ Zhang San ',
  'http://127.0.0.1:5000/ Li Si ',
  'http://127.0.0.1:5000/ Wang Wu ',
]
tasks = []
for url in urls:
  c = get_page(url)
  task = asyncio.ensure_future(c)
  tasks.append(task)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))

end = time.time()
print(end-start)

Two ,aiohttp Module implements single thread + Multitask asynchronous process

import aiohttp
import asyncio
from lxml import etree
import time

start = time.time()
#  Special functions : Request sending and data capture 
#  Be careful async with await keyword 
async def get_request(url):
  async with aiohttp.ClientSession() as s:
    async with await s.get(url=url) as response:
      page_text = await response.text()
      return page_text    #  Return to page source code 

#  Callback function , Parsing data 
def parse(task):
  page_text = task.result()
  tree = etree.HTML(page_text)
  msg = "".join(tree.xpath('//text()'))
  print(msg)

urls = [
 'http://127.0.0.1:5000/ Zhang San ',
  'http://127.0.0.1:5000/ Li Si ',
  'http://127.0.0.1:5000/ Wang Wu ',
]
tasks = []
for url in urls:
  c = get_request(url)
  task = asyncio.ensure_future(c)
  task.add_done_callback(parse) # Bind callback function !
  tasks.append(task)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))

end = time.time()
print(end-start)

3、 ... and ,requests modular + Thread pool

import time
import requests
from multiprocessing.dummy import Pool

start = time.time()
urls = [
 'http://127.0.0.1:5000/ Zhang San ',
  'http://127.0.0.1:5000/ Li Si ',
  'http://127.0.0.1:5000/ Wang Wu ',
]
def get_request(url):
  page_text = requests.get(url=url).text
  print(page_text)
  return page_text

pool = Pool(3)
pool.map(get_request, urls)
end = time.time()
print(' Total time :', end-start)