3.win10 下 python3 爬虫美女图片逐步优化（多线程版本）

# coding: utf-8
import re
import os
import time
import threading
from multiprocessing import Pool, cpu_count
import requests
from bs4 import BeautifulSoup


HEADERS = {
    'X-Requested-With': 'XMLHttpRequest',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 '
                  '(KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
    'Referer': 'http://www.mzitu.com'
}
save_path='D:\data\crawl\meizitujk\\'
base_url = 'http://m.772586.com'

lock = threading.Lock()     # 全局资源锁

def urls_crawler_index(url):
    """
    爬虫入口，主要爬取操作
    """
    try:
        html = requests.get(url, headers=HEADERS, timeout=50)
        html.encoding = 'utf-8'
        soup = BeautifulSoup(html.text, "lxml", from_encoding='utf-8')
        tags =soup.find('ul',class_='tag').find_all('li')
        allIndexList = []
        allIndexList.append(base_url+'/meinv/')
        allIndexList.append(base_url+'/mxmn/')
        for tag in tags:
            print tag.find('a')['href']
            allIndexList.append(base_url+tag.find('a')['href'])
        print "========"
        for index_detail_url in allIndexList:
            urls_crawler_index_list(index_detail_url)

    except Exception as e:
        print(e)


def urls_crawler_index_list(index_url):
    """
    爬虫入口，主要爬取操作
    """
    try:
        html = requests.get(index_url, headers=HEADERS, timeout=10)
        html.encoding = 'utf-8'
        soup = BeautifulSoup(html.text, "lxml", from_encoding='utf-8')

        max_page_txt = soup.find('div',class_='page').find('a',class_='allpage').text
        max_page=max_page_txt[max_page_txt.rfind('/')+1:]
        max_page=int(max_page)
        print(max_page)



        for xxx_page in xrange(max_page):
            if 0==xxx_page:
                craw_content_page_url=index_url+'index.html'
            else:
                craw_content_page_url = index_url + 'index_' + str(xxx_page+1) + '.html'

            html_content_page = requests.get(craw_content_page_url, headers=HEADERS, timeout=10)
            html_content_page.encoding = 'utf-8'
            soup_content_page = BeautifulSoup(html_content_page.text, "lxml", from_encoding='utf-8')
            index_page_details =soup_content_page.find('div',id='list').find('ul').find_all('li')
            detail_urls=[]
            for page_url_li in index_page_details:
                detail_crawl_url = base_url + page_url_li.find('a')['href']
                print("detail_crawl_url:" + detail_crawl_url)
                detail_urls.append(detail_crawl_url)
            try:
                pool.map(urls_crawler_page, detail_urls)
            except Exception:
                time.sleep(30)
                pool.map(urls_crawler_page, detail_urls)
    except Exception as e:
        print(e)

def urls_crawler_page(url):
    """
    爬虫入口，主要爬取操作
    """
    try:
        response = requests.get(url, headers=HEADERS, timeout=50)
        response.encoding = 'utf-8'
        r=response.text
        folder_name = BeautifulSoup(r, 'lxml').find(
            'div', class_="contimglist").find('img')['alt'].replace("?", " ")
        print(folder_name)
        with lock:
            # 套图里图片张数
            max_count = BeautifulSoup(r, 'lxml').find(
                'div', class_='page').find_all(
                'span')[0].next_sibling[1:]
            page_urls = []
            for i in range(1, int(max_count) + 1):
                if i==1:
                    page_urls.append(url[0:url.rfind('.html')] +'.html')
                else:
                    page_urls.append(url[0:url.rfind('.html')] +'_'+str(i)+'.html')
            url_detail_crawler(page_urls,folder_name)
    except Exception as e:
        print(e)



def url_detail_crawler(page_urls,folder_name):
    img_urls = []
    for i,page_url in enumerate(page_urls):
        time.sleep(0.25)
        result = requests.get(page_url, headers=HEADERS, timeout=10).text
        img_url = BeautifulSoup(result, 'lxml').find(
            'div', class_="contimglist").find('a').find('img')['src']
        img_urls.append(img_url)

    for cnt, url in enumerate(img_urls):
        save_pic(url, cnt,folder_name)

def save_pic(pic_src, pic_cnt,folder_name):
    """
    保存图片到本地
    """
    try:
        time.sleep(0.10)
        base_path = save_path + folder_name
        if not os.path.isdir(base_path):
            os.mkdir(base_path)
        img = requests.get(pic_src, headers=HEADERS, timeout=10)
        img_name = base_path+"\\"+"pic_cnt_{}.jpg".format(pic_cnt + 1)
        with open(img_name, 'ab') as f:
            f.write(img.content)
            print(img_name)
    except Exception as e:
        print(e)


if __name__ == "__main__":
    # urls = ['http://m.772586.com/qingchun/17454.html']
    # pool = Pool(processes=cpu_count())
    # try:
    #     pool.map(urls_crawler_page, urls)
    # except Exception:
    #     time.sleep(30)
    #     pool.map(urls_crawler_page, urls)
    #测试
    #
    # urls_crawler(url)
    # page_urls=['http://m.772586.com/qingchun/17454_2.html']
    # urls_crawler(page_urls)
    #从首页开始
    pool = Pool(processes=10)
    urls = 'http://m.772586.com/'
    urls_crawler_index(urls)

​