scrapy genspider –t crawl <spider_name> <domains>

rules = (Rule(LinkExtractor(allow=r'http:///digimon/.*/index.html'), callback='parse_item', follow=False),)

# 信號，這個名詞在scrapy自定義拓展中很重要
from scrapy import signals
# 本地配置的類，代碼見后續(xù)，可以搭在自己的IP池上，也可以直接掛在收費IP（比如我）
from proxyhelper import Proxyhelper
# 多線程操作同一個對象需要鎖，用法就是實例化以后一鎖一釋放
from twisted.internet.defer import DeferredLock

class DoubanSpiderMiddleware(object): # spider中間件不設置
    pass

class DoubanDownloaderMiddleware(object):
    def __init__(self):
        # 對IP配置的代理和鎖都實例化
        self.helper = Proxyhelper()
        self.lock = DeferredLock()

    @classmethod
    def from_crawler(cls, crawler):  # 不修改
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # request的數(shù)據(jù)流到達下載中間件的時候出發(fā)
        self.lock.acquire()
        request.meta['Proxy'] = self.helper.get_proxy()
        self.lock.release()
        return None

    def process_response(self, request, response, spider):
        # 對響應判斷，如果不符合就換代理重新請求
        if response.status != 200:   
            self.lock.acquire()
            self.helper.update_proxy(request.meta['Proxy'])
            self.lock.release()
            return request
        return response

    def process_exception(self, request, exception, spider):
        self.lock.acquire()
        self.helper.update_proxy(request.meta['Proxy'])
        self.lock.release()
        return request

    def spider_opened(self, spider):  # 不修改
        spider.logger.info('Spider opened: %s' % spider.name)

# 載入本地的mysql持久化類，按需自己寫
from mysqlhelper import Mysqlhelper
# 載入ImagesPipeline便于重載，自定義一些功能
from scrapy.pipelines.images import ImagesPipeline
import hashlib
from scrapy.utils.python import to_bytes
from scrapy.http import Request

class DoubanImagesPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        request_lst = []
        for x in item.get(self.images_urls_field, []):
            req = Request(x)
            req.meta['movie_name'] = item['title']  # 獲取名字
            request_lst.append(req)
        return request_lst
    # 重載
    def file_path(self, request, response=None, info=None):
        image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
        return 'full/%s.jpg' % (request.meta['movie_name']) # 修改圖片名

# 無特殊，有些步驟在items已經(jīng)寫完，實現(xiàn)pipelines和items功能上的分離
class DoubanPipeline(object):
    def __init__(self):
        self.mysqlhelper = Mysqlhelper()

    def process_item(self, item, spider):
        if 'get_insert_sql_and_data' in dir(item):
            (insert_sql, data) = item.get_insert_sql_and_data()
            self.mysqlhelper.execute_sql(insert_sql, data)
        return item

from scrapy import signals
from message import Message

class MyExtension(object):
    def __init__(self, value):
        self.value = value

    @classmethod
    def from_crawler(cls, crawler):
        val = crawler.settings.getint('MMMM')
        ext = cls(val)

        crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)

        return ext

    def spider_opened(self, spider):
        print('spider running')

    def spider_closed(self, spider):
        message = Message('spider運行結束')
        message.push()
        print('spider closed')