Eamonn

人生苦短,我用Python

· Python · · 278次浏览

Scrapy分发

以下代码可以做到fuc1接收到了http://www.baidu.com的链接,fuc2接收到了http://www.bilibili.com的链接

# -*- coding: utf-8 -*-
import scrapy

class Fenfa(scrapy.Spider):
    name = 'fenfa_test'
    start_urls={
        "fuc1":"http://www.baidu.com",
        "fuc2":'http://www.bilibili.com'
    }


    def start_requests(self):
        for fuc,url in self.start_urls.items():
            yield scrapy.Request(
                url=url,
                callback=getattr(self,fuc)
            )

    def fuc1(self,response):
        print(response.url,1)

    def fuc2(self,response):
        print(response.url,2)



if __name__ == '__main__':
    from scrapy import cmdline
    cmdline.execute('scrapy crawl fenfa_test'.split())
评论 (0条)