Scrapy-redis reported an error

class jingzhun(RedisSpider):
    name = "jingzhun"
    -sharp start_urls = ["https://rong.36kr.com/"]

    redis_key = "spiderQueue"

    custom_settings = {
        "RANDOM_DELAY": 5,
        "SCHEDULER": "scrapy_redis.scheduler.Scheduler",
        "DUPEFILTER_CLASS": "scrapy_redis.dupefilter.RFPDupeFilter",
        "SCHEDULER_QUEUE_CLASS": "scrapy_redis.queue.SpiderPriorityQueue",
        "SCHEDULER_PERSIST": True,
        "ITEM_PIPELINES": {
            "scrapy_redis.pipelines.RedisPipeline": 400
        },
        "REDIS_URL": "redis://685e545f59634200:WutongaMINUS1968@685e545f59634200.m.cnqda.kvstore.aliyuncs.com:6379/27",
        "REDIS_HOST": "685e545f59634200.m.cnqda.kvstore.aliyuncs.com",
        "REDIS_PORT": 6379,
        "REDIS_PARAMS": {
            "password": "685e545f59634200:WutongaMINUS1968"
        }

    }

    def __init__(self, *args, **kwargs):
        print("")
        domain = kwargs.pop("domain", "")
        self.allowed_domains = filter(None, domain.split(","))
        super(jingzhun, self).__init__(*args, **kwargs)
        self.cookie_str = "acw_tc=b65cfd2515395760831792797e7a30fed7278a95d7c68d0dcad0b9cbc4ac1b; kwlo_iv=1h; kr_stat_uuid=TRRfp25694452; Hm_lvt_e8ec47088ed7458ec32cde3617b23ee3=1541062621,1541150329,1541661241; Hm_lpvt_e8ec47088ed7458ec32cde3617b23ee3=1541667148; download_animation=1; _kr_p_se=9867c144-9614-4298-96f7-0e46ed5efefe; krid_user_id=2014445492; krid_user_version=2; kr_plus_id=2014445492; kr_plus_token=8dnyAhS2t87wW1PU1p91L_jUAHPFmepeJJ75____; kr_plus_utype=0; device-uid=5fa2cef0-e334-11e8-978f-67115035d613"
        self.headers = {"Referer": "https://rong.36kr.com/list/detail&?sortField=HOT_SCORE",
                        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
            , "Host": "rong.36kr.com"}
        self.co_headers = {"Referer": "https://rong.36kr.com/list/detail&?sortField=HOT_SCORE",
                           "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
            , "Host": "rong.36kr.com", "cookie": self.cookie_str}
        self.js_read = open("./js/jingzhun.js", "r").read()

    def make_request_from_data(self, data):
        print("")


        str_data = bytes_to_str(data, self.redis_encoding)
        print("")
        print(str_data)
        url=json.loads(str_data)["data"]["parseUrl"]
        print("url ")
        print(url)
        return self.make_requests_from_url(url)

    def make_requests_from_url(self, url):
        yield scrapy.Request(
            url=url,
            callback=self.get_all_info,
            headers=self.headers,
            dont_filter=True,
            cookies=get_cookies(self.cookie_str),
        )

    def get_all_info(self, response):
        item = jingzhun_item()
        for data in json.loads(response.text)["data"]["pageData"]["data"]:
            id = data["id"]
            brief = data["brief"]
            industryStr = data["industryStr"]
            tags = data["tags"]
            name = data["name"]
            cityStr = data["cityStr"]
            print("")
            item["id"] = id
            item["brief"] = brief
            item["industryStr"] = industryStr
            item["tags"] = tags
            item["name"] = name
            item["cityStr"] = cityStr

            print(id, brief, industryStr, tags, name, cityStr)
            asTs, asEncryptedTs = execjs.compile(self.js_read).call("p")
            company_link = "https://rong.36kr.com/n/api/company/{}?asEncryptedTs={}&asTs={}".format(id, asEncryptedTs,
                                                                                                    asTs)
            yield scrapy.Request(
                url=company_link,
                callback=self.get_company_info,
                headers=self.headers,
                -sharpdont_filter=True,
                cookies=get_cookies(self.cookie_str),
                meta={"id": id, "item": item},
            )





2018-11-18 23:59:52 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2018-11-18 23:59:52 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6039


{"type":2,"data":{"parseUrl":"https://rong.36kr.com/n/api/column/0/company?sortField=HOT_SCORE&p=1"},"callBack":"https://cw-project.dev.ethercap.com/spider/source/parse-website-content?id=2566&type=2"}
url 
https://rong.36kr.com/n/api/column/0/company?sortField=HOT_SCORE&p=1
2018-11-18 23:59:53 [scrapy.utils.signal] ERROR: Error caught on signal handler: <bound method RefererMiddleware.request_scheduled of <scrapy.spidermiddlewares.referer.RefererMiddleware object at 0x7ff6dbe449b0>>
Traceback (most recent call last):
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/scrapy/utils/signal.py", line 30, in send_catch_log
    *arguments, **named)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/pydispatch/robustapply.py", line 55, in robustApply
    return receiver(*arguments, **named)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/scrapy/spidermiddlewares/referer.py", line 343, in request_scheduled
    redirected_urls = request.meta.get("redirect_urls", [])
AttributeError: "generator" object has no attribute "meta"
Unhandled Error
Traceback (most recent call last):
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/scrapy/commands/crawl.py", line 58, in run
    self.crawler_process.start()
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/scrapy/crawler.py", line 291, in start
    reactor.run(installSignalHandlers=False)  -sharp blocking call
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/base.py", line 1261, in run
    self.mainLoop()
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/base.py", line 1270, in mainLoop
    self.runUntilCurrent()
--- <exception caught here> ---
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/base.py", line 896, in runUntilCurrent
    call.func(*call.args, **call.kw)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/scrapy/utils/reactor.py", line 41, in __call__
    return self._func(*self._a, **self._kw)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/scrapy/core/engine.py", line 135, in _next_request
    self.crawl(request, spider)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/scrapy/core/engine.py", line 210, in crawl
    self.schedule(request, spider)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/scrapy/core/engine.py", line 216, in schedule
    if not self.slot.scheduler.enqueue_request(request):
  File "/usr/lib/python3.4/site-packages/scrapy_redis/scheduler.py", line 162, in enqueue_request
    if not request.dont_filter and self.df.request_seen(request):
builtins.AttributeError: "generator" object has no attribute "dont_filter"

2018-11-18 23:59:53 [twisted] CRITICAL: Unhandled Error
Traceback (most recent call last):
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/scrapy/commands/crawl.py", line 58, in run
    self.crawler_process.start()
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/scrapy/crawler.py", line 291, in start
    reactor.run(installSignalHandlers=False)  -sharp blocking call
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/base.py", line 1261, in run
    self.mainLoop()
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/base.py", line 1270, in mainLoop
    self.runUntilCurrent()
--- <exception caught here> ---
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/base.py", line 896, in runUntilCurrent
    call.func(*call.args, **call.kw)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/scrapy/utils/reactor.py", line 41, in __call__
    return self._func(*self._a, **self._kw)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/scrapy/core/engine.py", line 135, in _next_request
    self.crawl(request, spider)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/scrapy/core/engine.py", line 210, in crawl
    self.schedule(request, spider)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/scrapy/core/engine.py", line 216, in schedule
    if not self.slot.scheduler.enqueue_request(request):
  File "/usr/lib/python3.4/site-packages/scrapy_redis/scheduler.py", line 162, in enqueue_request
    if not request.dont_filter and self.df.request_seen(request):
builtins.AttributeError: "generator" object has no attribute "dont_filter"
Python
Nov.25,2021

you replace the two yield in the code with return .

make_requests_from_url change the yield under this method to return, because the principle of de-duplication in scrapy-redis is to remove the weight according to the parameters of the request object, if you use yield, then this method becomes a generator object, so you can not get some parameters of the request object, so an error is reported.

Previous: Python pymysql data migration

Next: How does reactjs solve the problem that one of the two-tier nested components will fail?

css mysql arrays josn react html typescript webpack npm sass R objective-c .net sql-server jquery python-3.x angularjs django angular excel regex iphone ajax linux xml pandas vba spring database wordpress string wpf xcode windows bash postgresql oracle multithreading eclipse list firebase algorithm macos forms image scala visual-studio azure bootstrap spring-boot react-native python-2.7 docker performance function winforms matlab powershell apache dataframe api sqlite numpy rest shell selenium flutter dart maven loops qt swing android-studio csv express file class tensorflow sorting codeigniter perl
© 2021 CodesHelper
Advertising Contact us About us
Menu