Scrapy writes multiple crawlers and reports errors at the same time, asking for guidance.

from scrapy.utils.project import get_project_settings
from Espider.spiders.chinafof import chinafof
from Espider.spiders.ifenxi import ifenxi
from Espider.spiders.vcbeat import vcbeat
from Espider.spiders.zdreview import zdreview
from scrapy.crawler import CrawlerProcess
from multiprocessing import Process

def run_crawl ():

process=CrawlerProcess(get_project_settings())
process.crawl(chinafof)
process.crawl(ifenxi)
process.crawl(vcbeat)
process.crawl(zdreview)
process.start()


if _ _ name__=="__main__":

[Process(target=run_crawl).start() for _ in range(4)]


File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/log.py", line 86, in callWithContext
    return context.call({ILogContext: newCtx}, func, *args, **kw)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/context.py", line 122, in callWithContext
    return self.currentContext().callWithContext(ctx, func, *args, **kw)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/context.py", line 85, in callWithContext
    return func(*args,**kw)
--- <exception caught here> ---
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/posixbase.py", line 619, in _doReadOrWrite
    why = selectable.doWrite()
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/base.py", line 1186, in doWrite
    "doWrite called on a %s" % reflect.qual(self.__class__))
builtins.RuntimeError: doWrite called on a twisted.internet.tcp.Port

2018-12-02 17:25:54 [twisted] CRITICAL: Unhandled Error
Traceback (most recent call last):
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/log.py", line 103, in callWithLogger
    return callWithContext({"system": lp}, func, *args, **kw)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/log.py", line 86, in callWithContext
    return context.call({ILogContext: newCtx}, func, *args, **kw)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/context.py", line 122, in callWithContext
    return self.currentContext().callWithContext(ctx, func, *args, **kw)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/context.py", line 85, in callWithContext
    return func(*args,**kw)
--- <exception caught here> ---
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/posixbase.py", line 619, in _doReadOrWrite
    why = selectable.doWrite()
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/base.py", line 1186, in doWrite
    "doWrite called on a %s" % reflect.qual(self.__class__))
builtins.RuntimeError: doWrite called on a twisted.internet.tcp.Port

Dec.31,2021
Menu