from fake_useragent import UserAgent
class RandomUserAgent(object):
def __init__(self, crawler):
super(RandomUserAgent, self).__init__()
self.ua = UserAgent(verify_ssl=False)
self.ua_type = crawler.settings.get("RANDOM_UA_TYPE", "random")
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def process_request(self, request):
def get_ua():
return getattr(self.ua, self.ua_type)
request.headers.setdefault("User_Agent", get_ua())
class RandomProxy(object):
def __init__(self, ip):
super(RandomProxy, self).__init__()
self.iplist = ip
@classmethod
def from_crawler(cls, crawler):
return cls(
iplist=crawler.settings.get("PROXY")
)
def process_request(self, request, spider):
proxy = self.ip
request.meta["proxy"] = proxyI always report an error when I write an agent on middleware in scrapy
uring handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/fake_useragent/utils.py", line 67, in get
context=context,
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 526, in open
response = self._open(req, data)
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 544, in _open
"_open", req)
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 504, in _call_chain
result = func(*args)
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 1361, in https_open
context=self._context, check_hostname=self._check_hostname)
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 1320, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 65] No route to host>
2018-10-09 14:52:25 [fake_useragent] DEBUG: Sleeping for 0.1 seconds
self._send_output(message_body, encode_chunked=encode_chunked)File "/ usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1026, in _ send_output
self.send(msg)File "/ usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/htt
p/client.py", line 964, in send
self.connect()
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1392, in connect
super().connect()
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 936, in connect
(self.host,self.port), self.timeout, self.source_address)
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/socket.py", line 724, in create_connection
raise err
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/socket.py", line 713, in create_connection
sock.connect(sa)
OSError: [Errno 65] No route to host