def process_item(self, item, spider):
print("")
print(item["file_url"], item["name"])
key_word = {"file_url":item["file_url"] , "name": item["name"]}
res = self.db.find(key_word)
if list(res):
print("")
raise DropItem("Duplicate item found: %s" % item)
else:
print("*******************************************************************************")
self.db.insert({"file_url": item["file_url"], "name": item["name"]})
return item
my friend says it"s slow for me to find it like this, so let me build an index? Is it a _ id index?