About the problem that the mapLimit method of async is stuck.

the code is as follows:

var cheerio     = require("cheerio");
var superagent1 = require("superagent");
var eventproxy  = require("eventproxy");
var async       = require("async");
var utils       = require("./utils");
var install     = require("superagent-charset");
var fs             = require("fs");
var superagent  = install(superagent1);
var current_book = {};
var error_list = [];
var n=1;
superagent.get("http://www.biquku.com/0/330/").charset("gbk")
    .end(function (err, sres) {
      // 
      if (err) {
        console.log(err);
      }
      // sres.text  html  cheerio.load 
      //  jquery  `$`
      //  jquery 
      var $ = cheerio.load(sres.text);
      var urls = $("-sharplist a");  

      current_book.title = $("-sharpmaininfo h1").text()
      current_book.author = $("-sharpinfo p").eq(0).text()
      current_book.update_time = $("-sharpinfo p").eq(2).text()
      current_book.latest_chapter = $("-sharpinfo p").eq(3).html()
      current_book.intro = $("-sharpintro").html()
      current_book.chapters = [];

      for(var i = 0; i< urls.length; iPP){
        var url = urls[i]
        var _url = $(url).attr("href")+"";
        var num = _url.replace(".html","");
        var title = $(url).text();
        current_book.chapters.push({
          num: num,
          title: title,
          url: _url
        })
      }
      console.log(current_book.chapters.length);

      async.mapLimit(current_book.chapters, 200, function (url, callback) {
          fetchUrl(url, callback);
        }, function (err, result) {
          console.log("final:");
          console.log(error_list);
        });
    });

function fetchUrl(url, callback){
    console.log(nPP);
    if(n==1637){
        console.log(error_list)
    }
    superagent.get("http://www.biquku.com/0/330/" + url.num + ".html").charset("gbk").end(function(err,res){
        var timer = setTimeout(function(){
            callback(null);
        },500);
        if(!err){
            clearTimeout(timer)
            if(res){    
            if(res.text){
                var $ = cheerio.load(res.text);
                var content = $("-sharpcontent").html();
                fs.writeFile("dist/0/330/" + url.num + ".html", content, function (err) {
                    if (err) throw err;
                    callback(null);
                  });

                }
            }else{
                error_list.push(url);
                callback(null);
            }
        }else{
            clearTimeout(timer)
            callback(null);
        }
        
    })
}

the final callback of async.mapLimit, which does not execute the program, is in a state of suspended animation; it is displayed on the command line as follows:

clipboard.png

ask the great gods to analyze why this is so;

Oct.08,2021

I also encounter this problem, have you solved it?


I also use maplimit, but that's not how I use it.
each of my request will come back to html, and then cheerio into an array, directly callback,
and then wrap the request in maplimti to get a large array, mySql

.

if you look at the code building, you want to extract the chapter and then pick the content (similar). It is recommended to pick the chapter first, and then get the content after all the chapters are completed

Menu