124 lines
3.0 KiB
JavaScript
124 lines
3.0 KiB
JavaScript
/**
|
|
*
|
|
* @authors Tony.Deng (dengtongyu123@163.com)
|
|
* @date 2017-07-19 13:33:47
|
|
* @version $Id$
|
|
*/
|
|
|
|
const request = require('request');
|
|
const cheerio = require('cheerio');
|
|
const path = require('path');
|
|
const fs = require('fs');
|
|
const async = require('async');
|
|
const sql = require('./server/sql');
|
|
|
|
let options = []; //用于存储网址链接的数组
|
|
let n = 0;
|
|
let m = 0;
|
|
|
|
let elementId = 0;
|
|
|
|
|
|
//先生成图片地址链接的数组
|
|
for (let i = 191; i <196; i++) {
|
|
let obj = {
|
|
url: 'http://jandan.net/ooxx/page-' + i,
|
|
headers:{
|
|
'User-Agent': 'request'
|
|
}
|
|
}
|
|
options.push(obj);
|
|
}
|
|
|
|
let connection = sql.getConnection();
|
|
|
|
connection.connect();
|
|
|
|
//用来处理这个调用逻辑的总函数
|
|
function all(err, res, body) {
|
|
let $ = cheerio.load(body);
|
|
n=n+$(".commentlist img").length;
|
|
console.log(n);
|
|
|
|
$(".commentlist li").each(function(index,element){
|
|
elementId = parseInt($(this).attr('id').split('-')[1]);
|
|
|
|
// sql.getPictureList('element_id',0,1,function(list){
|
|
// if(list.length != 0) {
|
|
// console.log(list)
|
|
// if(elementId > list[0].element_id) {
|
|
|
|
// }
|
|
// }
|
|
// });
|
|
|
|
// return;
|
|
// if(_elementId > elementId) {
|
|
// elementId = _elementId;
|
|
// }else{
|
|
// return;
|
|
// }
|
|
|
|
$(this).find('img').each(function (i, ele) {
|
|
saveImgAndDownload($(this));
|
|
});
|
|
});
|
|
}
|
|
|
|
//格式化图片名称
|
|
function FileName(url) {
|
|
let fileName = path.basename(url);
|
|
return fileName;
|
|
}
|
|
//利用fs模块download图片
|
|
function downloadImg(url, filename, callback) {
|
|
let stream = fs.createWriteStream(filename);
|
|
request(url).on('error',function(){
|
|
console.log('done no 失败！！！');
|
|
m++;
|
|
}).pipe(stream).on('close', callback);
|
|
}
|
|
|
|
//获取图片下载，并保存数据库
|
|
function saveImgAndDownload(element) {
|
|
let imgsrc = 'http:' + element.attr('src');
|
|
let orgUrl = element.attr('org_src')?('http:' + element.attr('org_src')):null;
|
|
|
|
let fileName = 'res/images/webcrawler/' + FileName(imgsrc.toString());
|
|
let gifFileName = null;
|
|
|
|
if(orgUrl) {
|
|
gifFileName = 'res/images/webcrawler/gif/' + FileName(orgUrl.toString());
|
|
downloadImg(orgUrl, gifFileName, function () {
|
|
console.log(gifFileName + ' gif download 完成');
|
|
});
|
|
}
|
|
|
|
//下载文件操作
|
|
downloadImg(imgsrc, fileName, function () {
|
|
console.log(fileName + ' download 完成');
|
|
|
|
sql.insertDataToBeauty(elementId,fileName,imgsrc,gifFileName,orgUrl,connection);
|
|
|
|
m++;
|
|
console.log(m);
|
|
if(m >= n){
|
|
connection.end();
|
|
}
|
|
|
|
});
|
|
}
|
|
|
|
|
|
//利用async的mapLimit方法实现限定并发数为3的调用
|
|
async.mapLimit(options,3, function (option, callback) {
|
|
request(option, all);
|
|
callback(null);
|
|
}, function (err, result) {
|
|
if (err) {
|
|
console.log(err);
|
|
} else {
|
|
// console.log(result);
|
|
console.log('全部检索完毕');
|
|
}
|
|
}) |