init pages service

webcrawler.js

@@ -0,0 +1,124 @@
+/**
+ * 
+ * @authors Tony.Deng (dengtongyu123@163.com)
+ * @date    2017-07-19 13:33:47
+ * @version $Id$
+ */
+
+const request = require('request');
+const cheerio = require('cheerio');
+const path = require('path');
+const fs = require('fs');
+const async = require('async');
+const sql   = require('./server/sql');
+
+let options = [];  //用于存储网址链接的数组
+let n = 0;
+let m = 0;
+
+let elementId = 0;
+
+
+//先生成图片地址链接的数组
+for (let i = 191; i <196; i++) {
+    let obj = {
+        url: 'http://jandan.net/ooxx/page-' + i,
+        headers:{
+            'User-Agent': 'request'
+        }
+    }
+    options.push(obj);
+}
+
+let connection = sql.getConnection();
+
+connection.connect();
+
+//用来处理这个调用逻辑的总函数
+function all(err, res, body) {
+    let $ = cheerio.load(body);
+    n=n+$(".commentlist img").length;
+    console.log(n);
+
+    $(".commentlist li").each(function(index,element){
+        elementId = parseInt($(this).attr('id').split('-')[1]);
+        
+        // sql.getPictureList('element_id',0,1,function(list){
+        //     if(list.length != 0) {
+        //         console.log(list)
+        //         if(elementId > list[0].element_id) {
+
+        //         }
+        //     }
+        // });
+
+        // return;
+        // if(_elementId > elementId) {
+        //     elementId = _elementId;
+        // }else{
+        //     return;
+        // }
+
+        $(this).find('img').each(function (i, ele) {
+            saveImgAndDownload($(this));
+        });
+    });    
+}
+
+//格式化图片名称
+function FileName(url) {
+    let fileName = path.basename(url);
+    return fileName;
+}
+//利用fs模块download图片
+function downloadImg(url, filename, callback) {
+    let stream = fs.createWriteStream(filename);
+    request(url).on('error',function(){
+        console.log('done no 失败！！！');
+        m++;
+    }).pipe(stream).on('close', callback);
+}
+
+//获取图片下载，并保存数据库
+function saveImgAndDownload(element) {
+    let imgsrc = 'http:' + element.attr('src');
+    let orgUrl = element.attr('org_src')?('http:' + element.attr('org_src')):null;
+
+    let fileName = 'res/images/webcrawler/' + FileName(imgsrc.toString());
+    let gifFileName = null;
+
+    if(orgUrl) {
+        gifFileName = 'res/images/webcrawler/gif/' + FileName(orgUrl.toString());
+            downloadImg(orgUrl, gifFileName, function () {
+            console.log(gifFileName + ' gif download 完成');
+        });
+    }
+
+    //下载文件操作
+    downloadImg(imgsrc, fileName, function () {
+        console.log(fileName + ' download 完成');
+
+        sql.insertDataToBeauty(elementId,fileName,imgsrc,gifFileName,orgUrl,connection);
+
+        m++;
+        console.log(m);
+        if(m >= n){
+            connection.end();
+        }
+
+    });
+}
+ 
+
+//利用async的mapLimit方法实现限定并发数为3的调用
+async.mapLimit(options,3, function (option, callback) {
+    request(option, all);
+    callback(null);
+}, function (err, result) {
+    if (err) {
+        console.log(err);
+    } else {
+        // console.log(result);
+        console.log('全部检索完毕');
+    }
+})