init pages service

.gitignore

@@ -28,3 +28,5 @@ build/Release
 # https://docs.npmjs.com/misc/faq#should-i-check-my-node-modules-folder-into-git
 node_modules
 
+pages
+res

app.js

@@ -0,0 +1,62 @@
+/**
+ *
+ * @authors Tony.Deng (dengtongyu123@163.com)
+ * @date    2017-05-16 16:25:01
+ * @version $Id$
+ */
+
+const app = require('express')();
+const http = require('http');
+const httpServer = http.createServer(app);
+const PORT = 8080;
+
+// const io = require('socket.io')(httpServer,{
+// 	origin:'*'
+// });
+
+httpServer.listen(PORT, function() {
+    console.log('HTTP Server is running on: http://localhost:%s', PORT);
+});
+
+require('./server/picturedata')(app);
+
+require('./server/musicdata')(app);
+
+require('./server/socket')(httpServer);
+
+// Welcome
+app.get('/', function(req, res) {
+    res.status(200).send('Welcome!');
+});
+
+app.get('/**/**', function (req, res) {
+	console.log(req.url);
+	let url = req.url.split('?')[0];
+	//判断是否有中文
+	let reg = /[\u4e00-\u9FA5]+/;
+	let isHans = reg.test(decodeURI(url));
+	if(isHans) {
+		url = decodeURI(url);
+	}
+	//console.log(__dirname+req.url);
+	res.sendFile(__dirname+url);
+
+});
+
+// io.on('connection', function(socket){
+// 	console.log('a user connected');
+// 	socket.on('chat_message', function(data){
+// 		console.log(data);
+// 	    io.emit('chat_message', data);
+// 	});
+
+// 	socket.on('get_online_users', function(data){
+// 		//console.log(io.sockets.sockets);
+
+// 		console.log(Object.keys(io.engine.clients));
+
+// 		data.userNum = Object.keys(io.engine.clients).length;
+// 		console.log(data);
+// 	    io.emit('get_online_users', data);
+// 	});
+// });

package.json

@@ -0,0 +1,21 @@
+{
+  "name": "node_server",
+  "version": "1.0.0",
+  "description": "node server",
+  "main": "app.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "dengtongyu",
+  "license": "ISC",
+  "devDependencies": {
+    "async": "^2.5.0",
+    "cheerio": "^1.0.0-rc.2",
+    "express": "^4.15.2",
+    "fs": "0.0.1-security",
+    "mysql": "^2.14.0",
+    "path": "^0.12.7",
+    "request": "^2.81.0",
+    "socket.io": "^2.0.1"
+  }
+}

server/forwarding.js

@@ -0,0 +1,78 @@
+/**
+ * 
+ * @authors Your Name (you@example.org)
+ * @date    2017-08-03 10:29:18
+ * @version $Id$
+ */
+
+const request = require('request');
+
+module.exports = {
+
+	getSearchSongList:function(param,req,callback){
+		//let proxy_url = 'http://songsearch.kugou.com/song_search_v2?keyword='+param+'&page=1&pagesize=30&userid=-1&clientver=&platform=WebFilter&tag=em&filter=2&iscorrection=1&privilege_filter=0&_='+Date.now();
+        let proxy_url = 'http://songsearch.kugou.com/song_search_v2?' + param + '&_='+Date.now();
+        let options = {
+            headers: {"Connection": "close"},
+            url: proxy_url,
+            method: req.method,
+            json: true,
+            body: req.body
+        };
+
+        console.log(req.method);
+
+        request(options, function(error, response, data){
+            if (!error && response.statusCode == 200) {
+                //console.log('------接口代理数据------',response,data);
+
+                if(callback) callback(data);
+                //res.json(data)
+            }
+        });
+	},
+
+    getNewSongList:function(param,req,callback){
+        let proxy_url = 'http://m.kugou.com/?' + param;
+        let options = {
+            headers: {"Connection": "close"},
+            url: proxy_url,
+            method: req.method,
+            json: true,
+            body: req.body
+        };
+
+        console.log(req.method);
+
+        request(options, function(error, response, data){
+            if (!error && response.statusCode == 200) {
+                //console.log('------接口代理数据------',response,data);
+
+                if(callback) callback(data);
+                //res.json(data)
+            }
+        });
+    },
+
+	getSongMsg:function(hash,req,callback){
+		let proxy_url = 'http://www.kugou.com/yy/index.php?r=play/getdata&hash='+hash;
+
+        let options = {
+            headers: {"Connection": "close"},
+            url: proxy_url,
+            method: req.method.toUpperCase(),
+            json: true,
+            body: req.body
+        };
+
+        request(options, function(error, response, data){
+            if (!error && response.statusCode == 200) {
+                //console.log('------接口数据------',data);
+
+                if(callback) callback(data);
+                //res.json(data)
+            }
+        });
+	}
+
+}

server/musicdata.js

@@ -0,0 +1,90 @@
+/**
+ *
+ * @authors Tony.Deng (dengtongyu123@163.com)
+ * @date    2017-11-07 17:54:22
+ * @version $Id$
+ */
+const forwarding = require('./forwarding');
+const url = require('url');
+
+ module.exports = function(app) {
+
+ 	app.get('/data/songSearch', function(req, res) {
+ 	    let param = url.parse(req.url).query;
+ 	    let json = {};
+ 	    if(param) {
+ 	        forwarding.getSearchSongList(param,req,function(data){
+ 	            //console.log('------接口数据------',data);
+ 	            if(data.status == 1) {
+ 	                json.desc = '';
+ 	                json.result = true;
+ 	                json.data = data.data;
+ 	                res.send(json);
+ 	            }else{
+ 	                json.desc = '请求出错，错误代码：'+data.err_code;
+ 	                json.result = false;
+ 	                json.data = null;
+ 	                res.send(json);
+ 	            }
+ 	        });
+ 	    }else{
+ 	        json.desc = '缺少参数';
+ 	        json.result = false;
+ 	        json.data = null;
+ 	        res.send(json);
+ 	    }
+ 	});
+
+ 	app.get('/data/newSong', function(req, res) {
+ 	    let param = url.parse(req.url).query;
+ 	    let json = {};
+ 	    if(param) {
+ 	        forwarding.getNewSongList(param,req,function(data){
+ 	            //console.log('------接口数据------',data.data);
+ 	            if(data.data && data.data.length && data.data.length > 0) {
+ 	                json.desc = '';
+ 	                json.result = true;
+ 	                json.data = data.data;
+ 	                res.send(json);
+ 	            }else{
+ 	                json.desc = '请求出错，错误代码：'+data.err_code;
+ 	                json.result = false;
+ 	                json.data = null;
+ 	                res.send(json);
+ 	            }
+ 	        });
+ 	    }else{
+ 	        json.desc = '缺少参数';
+ 	        json.result = false;
+ 	        json.data = null;
+ 	        res.send(json);
+ 	    }
+ 	});
+
+ 	app.get('/data/songMsg', function(req, res) {
+ 	    let hash = req.query.hash;
+ 	    let json = {};
+ 	    if(hash) {
+ 	        forwarding.getSongMsg(hash,req,function(data){
+ 	            //console.log('------接口数据------',data);
+ 	            if(data.status == 1) {
+ 	                json.desc = '';
+ 	                json.result = true;
+ 	                json.data = data.data;
+ 	                res.send(json);
+ 	            }else{
+ 	                json.desc = '请求出错，错误代码：'+data.err_code;
+ 	                json.result = false;
+ 	                json.data = null;
+ 	                res.send(json);
+ 	            }
+ 	        });
+ 	    }else{
+ 	        json.desc = '缺少参数';
+ 	        json.result = false;
+ 	        json.data = null;
+ 	        res.send(json);
+ 	    }
+ 	});
+
+ }

server/picturedata.js

@@ -0,0 +1,47 @@
+/**
+ *
+ * @authors Tony.Deng (dengtongyu123@163.com)
+ * @date    2017-11-08 11:37:22
+ * @version $Id$
+ */
+const sql = require('./sql');
+
+ module.exports = function(app) {
+
+ 	//数据接口
+    app.get('/data/getPictureList', function(req, res) {
+ 	   let start = req.query.start;
+ 	   let number = req.query.number;
+ 	   let json = {};
+ 	   if(start && number) {
+ 		   sql.getPictureList('record_time',start,number,function(list){
+ 			   if(list.length != 0) {
+ 				   let data = [];
+ 				   for (let i=0; i<list.length; i++) {
+ 					   let obj = {};
+ 					   obj.srcUrl = list[i].src_image_url;
+ 					   obj.localUrl = list[i].local_image_url;
+ 					   obj.srcGifUrl = list[i].src_gif_url;
+ 					   obj.localGifUrl = list[i].local_gif_url;
+ 					   data.push(obj);
+ 				   }
+ 				   json.desc = '';
+ 				   json.result = true;
+ 				   json.data = data;
+ 				   res.send(json);
+ 			   }else {
+ 				   json.desc = '没有数据了';
+ 				   json.result = false;
+ 				   json.data = null;
+ 				   res.send(json);
+ 			   }
+ 		   })
+ 	   }else{
+ 		   json.desc = '缺少参数';
+ 		   json.result = false;
+ 		   json.data = null;
+ 		   res.send(json);
+ 	   }
+    });
+
+ }

server/socket.js

@@ -0,0 +1,49 @@
+/**
+ * 
+ * @authors Tony.Deng (dengtongyu123@163.com)
+ * @date    2017-06-01 11:00:22
+ * @version $Id$
+ */
+
+
+module.exports = function(server) {
+
+	const io = require('socket.io')(server,{
+		origin:'*'
+	});
+
+	let onlineUsers = 0;
+
+	const chat = io.of('/chat').on('connection', function(socket){
+
+		console.log('a user connected');
+
+		socket.on('client_chat_message', function(data){
+			console.log(data);
+		    chat.emit('broadcast_chat_message', data);
+		});
+
+		socket.on('client_get_online_users', function(data){
+			data.onlineUsers = onlineUsers;
+			console.log(data);
+			//单播
+		    socket.emit('broadcast_online_users', data);
+		});
+
+		
+	});
+
+	function getOnlineUsers() {
+		setTimeout(function(){
+			let clients = Object.keys(io.engine.clients).length;
+			if(onlineUsers != clients) {
+				onlineUsers = clients;
+				chat.emit('online_users_change', {onlineUsers:onlineUsers});
+			}
+			getOnlineUsers();
+		},1000);
+	} 
+	
+	getOnlineUsers();
+
+}

server/sql.js

@@ -0,0 +1,66 @@
+/**
+ * 
+ * @authors Tony.Deng (dengtongyu123@163.com)
+ * @date    2017-07-21 10:54:14
+ * @version $Id$
+ */
+
+const mysql   = require('mysql');
+
+let mysqlConfig = {
+	host   : 'localhost',
+    user   : 'picture',
+    password : 'tony112403',
+    port: '3306', 
+    database : 'picture'
+}
+
+module.exports = {
+
+	//获取connection
+	getConnection:function() {
+		
+		let connection = mysql.createConnection(mysqlConfig);
+
+		return connection;
+	},
+
+	//读取数据
+	getPictureList:function(sortField,start,number,callback){
+		let connection = mysql.createConnection(mysqlConfig);
+		connection.connect();
+
+		let  sql = 'select * from beauty order by '+sortField+' desc limit '+start+','+number+';';
+		//查
+		connection.query(sql,function (err, result) {
+        	if(err){
+          		console.log('[SELECT ERROR] - ',err.message);
+        		return;
+        	}
+ 			
+ 			if(callback) callback(result);
+		});
+ 
+		connection.end();
+	},
+
+	//插入数据
+	insertDataToBeauty:function(elementId,fileName,imgsrc,gifFileName,orgUrl,connection){
+		let date = new Date();
+
+		//在mysql中增加记录
+        let  addSql = 'insert into beauty(element_id,local_image_url,src_image_url,local_gif_url,src_gif_url,record_time) VALUES(?,?,?,?,?,?)';
+        let  addSqlParams = [elementId,'/'+fileName,imgsrc,gifFileName?('/'+gifFileName):null,orgUrl,date];
+        connection.query(addSql,addSqlParams,function (err, result) {
+            if(err){
+                console.log('[INSERT ERROR] - ',err.message);
+                return;
+            }        
+ 
+            console.log('--------------------------INSERT----------------------------');
+            //console.log('INSERT ID:',result.insertId);        
+            console.log('INSERT ID:',result);        
+            console.log('-----------------------------------------------------------------\n\n');  
+        });
+	}
+}

webcrawler.js

@@ -0,0 +1,124 @@
+/**
+ * 
+ * @authors Tony.Deng (dengtongyu123@163.com)
+ * @date    2017-07-19 13:33:47
+ * @version $Id$
+ */
+
+const request = require('request');
+const cheerio = require('cheerio');
+const path = require('path');
+const fs = require('fs');
+const async = require('async');
+const sql   = require('./server/sql');
+
+let options = [];  //用于存储网址链接的数组
+let n = 0;
+let m = 0;
+
+let elementId = 0;
+
+
+//先生成图片地址链接的数组
+for (let i = 191; i <196; i++) {
+    let obj = {
+        url: 'http://jandan.net/ooxx/page-' + i,
+        headers:{
+            'User-Agent': 'request'
+        }
+    }
+    options.push(obj);
+}
+
+let connection = sql.getConnection();
+
+connection.connect();
+
+//用来处理这个调用逻辑的总函数
+function all(err, res, body) {
+    let $ = cheerio.load(body);
+    n=n+$(".commentlist img").length;
+    console.log(n);
+
+    $(".commentlist li").each(function(index,element){
+        elementId = parseInt($(this).attr('id').split('-')[1]);
+        
+        // sql.getPictureList('element_id',0,1,function(list){
+        //     if(list.length != 0) {
+        //         console.log(list)
+        //         if(elementId > list[0].element_id) {
+
+        //         }
+        //     }
+        // });
+
+        // return;
+        // if(_elementId > elementId) {
+        //     elementId = _elementId;
+        // }else{
+        //     return;
+        // }
+
+        $(this).find('img').each(function (i, ele) {
+            saveImgAndDownload($(this));
+        });
+    });    
+}
+
+//格式化图片名称
+function FileName(url) {
+    let fileName = path.basename(url);
+    return fileName;
+}
+//利用fs模块download图片
+function downloadImg(url, filename, callback) {
+    let stream = fs.createWriteStream(filename);
+    request(url).on('error',function(){
+        console.log('done no 失败！！！');
+        m++;
+    }).pipe(stream).on('close', callback);
+}
+
+//获取图片下载，并保存数据库
+function saveImgAndDownload(element) {
+    let imgsrc = 'http:' + element.attr('src');
+    let orgUrl = element.attr('org_src')?('http:' + element.attr('org_src')):null;
+
+    let fileName = 'res/images/webcrawler/' + FileName(imgsrc.toString());
+    let gifFileName = null;
+
+    if(orgUrl) {
+        gifFileName = 'res/images/webcrawler/gif/' + FileName(orgUrl.toString());
+            downloadImg(orgUrl, gifFileName, function () {
+            console.log(gifFileName + ' gif download 完成');
+        });
+    }
+
+    //下载文件操作
+    downloadImg(imgsrc, fileName, function () {
+        console.log(fileName + ' download 完成');
+
+        sql.insertDataToBeauty(elementId,fileName,imgsrc,gifFileName,orgUrl,connection);
+
+        m++;
+        console.log(m);
+        if(m >= n){
+            connection.end();
+        }
+
+    });
+}
+ 
+
+//利用async的mapLimit方法实现限定并发数为3的调用
+async.mapLimit(options,3, function (option, callback) {
+    request(option, all);
+    callback(null);
+}, function (err, result) {
+    if (err) {
+        console.log(err);
+    } else {
+        // console.log(result);
+        console.log('全部检索完毕');
+    }
+})