苹果Mac专业的 Go 开发集成环境:JetBrains GoLand

Convolutional Neural Networks for Sentence Classification阅读梗概

  返回  

nodejs 采集新闻数据

2021/8/21 17:25:01 浏览:
var express = require( 'express' );
var $ = require( 'jQuery' );
var app = express();
var colors = require( 'colors' );
var message_list = require( './zui/message_list' );
var findData = require( './findData' );
//设置全局跨域访问 
app.all( '*' , function(req, res, next) { 
     res.header( "Access-Control-Allow-Origin" , "*" ); 
     res.header( "Access-Control-Allow-Headers" , "Origin, X-Requested-With, Content-Type, Accept" ); 
     res.header( "Access-Control-Allow-Methods" , "PUT,POST,GET,DELETE,OPTIONS" ); 
     res.header( "X-Powered-By" , ' 3.2.1'
     res.header( "Content-Type" , "application/json;charset=utf-8" );
     next(); 
});
//输出主页列表数据
app.get( '/getIndexData' , function (req, res) {
     findData.indexData(req, res);
});
/*
*输出列表数据。支持范围选择
*demo:http://localhost:3000/getList/0,9 前9条数据 从0开始
*demo: http://localhost:3000/getList/0  所有的数据  从0开始
*/
app.get(/^\/getList?(?:\/(\d+)(?:\,(\d+))?)?/, function (req, res) {
     var start=undefined,end=undefined;
     if (req.params.length== 1 ){
         start=req.params[ 0 ];
     } else if (req.params.length== 2 ){
         start=req.params[ 0 ];
         end=req.params[ 1 ];
     }
     findData.list(req, res,start,end,function(data){
         for (var i= 0 ,j=data.length;i<j;i++){ data[i].orderby= "1" ;= "" }= "" return = "" data;= "" });= "" 数据列表= "" var= "" newslistdata= "undefined;" app.get( '="" getnewslist' ,= "" function= "" (req,= "" res)= "" {= "" if (typeof= "" finddata.list(req,= "" res,undefined,undefined,function(data,length){= "" for (var= "" i= "0,j=data.length;i<j;i++){" data[i].summary= "暂无摘要" data[i].updateusername= "admin" getnewslist(req,= "" res);= "" } else {= "" 缓存数据列表= "" res){= "" pagenum= "req.query.pageNum,pageSize=req.query.pageSize;" start= "(pageNum-1)*pageSize,end=pageNum*pageSize;" newdata= "newsListData.slice(start," end);= "" res.send(json.stringify(newdata));= "" app.listen( 3000 );= "" console.log( "nodejs=" " was=" " start" .green);= "" <= "" pre= "" ><br>
<pre class = "brush:java;" >var http = require( 'http' ),
     fs = require( 'fs' ),
     jquery = fs.readFileSync( "lib/jquery.min.js" , "utf-8" ),
     jsdom = require( 'jsdom' );
function get(url, callback) {
     //使用代理
         var opt = {
         host: '127.0.0.1' ,
         port: '7070' ,
         method: 'get' , //这里是发送的方法
         path: url
     }
     //以下是接受数据的代码
     var req = http.request(opt, function(res) {
         res.setEncoding( 'utf8' );
         var html = '' ;
         res.on( 'data' , function(d) {
             html += d;
         }).on( 'end' , function() {
             jsdom.env({
                 html: html,
                 src: [jquery],
                 done: function(errors, window) {
                     var $ = window.$;
                     callback && callback(errors, $);
                     window.close(); // 释放window相关资源,否则将会占用很高的内存
                 }
             });
         });
     }).on( 'error' , function(e) {
         console.log( "Got error: " + e.message);
     })
     req.end();
}
//导出list
exports.indexData = function(request, response, start, end) {
     get( 'http://www.news.cn/edu/index.htm' , function(errors, $) {
         var $list = $( ".list" );
         var data = [];
         $list.each(function(index) {
             var $ this = $( this ),
                 $li = $ this .find( "li" ),
                 li_data = [];
             $li.each(function(index) {
                 var $a = $( this ).find( "a" );
                 var href=$a.attr( "href" ).replace( "http://news.xinhuanet.com/edu/" , "" );
                 var temp_href= "/" +href.replace(/[^c]*/, "" );
                 var date=href.replace(temp_href, "" ).replace( "/" , "-" );
                 if (date.length> 10 ){
                     date= "2013-01-04" ;
                 }
                 li_data.push({
                     title: $a.text(),
                     href: $a.attr( "href" ),
                     date:date,
                     id: index
                 });
             });
             data.push({
                 id: index,
                 list: li_data
             });
         });
         if (typeof start != "undefined" && typeof end == "undefined" ) {
             data = data.slice(start);
         } else if (typeof start != "undefined" && typeof end != "undefined" ) {
             data = data.slice(start, end);
         }
         response.send(JSON.stringify(data));
     });
}
//导出list
exports.list = function(request, response, start, end,callback) {
     get( 'http://www.news.cn/edu/index.htm' , function(errors, $) {
         var $li = $( ".list li" );
         var data = [];
         $li.each(function(index) {
             var $a = $( this ).find( "a" );
             var href=$a.attr( "href" ).replace( "http://news.xinhuanet.com/edu/" , "" );
             var temp_href= "/" +href.replace(/[^c]*/, "" );
             var date=href.replace(temp_href, "" ).replace( "/" , "-" );
             if (date.length> 10 ){
                 date= "2013-01-04" ;
             }
             
             data.push({
                 id: index,
                 title: $a.text(),
                 attachment: $a.attr( "href" ),
                 updateTime:date
             });
         });
         var newData=data;
         if (typeof start != "undefined" && typeof end == "undefined" ) {
             newData = data.slice(start);
         } else if (typeof start != "undefined" && typeof end != "undefined" ) {
             newData = data.slice(start, end);
         }
         if (typeof callback== "function" ){
             newData=callback(newData,data.length);
         }
         //response.header("Access-Control-Allow-Origin", "*");   //设置跨域访问 
         response.send(JSON.stringify(newData));
     });
}</pre><br>
访问地址<p></p>
<p></p><pre class = "brush:java;" >http: //localhost:3000/getIndexData
http: //localhost:3000/getList/0,4
http: //localhost:3000/getNewsList?pageNum=1&pageSize=10</pre><p></p>
<p>over<br>
</p>                        </j;i++){>
​​​​​​​

延伸阅读


▶ Walkthrough007

联系我们

如果您对我们的服务有兴趣,请及时和我们联系!

服务热线:18288888888
座机:18288888888
传真:
邮箱:888888@qq.com
地址:郑州市文化路红专路93号