Skip to content

Commit

Permalink
set to empty string if error is null or undefined
Browse files Browse the repository at this point in the history
  • Loading branch information
@mike442144
mike442144 committed Apr 24, 2020
1 parent d50088e commit 661c1e1
Show file tree
Hide file tree
Showing 7 changed files with 306 additions and 305 deletions.
14 changes: 7 additions & 7 deletions client/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,17 +136,17 @@ module.exports = class Client extends Emitter{
// let bulk = this.db.collection(this.name).initializeUnorderedBulkOp();

// for(let i =0;i<items.length;i++){
// bulk.insert(items[i]);
// bulk.insert(items[i]);
// }

// this.dbTasks.push(2);
// logClient.silly('Inserting jobs to mongodb...');
// //bulk.execute(callback);
// bulk.execute(err => {
// logClient.silly('Inserted jobs. Here in callback.');
// if(err) logClient.error(err);
// self.dbTasks.pop();
// callback();
// logClient.silly('Inserted jobs. Here in callback.');
// if(err) logClient.error(err);
// self.dbTasks.pop();
// callback();
// });
});
}
Expand All @@ -164,7 +164,7 @@ module.exports = class Client extends Emitter{

_toJob(task){
let opt = typeof task.opt === 'string' ? {uri:task.opt} : task.opt;
let priority = task.hasOwnProperty('priority')? task.priority : (opt.hasOwnProperty('priority')?opt.priority:DEFAULT_PRIORITY);
let priority = Object.prototype.hasOwnProperty.call(task,'priority')? task.priority : (Object.prototype.hasOwnProperty.call(opt,'priority')?opt.priority:DEFAULT_PRIORITY);

if(priority !== DEFAULT_PRIORITY){
opt.priority = priority;
Expand Down Expand Up @@ -310,7 +310,7 @@ module.exports = class Client extends Emitter{
_goOne(job){
let argv = job.opt;
let fn = job.next;
let priority = this._getPriority(job.hasOwnProperty('priority')?job.priority:DEFAULT_PRIORITY);
let priority = this._getPriority(Object.prototype.hasOwnProperty.call(job,'priority')?job.priority:DEFAULT_PRIORITY);

let objJob = this.gearmanClient.submitJob(this.name+'_'+fn, JSON.stringify(argv),{background: false, priority:priority});
objJob.jobVO = job;
Expand Down
8 changes: 4 additions & 4 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "floodesh",
"version": "0.8.18",
"version": "0.8.19",
"description": "Floodesh is a distributed web spider/crawler written with Nodejs.",
"main": "index.js",
"scripts": {
"hint": "eslint lib/*.js worker/*.js client/*.js",
"hint": "eslint lib/*.js worker/*.js client/*.js tests/*.js",
"test": "./node_modules/mocha/bin/mocha --timeout 10000 tests/test-*.js"
},
"repository": {
Expand All @@ -29,10 +29,10 @@
},
"dependencies": {
"commander": "^2.9.0",
"floodesh-lib": "^1.1.1",
"floodesh-lib": "^1.1.3",
"gearman-node-bda": "^0.10.0",
"mongodb": "^2.1.4",
"ramda": "^0.26.1",
"ramda": "^0.27.0",
"seenreq": "^2.0.0",
"winston": "^2.1.1"
},
Expand Down
28 changes: 14 additions & 14 deletions tests/config.js
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
module.exports = {
production:{
gearman:{
servers:[{"host":"127.0.0.1"}]
production:{
gearman:{
servers:[{'host':'127.0.0.1'}]
},
retry:3,
mongodb:'mongodb://10.252.25.62:27017/gearman'
},
retry:3,
mongodb:"mongodb://10.252.25.62:27017/gearman"
},
development:{
gearman:{
servers:[{"host":"192.168.98.116"}]
},
retry:3,
mongodb:"mongodb://192.168.98.116:27017/gearman"
}
}
development:{
gearman:{
servers:[{'host':'192.168.98.116'}]
},
retry:3,
mongodb:'mongodb://192.168.98.116:27017/gearman'
}
};
179 changes: 87 additions & 92 deletions tests/test-basic.js
View file
Original file line number Diff line number Diff line change
@@ -1,109 +1,104 @@

"use strict"
'use strict';

const path = require('path')
const path = require('path');
process.chdir(path.join(process.cwd(),'tests'));

const should = require('should')
const sinon = require('sinon')
require('should-sinon')
const App = require('./lib/client.js')
const bottleneck = require('mof-bottleneck')
const request = require('mof-request')
const cheerio = require('mof-cheerio')
const iconv = require('mof-iconv')
const co = require('co')
const should = require('should');
const sinon = require('sinon');
require('should-sinon');
const App = require('./lib/client.js');
const bottleneck = require('mof-bottleneck');
const request = require('mof-request');
const cheerio = require('mof-cheerio');
const iconv = require('mof-iconv');
const co = require('co');

describe('Test worker in floodesh', ()=>{
const Worker = require('../worker');
let worker;
beforeEach(() => {

});

it('should load config file', ()=>{
worker = new Worker();
should.exist(worker.config.logBaseDir);
should.exist(worker.config.retry);
should.exist(worker.config.gearman);
should.exist(worker.config.logger);
should.exist(worker.config.mongodb);
worker.exit();
});

it("should new a worker", ()=> {
worker = new Worker();
worker.should.be.an.instanceOf(Worker);
});

it('should retry when time out', done=>{
worker = new Worker();
let getError = ()=>{
let e = new Error();
e.code = 'ETIMEDOUT';
return e;
};

worker.use((ctx, next)=>{
should.exist(ctx.opt);
console.log("middleware: %j", ctx.opt);
should.exist(ctx.opt.retries);

return next();
const Worker = require('../worker');
let worker;
beforeEach(() => {

});
worker.use(co.wrap(request()));
// worker.use((ctx, next)=>{
// let e = getError();
// throw e;
// return next();
// });

worker.on('complete', ctx=> {
process.nextTick(()=>{
should.exist(ctx);

it('should load config file', ()=>{
worker = new Worker();
should.exist(worker.config.logBaseDir);
should.exist(worker.config.retry);
should.exist(worker.config.gearman);
should.exist(worker.config.logger);
should.exist(worker.config.mongodb);
worker.exit();
done();
});
});

let job = {workComplete:function(){},reportException:function(){}};
let ctx = {opt:{uri:"http://www.baidu.com"}, app:{},request:{}, response:{},resourceList:{},job:job};
worker.jobs.add(job);
worker.emit("error",getError(),ctx);
});

it('should emit complete even if one of middlewares do not call next', done=>{
worker = new Worker();
let number=[];
worker.use((ctx, next)=>{
number.push(2);
return next();
it('should new a worker', ()=> {
worker = new Worker();
worker.should.be.an.instanceOf(Worker);
});

worker.use((ctx, next)=>{
should.exist(ctx);
should.exist(next);
it('should retry when time out', done=>{
worker = new Worker();
let getError = ()=>{
let e = new Error();
e.code = 'ETIMEDOUT';
return e;
};

worker.use((ctx, next)=>{
should.exist(ctx.opt);
console.log('middleware: %j', ctx.opt);
should.exist(ctx.opt.retries);

return next();
});
worker.use(co.wrap(request()));

worker.on('complete', ctx=> {
process.nextTick(()=>{
should.exist(ctx);
worker.exit();
done();
});
});

let job = {workComplete:function(){},reportError:function(){},reportWarning:function(){}};
let ctx = {opt:{uri:'http://www.baidu.com'}, app:{},request:{}, response:{},resourceList:{},job:job};
worker.jobs.add(job);
worker.emit('error',getError(),ctx);
});

worker.use((ctx, next)=>{
number.push(3);
return next();
});

worker.on('complete',ctx=>{
number.should.eql([1,2]);
process.nextTick(()=>{
should.exist(ctx);
worker.exit();
done();
});
});
it('should emit complete even if one of middlewares do not call next', done=>{
worker = new Worker();
let number=[];
worker.use((ctx, next)=>{
number.push(2);
return next();
});

worker.use((ctx, next)=>{
should.exist(ctx);
should.exist(next);
});

worker.use((ctx, next)=>{
number.push(3);
return next();
});

worker.on('complete',ctx=>{
number.should.eql([1,2]);
process.nextTick(()=>{
should.exist(ctx);
worker.exit();
done();
});
});

let ctx = worker.enqueue({uri:"http://www.baidu.com"});
number.push(1);
let ctx = worker.enqueue({uri:'http://www.baidu.com'});
number.push(1);

ctx.func='home';
ctx.job = {workComplete:function(){}};
worker.jobs.add(ctx.job);
});
ctx.func='home';
ctx.job = {workComplete:function(){}};
worker.jobs.add(ctx.job);
});
});
Loading

0 comments on commit 661c1e1

Please sign in to comment.