Skip to content

Commit

Permalink
upd
Browse files Browse the repository at this point in the history
  • Loading branch information
Allen Zhang (张涛) committed Apr 9, 2024
1 parent cd9fa97 commit c8f32d7
Show file tree
Hide file tree
Showing 8 changed files with 132 additions and 9 deletions.
4 changes: 3 additions & 1 deletion packages/canyon-backend/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,6 @@ lerna-debug.log*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
!.vscode/extensions.json

db/sql
6 changes: 5 additions & 1 deletion packages/canyon-backend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
"@nestjs/platform-express": "^10.3.0",
"@nestjs/schedule": "^4.0.0",
"@nestjs/serve-static": "^4.0.0",
"@nestjs/typeorm": "^10.0.2",
"adm-zip": "^0.5.10",
"axios": "^1.6.5",
"body-parser": "^1.20.2",
Expand All @@ -59,7 +60,10 @@
"prisma": "5.8.1",
"reflect-metadata": "^0.1.14",
"rxjs": "^7.8.1",
"test-exclude": "^6.0.0"
"sqlite": "^5.1.1",
"sqlite3": "^5.1.7",
"test-exclude": "^6.0.0",
"typeorm": "^0.3.20"
},
"devDependencies": {
"@nestjs/cli": "^10.3.0",
Expand Down
8 changes: 8 additions & 0 deletions packages/canyon-backend/src/app.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,18 @@ import { UploadModule } from './upload/upload.module';
// import { UsageModule } from './usage/usage.module';
import { CoveragedataModule } from './coveragedata/coveragedata.module';
import { MongooseModule } from '@nestjs/mongoose';
import { TypeOrmModule } from '@nestjs/typeorm';
import { CoveragediskEntity } from './coverage/coveragedisk.entity';
// const condition = process.env.MODE === 'task';
@Module({
imports: [
MongooseModule.forRoot(process.env.MONGODB_URL),
TypeOrmModule.forRoot({
type: 'sqlite',
database: 'db/sql',
synchronize: true,
entities: [CoveragediskEntity],
}),
ScheduleModule.forRoot(),
AuthModule,
UserModule,
Expand Down
5 changes: 5 additions & 0 deletions packages/canyon-backend/src/coverage/coverage.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ import { CoverageDataAdapterService } from './services/coverage-data-adapter.ser
import { ConsumerCoverageService } from './services/consumer-coverage.service';
import { PullChangeCodeAndInsertDbService } from './services/pull-change-code-and-insert-db.service';
import { CoverageLog, CoverageLogSchema } from './schemas/coverage-log.schema';
import { TypeOrmModule } from '@nestjs/typeorm';
import { CoveragediskEntity } from './coveragedisk.entity';
import { MenuService } from './services/menu.service';

@Module({
imports: [
Expand All @@ -29,6 +32,7 @@ import { CoverageLog, CoverageLogSchema } from './schemas/coverage-log.schema';
collection: 'canyon_coverage_log',
},
]),
TypeOrmModule.forFeature([CoveragediskEntity]),
],
controllers: [CoverageController],
providers: [
Expand All @@ -40,6 +44,7 @@ import { CoverageLog, CoverageLogSchema } from './schemas/coverage-log.schema';
CoverageDataAdapterService,
ConsumerCoverageService,
PullChangeCodeAndInsertDbService,
MenuService,
],
})
export class CoverageModule {}
22 changes: 22 additions & 0 deletions packages/canyon-backend/src/coverage/coveragedisk.entity.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm';

@Entity('coveragedisk')
export class CoveragediskEntity {
@PrimaryGeneratedColumn()
id: number;

@Column()
pid: string;

@Column()
projectID: string;

@Column()
sha: string;

@Column()
data: string;

@Column()
createdAt: Date;
}
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import { Injectable } from '@nestjs/common';
import { PrismaService } from '../../prisma/prisma.service';
import {
getQueueWithSameShaAndProjectID,
pushQueue,
} from '../../queues/coverage';
// import {
// getQueueWithSameShaAndProjectID,
// pushQueue,
// } from '../../queues/coverage';
import {
genSummaryMapByCoverageMap,
getSummaryByPath,
Expand All @@ -14,6 +14,7 @@ import { validateObject } from '../../utils/coverage';
import { CoverageDataAdapterService } from './coverage-data-adapter.service';
import { PullChangeCodeAndInsertDbService } from './pull-change-code-and-insert-db.service';
import { logger } from '../../logger';
import { MenuService } from './menu.service';

const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

Expand All @@ -23,13 +24,15 @@ export class ConsumerCoverageService {
private readonly prisma: PrismaService,
private readonly coverageDataAdapterService: CoverageDataAdapterService,
private readonly pullChangeCodeAndInsertDbService: PullChangeCodeAndInsertDbService,
private readonly menuService: MenuService,
) {}

async invoke() {
while (true) {
try {
// 获取将要消费的队列数据,sha和projectID相同,并且已经聚合
const queueDataToBeConsumed = getQueueWithSameShaAndProjectID();
const queueDataToBeConsumed =
await this.menuService.getQueueWithSameShaAndProjectID();
// 如果存在
if (queueDataToBeConsumed) {
const _lockName = 'consumer_coverage';
Expand Down Expand Up @@ -73,7 +76,7 @@ export class ConsumerCoverageService {
});
} else {
// 被锁住了就push到队列尾部
pushQueue(queueDataToBeConsumed);
await this.menuService.pushQueue(queueDataToBeConsumed);
// return false;
}
} else {
Expand Down Expand Up @@ -104,6 +107,7 @@ export class ConsumerCoverageService {
}
} catch (e) {
console.log(e);
await sleep(2000);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { InjectModel } from '@nestjs/mongoose';
// import { CoverageData } from '../schemas/coverage-data.schema';
import { Model } from 'mongoose';
import { CoverageLog } from '../schemas/coverage-log.schema';
import { MenuService } from './menu.service';
/**
* 上传覆盖率,十分重要的服务
*/
Expand All @@ -34,6 +35,7 @@ export class CoverageClientService {
private readonly prisma: PrismaService,
@InjectModel(CoverageLog.name)
private coverageLogModel: Model<CoverageLog>,
private menuService: MenuService,
) {}

async invoke(currentUser, coverageClientDto: CoverageClientDto, ip) {
Expand Down Expand Up @@ -116,7 +118,7 @@ export class CoverageClientService {
ip,
},
});
pushQueue(cov);
await this.menuService.pushQueue(cov);
return {
msg: 'ok',
coverageId: '',
Expand Down
76 changes: 76 additions & 0 deletions packages/canyon-backend/src/coverage/services/menu.service.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { CoveragediskEntity } from '../coveragedisk.entity';
import { Repository } from 'typeorm';
// import { pushQueue } from '../../queues/coverage';
import * as process from 'node:process';
import { mergeCoverageMap } from '@canyon/data';

@Injectable()
export class MenuService {
constructor(
@InjectRepository(CoveragediskEntity)
private readonly menuRepository: Repository<CoveragediskEntity>,
) {}
async create() {
const s = await this.menuRepository.find();
console.log(s);
return this.menuRepository.insert({
projectID: 'test',
sha: 'test',
data: 'test',
});
}
// Add CRUD operations you intend to use here
async pushQueue(data) {
return this.menuRepository.insert({
pid: String(process.pid),
projectID: data.projectID,
sha: data.sha,
data: JSON.stringify(data),
createdAt: new Date(),
});
}
async getQueueWithSameShaAndProjectID() {
const old = await this.menuRepository.findOne({
where: {
pid: String(process.pid),
},
order: {
createdAt: 'ASC',
},
select: {
pid: true,
projectID: true,
sha: true,
data: true,
},
});
if (!old) {
return false;
}
const arr = await this.menuRepository.find({
where: {
projectID: old.projectID,
sha: old.sha,
pid: String(process.pid),
},
select: {
id: true,
},
});
let cov = {};
for (let i = 0; i < arr.length; i++) {
const toBeConsumedQueues = await this.menuRepository.findOne({
where: { id: arr[i].id, pid: String(process.pid) },
});
// 聚合
cov = mergeCoverageMap(cov, JSON.parse(toBeConsumedQueues.data).coverage);
await this.menuRepository.delete({ id: arr[i].id });
}
return {
...JSON.parse(old.data),
coverage: cov,
};
}
}

0 comments on commit c8f32d7

Please sign in to comment.