diff --git a/packages/canyon-collect/package.json b/packages/canyon-collect/package.json index 8fe8ab87..533310d4 100644 --- a/packages/canyon-collect/package.json +++ b/packages/canyon-collect/package.json @@ -32,7 +32,8 @@ "zod": "^3.23.8", "@nestjs/typeorm": "^10.0.2", "typeorm": "^0.3.20", - "sqlite3": "^5.1.7" + "sqlite3": "^5.1.7", + "canyon-data2": "workspace:^" }, "devDependencies": { "@nestjs/cli": "^10.0.0", diff --git a/packages/canyon-collect/src/apps/collect/services/core/consumer-coverage.service.ts b/packages/canyon-collect/src/apps/collect/services/core/consumer-coverage.service.ts index a8e1876e..21b54c01 100644 --- a/packages/canyon-collect/src/apps/collect/services/core/consumer-coverage.service.ts +++ b/packages/canyon-collect/src/apps/collect/services/core/consumer-coverage.service.ts @@ -1,26 +1,18 @@ import { Injectable } from '@nestjs/common'; -// import { PrismaService } from '../../../prisma/prisma.service'; import { genSummaryMapByCoverageMap, getSummaryByPath, - mergeCoverageMap, } from '../../../../canyon-data/src'; -// import { mergeCoverageMap, resetCoverageData } from '../../../utils/coverage'; -// import { removeNullKeys, resolveProjectID } from '../../../utils/utils'; -// import { PullChangeCodeAndInsertDbService } from '../common/pull-change-code-and-insert-db.service'; -// import { logger } from "../../../logger"; + import { CoveragediskService } from './coveragedisk.service'; -// import { TestExcludeService } from '../common/test-exclude.service'; -// import { PullFilePathAndInsertDbService } from '../common/pull-file-path-and-insert-db.service'; -// import { compressedData, decompressedData } from '../../../utils/zstd'; -// import { logger } from '../../../../logger'; import { PrismaService } from '../../../../prisma/prisma.service'; import { removeNullKeys } from '../../../../utils/utils'; import { compressedData, decompressedData } from '../../../../utils/zstd'; -import { reorganizeCompleteCoverageObjects } from '../../../../data/coverage'; import { coverageObj } from '../../models/coverage.model'; -import fs from 'node:fs'; -import { removeStartEndNull } from '../../../../utils/coverage'; + +import { resetCoverageDataMap } from '../../../../utils/coverage'; +import { mergeCoverageMap } from 'canyon-data'; +import {remapCoverageWithInstrumentCwd, reorganizeCompleteCoverageObjects} from "canyon-data2"; const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -109,7 +101,7 @@ export class ConsumerCoverageService { reportID: covType === 'agg' ? queueDataToBeConsumed.reportID : null, }), }); - const map = await this.prisma.coverage + const { map, instrumentCwd } = await this.prisma.coverage .findFirst({ where: { sha: queueDataToBeConsumed.sha, @@ -117,7 +109,13 @@ export class ConsumerCoverageService { covType: 'all', }, }) - .then((res) => decompressedData(res.map)); + .then(async (res) => { + const map = await decompressedData(res.map); + return { + map: map, + instrumentCwd: res.instrumentCwd, + }; + }); const codechanges = []; // TODO cov应该是全量的,应该是find出来的hit,因为已经合并过了,避免重复 @@ -131,11 +129,14 @@ export class ConsumerCoverageService { // map不参与exclude过滤,需要保留完整的 - const newCoverage = removeStartEndNull( - reorganizeCompleteCoverageObjects( - map, // - mergedHit, - ), + const reMapMap = await remapCoverageWithInstrumentCwd( + resetCoverageDataMap(map), + instrumentCwd, + ); + + const newCoverage = reorganizeCompleteCoverageObjects( + reMapMap, // + mergedHit, ); const summary = genSummaryMapByCoverageMap( diff --git a/packages/canyon-collect/src/apps/collect/services/coverage-client.service.ts b/packages/canyon-collect/src/apps/collect/services/coverage-client.service.ts index 1773cd81..b592fb0d 100755 --- a/packages/canyon-collect/src/apps/collect/services/coverage-client.service.ts +++ b/packages/canyon-collect/src/apps/collect/services/coverage-client.service.ts @@ -5,9 +5,9 @@ import { decompressedData } from '../../../utils/zstd'; import { formatReportObject, regularData } from '../../../utils/coverage'; import { CoveragediskService } from './core/coveragedisk.service'; import { - remapCoverage123, + remapCoverageWithInstrumentCwd, reorganizeCompleteCoverageObjects, -} from '../../../data/coverage'; +} from 'canyon-data2'; // 此代码重中之重、核心中的核心!!! @Injectable() @@ -60,7 +60,7 @@ export class CoverageClientService { const chongzu = reorganizeCompleteCoverageObjects(map, originalHit); // #region == Step x: 覆盖率回溯,在覆盖率存储之前转换(这里一定要用数据库里的instrumentCwd,因为和map是对应的!!!) - const hit = await remapCoverage123( + const hit = await remapCoverageWithInstrumentCwd( chongzu, coverageFromDatabase.instrumentCwd, ); diff --git a/packages/canyon-collect/src/canyon-data/src/index.ts b/packages/canyon-collect/src/canyon-data/src/index.ts index 87ffd3c3..eb43a42b 100755 --- a/packages/canyon-collect/src/canyon-data/src/index.ts +++ b/packages/canyon-collect/src/canyon-data/src/index.ts @@ -1,2 +1,2 @@ -export * from './coverage'; +// export * from './coverage'; export * from './summary'; diff --git a/packages/canyon-collect/src/data/README.md b/packages/canyon-collect/src/data/README.md deleted file mode 100644 index 17a6a7ed..00000000 --- a/packages/canyon-collect/src/data/README.md +++ /dev/null @@ -1 +0,0 @@ -未来单独做成包 diff --git a/packages/canyon-collect/src/data/coverage.ts b/packages/canyon-collect/src/data/coverage.ts deleted file mode 100644 index d8bcdd6b..00000000 --- a/packages/canyon-collect/src/data/coverage.ts +++ /dev/null @@ -1,53 +0,0 @@ -// 重组覆盖率数据 -import { remapCoverage } from '../utils/coverage'; - -export const reorganizeCompleteCoverageObjects = ( - map: { - [key: string]: object; - }, - hit: { - [key: string]: object; - }, -) => { - // istanbul数据结构 - const obj = {}; - for (const objKey in hit) { - const item = hit[objKey]; - const mapItem = map[objKey]; - obj[objKey] = { - ...item, - ...mapItem, - path: objKey, - }; - } - return obj; - // return {}; -}; - -// 重要方法,回溯源码覆盖率数据 -export const remapCoverage123 = async (noReMap, inser) => { - // 如果来自的插桩路径不同,要预处理!!! - const obj = {}; - for (const key in noReMap) { - const newKey = inser + '/' + key; - const item = noReMap[key]; - obj[newKey] = { - ...item, - path: newKey, - }; - } - - const reMapedCov = await remapCoverage(obj); - - const obj222: any = {}; - for (const coverageKey in reMapedCov) { - const newKey = coverageKey.replace(inser + '/', ''); - obj222[newKey] = { - ...reMapedCov[coverageKey], - path: newKey, - }; - } - - // 再把inser去掉 - return obj222; -}; diff --git a/packages/canyon-collect/src/utils/coverage.ts b/packages/canyon-collect/src/utils/coverage.ts index 19049d37..7b39ae4e 100644 --- a/packages/canyon-collect/src/utils/coverage.ts +++ b/packages/canyon-collect/src/utils/coverage.ts @@ -58,20 +58,6 @@ export function formatReportObject(c: any) { }; } -// 覆盖率回溯,在覆盖率存储之前转换 -export async function remapCoverage(obj: any) { - const res = await libSourceMaps - .createSourceMapStore() - .transformCoverage(libCoverage.createCoverageMap(obj)); - const { data: data_1 } = res; - const obj_1: any = {}; - for (const dataKey in data_1) { - const x = data_1[dataKey]['data']; - obj_1[x.path] = x; - } - return obj_1; -} - export function resetCoverageData(coverageData) { return Object.entries(coverageData).reduce((acc, [key, value]: any) => { acc[key] = { @@ -96,6 +82,30 @@ export function resetCoverageData(coverageData) { }, {}); } +export function resetCoverageDataMap(coverageData) { + return Object.entries(coverageData).reduce((acc, [key, value]: any) => { + acc[key] = { + ...value, + s: Object.entries(value.statementMap).reduce((accInside, [keyInside]) => { + accInside[keyInside] = 0; + return accInside; + }, {}), + f: Object.entries(value.fnMap).reduce((accInside, [keyInside]) => { + accInside[keyInside] = 0; + return accInside; + }, {}), + b: Object.entries(value.branchMap).reduce( + (accInside, [keyInside, valueInside]: any) => { + accInside[keyInside] = Array(valueInside.length).fill(0); + return accInside; + }, + {}, + ), + }; + return acc; + }, {}); +} + export function regularData(data: any) { const obj = {}; const coverage = data; diff --git a/packages/canyon-data2/package.json b/packages/canyon-data2/package.json new file mode 100644 index 00000000..6fd0f68c --- /dev/null +++ b/packages/canyon-data2/package.json @@ -0,0 +1,36 @@ +{ + "name": "canyon-data2", + "version": "1.0.0", + "description": "", + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "require": "./dist/index.cjs" + } + }, + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist" + ], + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "build": "rslib build", + "preinstall": "rslib build" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@rslib/core": "^0.0.18", + "@types/istanbul-lib-coverage": "^2.0.6", + "@types/istanbul-lib-source-maps": "^4.0.4" + }, + "dependencies": { + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-source-maps": "^5.0.6" + } +} diff --git a/packages/canyon-data2/rslib.config.ts b/packages/canyon-data2/rslib.config.ts new file mode 100644 index 00000000..e9397a12 --- /dev/null +++ b/packages/canyon-data2/rslib.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from '@rslib/core'; + +export default defineConfig({ + lib: [ + { + format: 'esm', + syntax: 'es2021', + dts: true, + }, + { + format: 'cjs', + syntax: 'es2021', + }, + ], +}); diff --git a/packages/canyon-data2/src/coverage.ts b/packages/canyon-data2/src/coverage.ts new file mode 100644 index 00000000..3c3e9382 --- /dev/null +++ b/packages/canyon-data2/src/coverage.ts @@ -0,0 +1,70 @@ +import libCoverage from "istanbul-lib-coverage"; +import libSourceMaps from "istanbul-lib-source-maps"; +// 覆盖率回溯,在覆盖率存储之前转换 +export async function remapCoverage(obj: any) { + const res = await libSourceMaps + .createSourceMapStore() + .transformCoverage(libCoverage.createCoverageMap(obj)); + const { data: data_1 } = res; + const obj_1: any = {}; + for (const dataKey in data_1) { + // @ts-ignore + const x = data_1[dataKey]["data"]; + obj_1[x.path] = x; + } + return obj_1; +} + +export const reorganizeCompleteCoverageObjects = ( + map: { + [key: string]: object; + }, + hit: { + [key: string]: object; + }, +) => { + // istanbul数据结构 + const obj = {}; + for (const objKey in hit) { + const item = hit[objKey]; + const mapItem = map[objKey]; + // @ts-ignore + obj[objKey] = { + ...mapItem, + // 一定要在下面!!! + ...item, + path: objKey, + }; + } + return obj; + // return {}; +}; + +// 回溯未经过reMapCoverage的数据,但是必须得传入插装路径,因为这里的noReMap是没有插装路径的 +export const remapCoverageWithInstrumentCwd = async (noReMap:any, inser:string) => { + // 如果来自的插桩路径不同,要预处理!!! + const obj = {}; + for (const key in noReMap) { + const newKey = inser + "/" + key; + const item = noReMap[key]; + // @ts-ignore + obj[newKey] = { + ...item, + path: newKey, + }; + } + + const reMapedCov = await remapCoverage(obj); + + const obj222: any = {}; + for (const coverageKey in reMapedCov) { + const newKey = coverageKey.replace(inser + "/", ""); + obj222[newKey] = { + ...reMapedCov[coverageKey], + path: newKey, + }; + } + + // 再把inser去掉 + return obj222; +}; diff --git a/packages/canyon-data2/src/index.ts b/packages/canyon-data2/src/index.ts new file mode 100644 index 00000000..25823c72 --- /dev/null +++ b/packages/canyon-data2/src/index.ts @@ -0,0 +1 @@ +export * from './coverage'; diff --git a/packages/canyon-data2/tsconfig.json b/packages/canyon-data2/tsconfig.json new file mode 100644 index 00000000..8c5c4d6d --- /dev/null +++ b/packages/canyon-data2/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "lib": ["ES2021"], + "module": "ESNext", + "noEmit": true, + "strict": true, + "skipLibCheck": true, + "isolatedModules": true, + "resolveJsonModule": true, + "moduleResolution": "bundler", + "useDefineForClassFields": true, + "allowImportingTsExtensions": true + }, + "include": ["src"] +} diff --git a/packages/canyon-platform/app/api/cov/map/route.ts b/packages/canyon-platform/app/api/cov/map/route.ts index 87c09f92..ea633ef5 100644 --- a/packages/canyon-platform/app/api/cov/map/route.ts +++ b/packages/canyon-platform/app/api/cov/map/route.ts @@ -1,11 +1,17 @@ import prisma from "@/lib/prisma"; import { decompressedData } from "@/utils/zstd"; import { - formatReportObject, - remapCoverage, - reorganizeCompleteCoverageObjects, + // formatReportObject, + // remapCoverage, + // remapCoverageWithInstrumentCwd, + // reorganizeCompleteCoverageObjects, + resetCoverageDataMap, } from "@/utils/coverage"; import { NextRequest } from "next/server"; +import { + remapCoverageWithInstrumentCwd, + reorganizeCompleteCoverageObjects, +} from "canyon-data2"; export async function GET(request: NextRequest) { const { searchParams } = request.nextUrl; @@ -28,11 +34,16 @@ export async function GET(request: NextRequest) { }, }); - const d = await decompressedData(data.map); + const map = await decompressedData(data.map); - const c = await decompressedData(hitdata.hit); + const hit = await decompressedData(hitdata.hit); + + const reMapMap = await remapCoverageWithInstrumentCwd( + resetCoverageDataMap(map), + hitdata.instrumentCwd, + ); - const obj = reorganizeCompleteCoverageObjects(d, c); + const obj = reorganizeCompleteCoverageObjects(reMapMap, hit); return Response.json( Object.entries(obj) diff --git a/packages/canyon-platform/package.json b/packages/canyon-platform/package.json index 75fbbfbb..735cef37 100755 --- a/packages/canyon-platform/package.json +++ b/packages/canyon-platform/package.json @@ -24,7 +24,8 @@ "react-dom": "19.0.0-rc-66855b96-20241106", "recharts": "^2.13.3", "shiki": "^1.22.1", - "swr": "^2.2.5" + "swr": "^2.2.5", + "canyon-data2": "workspace:^" }, "devDependencies": { "@iconify/json": "^2.2.242", diff --git a/packages/canyon-platform/utils/coverage.ts b/packages/canyon-platform/utils/coverage.ts index 6fad23e3..5817f368 100644 --- a/packages/canyon-platform/utils/coverage.ts +++ b/packages/canyon-platform/utils/coverage.ts @@ -1,19 +1,6 @@ -import libCoverage from "istanbul-lib-coverage"; -import libSourceMaps from "istanbul-lib-source-maps"; -// 覆盖率回溯,在覆盖率存储之前转换 -export async function remapCoverage(obj: any) { - const res = await libSourceMaps - .createSourceMapStore() - .transformCoverage(libCoverage.createCoverageMap(obj)); - const { data: data_1 } = res; - const obj_1: any = {}; - for (const dataKey in data_1) { - const x = data_1[dataKey]["data"]; - obj_1[x.path] = x; - } - return obj_1; -} + + function parseInstrumentCwd(instrumentCwd) { if (instrumentCwd.includes("=>")) { @@ -73,26 +60,28 @@ export function formatReportObject(c: any) { }; } - -export const reorganizeCompleteCoverageObjects = ( - map: { - [key: string]: object; - }, - hit: { - [key: string]: object; - }, -) => { - // istanbul数据结构 - const obj = {}; - for (const objKey in hit) { - const item = hit[objKey]; - const mapItem = map[objKey]; - obj[objKey] = { - ...item, - ...mapItem, - path: objKey, +export function resetCoverageDataMap(coverageData) { + return Object.entries(coverageData).reduce((acc, [key, value]: any) => { + acc[key] = { + ...value, + s: Object.entries(value.statementMap).reduce((accInside, [keyInside]) => { + accInside[keyInside] = 0; + return accInside; + }, {}), + f: Object.entries(value.fnMap).reduce((accInside, [keyInside]) => { + accInside[keyInside] = 0; + return accInside; + }, {}), + b: Object.entries(value.branchMap).reduce( + (accInside, [keyInside, valueInside]: any) => { + accInside[keyInside] = Array(valueInside.length).fill(0); + return accInside; + }, + {}, + ), }; - } - return obj; - // return {}; -}; + return acc; + }, {}); +} + +// 重要方法,回溯源码覆盖率数据