diff --git a/build/commands/lib/buildChromiumRelease.js b/build/commands/lib/buildChromiumRelease.js new file mode 100644 index 000000000000..0165c49d5a95 --- /dev/null +++ b/build/commands/lib/buildChromiumRelease.js @@ -0,0 +1,175 @@ +// Copyright (c) 2023 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +// A script to build and pack a Chromium release build from scratch +// Reuses the same /src folder +// Designed to be used on CI, but should work locally too. +// The script includes syncing; there is no need to run npm run sync before. + +const config = require('./config') +const util = require('./util') +const path = require('path') +const fs = require('fs-extra') +const syncUtil = require('./syncUtils') +const Log = require('./logging') + +// Use the same filename as for Brave archive. +const getOutputFilename = () => { + const platform = (() => { + if (config.getTargetOS() === 'win') + return 'win32' + if (config.getTargetOS() === 'mac') + return 'darwin' + return config.getTargetOS() + })() + return `chromium-${config.chromeVersion}-${platform}-${config.targetArch}` +} + +const chromiumConfigs = { + 'win': { + buildTarget: 'mini_installer', + processArtifacts: () => { + // Repack it to reduce the size and use .zip instead of .7z. + input = path.join(config.outputDir, 'chrome.7z') + output = path.join(config.outputDir, `${getOutputFilename()}.zip`) + util.run('python3', + [ + path.join(config.braveCoreDir, 'script', 'repack-archive.py'), + `--input=${input}`, + `--output=${output}`, + '--target_dir=Chrome-bin', + ], + config.defaultOptions) + } + }, + 'linux': { + buildTarget: 'chrome/installer/linux:stable_deb', + processArtifacts: () => { + const debArch = (() => { + if (config.targetArch === 'x64') return 'amd64' + return config.targetArch + })() + fs.moveSync( + path.join(config.outputDir, + `chromium-browser-stable_${config.chromeVersion}-1_${debArch}.deb`), + path.join(config.outputDir, `${getOutputFilename()}.deb`)) + } + }, + 'mac': { + buildTarget: 'chrome', + extraHooks: () => { + Log.progressScope('download_hermetic_xcode', () => { + util.run('vpython3', + [ + path.join(config.braveCoreDir, + 'build', 'mac', 'download_hermetic_xcode.py'), + ], + config.defaultOptions) + }) + }, + processArtifacts: () => { + util.run('zip', + ['-r', '-y', `${getOutputFilename()}.zip`, 'Chromium.app'], + { cwd: config.outputDir } + ) + } + }, + 'android': { + buildTarget: 'chrome_public_apk', + processArtifacts: () => { + fs.moveSync( + path.join(config.outputDir, 'apks', 'ChromePublic.apk'), + path.join(config.outputDir, `${getOutputFilename()}.apk`)) + } + }, +} + +// A function to make gn args to build a release Chromium build. +// There is two primarily sources: +// 1. Chromium perf builds: tools/mb/mb_config_expectations/chromium.perf.json +// 2. Brave Release build configuration +function getChromiumGnArgs() { + const targetOs = config.getTargetOS() + const targetArch = config.targetArch + const args = { + target_cpu: targetArch, + target_os: targetOs, + is_official_build: true, + enable_keystone_registration_framework: false, + ffmpeg_branding: 'Chrome', + enable_widevine: true, + ignore_missing_widevine_signing_cert: true, + ...config.extraGnArgs, + } + + if (targetOs === 'android') { + args.debuggable_apks = false + } else { + args.enable_hangout_services_extension = true + args.enable_nacl = false + } + + if (targetOs === 'mac') { + args.use_system_xcode = true + } + + return args +} + +function buildChromiumRelease(buildOptions = {}) { + if (!config.isCI && !buildOptions.force) { + console.error( + 'Warning: the command resets all changes in src/ folder.\n' + + 'src/brave stays untouched. Pass --force to continue.') + return 1 + } + config.buildConfig = 'Release' + config.isChromium = true + config.update(buildOptions) + + const chromiumConfig = chromiumConfigs[config.getTargetOS()] + if (chromiumConfig == undefined) + throw Error(`${config.getTargetOS()} is unsupported`) + + syncUtil.maybeInstallDepotTools() + syncUtil.buildDefaultGClientConfig( + [config.getTargetOS()], [config.targetArch], true) + + util.runGit(config.srcDir, ['clean', '-f', '-d']) + + + Log.progressScope('gclient sync', () => { + syncUtil.syncChromium(true, true, false) + }) + + Log.progressScope('gclient runhooks', () => { + util.runGClient(['runhooks']) + }) + + if (chromiumConfig.extraHooks != undefined) { + chromiumConfig.extraHooks() + } + + const options = config.defaultOptions + const buildArgsStr = util.buildArgsToString(getChromiumGnArgs()) + util.run('gn', ['gen', config.outputDir, '--args="' + buildArgsStr + '"'], + options) + + + Log.progressScope(`ninja`, () => { + const target = chromiumConfig.buildTarget + const ninjaOpts = [ + '-C', options.outputDir || config.outputDir, target, + ...config.extraNinjaOpts + ] + util.run('autoninja', ninjaOpts, config.defaultOptions) + }) + + Log.progressScope('make archive', () => { + chromiumConfig.processArtifacts() + }) +} + +module.exports = buildChromiumRelease diff --git a/build/commands/lib/config.js b/build/commands/lib/config.js index 67fa659f92e4..d06007bc2733 100644 --- a/build/commands/lib/config.js +++ b/build/commands/lib/config.js @@ -107,6 +107,7 @@ const Config = function () { this.buildTarget = 'brave' this.rootDir = rootDir this.isUniversalBinary = false + this.isChromium = false this.scriptDir = path.join(this.rootDir, 'scripts') this.srcDir = path.join(this.rootDir, 'src') this.chromeVersion = this.getProjectVersion('chrome') @@ -1289,6 +1290,9 @@ Object.defineProperty(Config.prototype, 'outputDir', { if (this.targetEnvironment) { buildConfigDir = buildConfigDir + "_" + this.targetEnvironment } + if (this.isChromium) { + buildConfigDir = buildConfigDir + "_chromium" + } return path.join(baseDir, buildConfigDir) }, diff --git a/build/commands/lib/syncUtils.js b/build/commands/lib/syncUtils.js new file mode 100644 index 000000000000..388d43513146 --- /dev/null +++ b/build/commands/lib/syncUtils.js @@ -0,0 +1,197 @@ +// Copyright (c) 2023 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +const chalk = require('chalk') +const config = require('./config') +const fs = require('fs') +const path = require('path') +const Log = require('./logging') +const util = require('./util') + + +function maybeInstallDepotTools(options = config.defaultOptions) { + options.cwd = config.braveCoreDir + + if (!fs.existsSync(config.depotToolsDir)) { + Log.progressScope('install depot_tools', () => { + fs.mkdirSync(config.depotToolsDir) + util.run( + 'git', + [ + '-C', + config.depotToolsDir, + 'clone', + 'https://chromium.googlesource.com/chromium/tools/depot_tools.git', + '.' + ], + options + ) + }) + } + + const ninjaLogCfgPath = path.join(config.depotToolsDir, 'ninjalog.cfg'); + if (!fs.existsSync(ninjaLogCfgPath)) { + // Create a ninja config to prevent (auto)ninja from calling goma_auth + // each time. See for details: + // https://chromium.googlesource.com/chromium/tools/depot_tools/+/main/ninjalog.README.md + const ninjaLogCfgConfig = { + 'is-googler': false, + 'version': 3, + 'countdown': 10, + 'opt-in': false, + }; + fs.writeFileSync(ninjaLogCfgPath, JSON.stringify(ninjaLogCfgConfig)) + } +} + +function toGClientConfigItem(name, value, pretty = true) { + // Convert value to json and replace "%True%" -> True, "%False%" -> False, + // "%None%" -> None. + const pythonLikeValue = + JSON.stringify(value, null, pretty ? 2 : 0).replace(/"%(.*?)%"/gm, '$1') + return `${name} = ${pythonLikeValue}\n` +} + +function buildDefaultGClientConfig( + targetOSList, targetArchList, onlyChromium = false) { + const items = [ + { + managed: '%False%', + name: 'src', + url: config.chromiumRepo, + custom_deps: { + 'src/third_party/WebKit/LayoutTests': '%None%', + 'src/chrome_frame/tools/test/reference_build/chrome': '%None%', + 'src/chrome_frame/tools/test/reference_build/chrome_win': '%None%', + 'src/chrome/tools/test/reference_build/chrome': '%None%', + 'src/chrome/tools/test/reference_build/chrome_linux': '%None%', + 'src/chrome/tools/test/reference_build/chrome_mac': '%None%', + 'src/chrome/tools/test/reference_build/chrome_win': '%None%' + }, + custom_vars: { + 'checkout_rust': '%True%', + 'checkout_pgo_profiles': config.isBraveReleaseBuild() ? '%True%' : + '%False%' + } + } + ] + + if (!onlyChromium) { + items.push({ + managed: '%False%', + name: 'src/brave', + // We do not use gclient to manage brave-core, so this should not + // actually get used. + url: 'https://github.com/brave/brave-core.git' + }) + } + + let out = toGClientConfigItem('solutions', items); + + if (process.env.GIT_CACHE_PATH) { + out += toGClientConfigItem('cache_dir', process.env.GIT_CACHE_PATH) + } + if (targetOSList) { + out += toGClientConfigItem('target_os', targetOSList, false) + } + if (targetArchList) { + out += toGClientConfigItem('target_cpu', targetArchList, false) + } + + fs.writeFileSync(config.defaultGClientFile, out) +} + +function shouldUpdateChromium(latestSyncInfo, expectedSyncInfo) { + const chromiumRef = expectedSyncInfo.chromiumRef + const headSHA = util.runGit(config.srcDir, ['rev-parse', 'HEAD'], true) + const targetSHA = util.runGit(config.srcDir, ['rev-parse', chromiumRef], true) + const needsUpdate = targetSHA !== headSHA || (!headSHA && !targetSHA) || + JSON.stringify(latestSyncInfo) !== JSON.stringify(expectedSyncInfo) + if (needsUpdate) { + const currentRef = util.getGitReadableLocalRef(config.srcDir) + console.log(`Chromium repo ${chalk.blue.bold('needs sync')}.\n target is ${ + chalk.italic(chromiumRef)} at commit ${ + targetSHA || '[missing]'}\n current commit is ${ + chalk.italic(currentRef || '[unknown]')} at commit ${ + chalk.inverse(headSHA || '[missing]')}\n latest successful sync is ${ + JSON.stringify(latestSyncInfo, null, 4)}`) + } + else { + console.log( + chalk.green.bold(`Chromium repo does not need sync as it is already ${ + chalk.italic(chromiumRef)} at commit ${targetSHA || '[missing]'}.`)) + } + return needsUpdate +} + +function syncChromium(syncWithForce, sync_chromium, delete_unused_deps) { + const requiredChromiumRef = config.getProjectRef('chrome') + let args = [ + 'sync', '--nohooks', '--revision', 'src@' + requiredChromiumRef, '--reset', + '--with_tags', '--with_branch_heads', '--upstream' + ]; + + if (syncWithForce) { + args.push('--force') + } + + const latestSyncInfoFilePath = + path.join(config.rootDir, '.brave_latest_successful_sync.json') + const latestSyncInfo = util.readJSON(latestSyncInfoFilePath, {}) + const expectedSyncInfo = { + chromiumRef: requiredChromiumRef, + gClientTimestamp: fs.statSync(config.gClientFile).mtimeMs.toString(), + } + + const chromiumNeedsUpdate = + shouldUpdateChromium(latestSyncInfo, expectedSyncInfo) + const shouldSyncChromium = chromiumNeedsUpdate || syncWithForce + if (!shouldSyncChromium && !sync_chromium) { + if (delete_unused_deps && !config.isCI) { + Log.warn( + '--delete_unused_deps is ignored for src/ dir because Chromium sync ' + + 'is required. Pass --sync_chromium to force it.') + } + return false + } + + if (delete_unused_deps) { + if (util.isGitExclusionExists(config.srcDir, 'brave/')) { + args.push('-D') + } else if (!config.isCI) { + Log.warn( + '--delete_unused_deps is ignored because sync has not yet added ' + + 'the exclusion for the src/brave/ directory, likely because sync ' + + 'has not previously successfully run before.') + } + } + + if (sync_chromium !== undefined) { + if (!sync_chromium) { + Log.warn( + 'Chromium needed sync but received the flag to skip performing the ' + + 'update. Working directory may not compile correctly.') + return false + } else if (!shouldSyncChromium) { + Log.warn( + 'Chromium doesn\'t need sync but received the flag to do it anyway.') + } + } + + util.runGClient(args) + util.addGitExclusion(config.srcDir, 'brave/') + util.writeJSON(latestSyncInfoFilePath, expectedSyncInfo) + + const postSyncChromiumRef = util.getGitReadableLocalRef(config.srcDir) + Log.status(`Chromium is now at ${postSyncChromiumRef || '[unknown]'}`) + return true +} + + +module.exports = { + maybeInstallDepotTools, + buildDefaultGClientConfig, + syncChromium +} diff --git a/build/commands/scripts/commands.js b/build/commands/scripts/commands.js index af08b0da0d65..114e06e12796 100644 --- a/build/commands/scripts/commands.js +++ b/build/commands/scripts/commands.js @@ -9,6 +9,7 @@ const fs = require('fs-extra') const config = require('../lib/config') const util = require('../lib/util') const build = require('../lib/build') +const buildChromiumRelease = require('../lib/buildChromiumRelease') const { buildFuzzer, runFuzzer } = require('../lib/fuzzer') const versions = require('../lib/versions') const start = require('../lib/start') @@ -166,6 +167,24 @@ program .arguments('[build_config]') .action(build) +program + .command('build_chromium_release') + .description( + 'Produces a chromium release build for performance testing.\n' + + 'Uses the same /src directory; all brave patches are reverted.\n' + + 'The default build_dir is `chromium_Release(_target_arch)`.\n' + + 'Intended for use on CI, use locally with care.') + .option('--force', 'Ignore a warning about non-CI build') + .option('-C ', 'build config (out/chromium_Release') + .option('--target_os ', 'target OS') + .option('--target_arch ', 'target architecture') + .option('--gn ', 'Additional gn args, in the form :', + collect, []) + .option('--ninja ', + 'Additional Ninja command-line options, in the form :', + collect, []) + .action(buildChromiumRelease) + program .command('create_dist') .option('-C ', 'build config (out/Debug, out/Release') diff --git a/build/commands/scripts/sync.js b/build/commands/scripts/sync.js index 4b3619506e9b..fae5de765c7e 100644 --- a/build/commands/scripts/sync.js +++ b/build/commands/scripts/sync.js @@ -9,7 +9,7 @@ const path = require('path') const config = require('../lib/config') const util = require('../lib/util') const Log = require('../lib/logging') -const chalk = require('chalk') +const syncUtil = require('../lib/syncUtils') program .version(process.env.npm_package_version) @@ -25,179 +25,6 @@ program .option('-D, --delete_unused_deps', 'delete from the working copy any dependencies that have been removed since the last sync') .option('--nohooks', 'Do not run hooks after updating') -function maybeInstallDepotTools(options = config.defaultOptions) { - options.cwd = config.braveCoreDir - - if (!fs.existsSync(config.depotToolsDir)) { - Log.progressScope('install depot_tools', () => { - fs.mkdirSync(config.depotToolsDir) - util.run( - 'git', - [ - '-C', - config.depotToolsDir, - 'clone', - 'https://chromium.googlesource.com/chromium/tools/depot_tools.git', - '.' - ], - options - ) - }) - } - - const ninjaLogCfgPath = path.join(config.depotToolsDir, 'ninjalog.cfg'); - if (!fs.existsSync(ninjaLogCfgPath)) { - // Create a ninja config to prevent (auto)ninja from calling goma_auth - // each time. See for details: - // https://chromium.googlesource.com/chromium/tools/depot_tools/+/main/ninjalog.README.md - const ninjaLogCfgConfig = { - 'is-googler': false, - 'version': 3, - 'countdown': 10, - 'opt-in': false, - }; - fs.writeFileSync(ninjaLogCfgPath, JSON.stringify(ninjaLogCfgConfig)) - } -} - -function toGClientConfigItem(name, value, pretty = true) { - // Convert value to json and replace "%True%" -> True, "%False%" -> False, - // "%None%" -> None. - const pythonLikeValue = - JSON.stringify(value, null, pretty ? 2 : 0).replace(/"%(.*?)%"/gm, '$1') - return `${name} = ${pythonLikeValue}\n` -} - -function buildDefaultGClientConfig(targetOSList, targetArchList) { - let out = toGClientConfigItem('solutions', [ - { - managed: '%False%', - name: 'src', - url: config.chromiumRepo, - custom_deps: { - 'src/third_party/WebKit/LayoutTests': '%None%', - 'src/chrome_frame/tools/test/reference_build/chrome': '%None%', - 'src/chrome_frame/tools/test/reference_build/chrome_win': '%None%', - 'src/chrome/tools/test/reference_build/chrome': '%None%', - 'src/chrome/tools/test/reference_build/chrome_linux': '%None%', - 'src/chrome/tools/test/reference_build/chrome_mac': '%None%', - 'src/chrome/tools/test/reference_build/chrome_win': '%None%' - }, - custom_vars: { - 'checkout_rust': '%True%', - 'checkout_pgo_profiles': config.isBraveReleaseBuild() ? '%True%' : - '%False%' - } - }, - { - managed: '%False%', - name: 'src/brave', - // We do not use gclient to manage brave-core, so this should not - // actually get used. - url: 'https://github.com/brave/brave-core.git' - } - ]) - - if (process.env.GIT_CACHE_PATH) { - out += toGClientConfigItem('cache_dir', process.env.GIT_CACHE_PATH) - } - if (targetOSList) { - out += toGClientConfigItem('target_os', targetOSList, false) - } - if (targetArchList) { - out += toGClientConfigItem('target_cpu', targetArchList, false) - } - - fs.writeFileSync(config.defaultGClientFile, out) -} - -function shouldUpdateChromium(latestSyncInfo, expectedSyncInfo) { - const chromiumRef = expectedSyncInfo.chromiumRef - const headSHA = util.runGit(config.srcDir, ['rev-parse', 'HEAD'], true) - const targetSHA = util.runGit(config.srcDir, ['rev-parse', chromiumRef], true) - const needsUpdate = targetSHA !== headSHA || (!headSHA && !targetSHA) || - JSON.stringify(latestSyncInfo) !== JSON.stringify(expectedSyncInfo) - if (needsUpdate) { - const currentRef = util.getGitReadableLocalRef(config.srcDir) - console.log(`Chromium repo ${chalk.blue.bold('needs sync')}.\n target is ${ - chalk.italic(chromiumRef)} at commit ${ - targetSHA || '[missing]'}\n current commit is ${ - chalk.italic(currentRef || '[unknown]')} at commit ${ - chalk.inverse(headSHA || '[missing]')}\n latest successful sync is ${ - JSON.stringify(latestSyncInfo, null, 4)}`) - } - else { - console.log( - chalk.green.bold(`Chromium repo does not need sync as it is already ${ - chalk.italic(chromiumRef)} at commit ${targetSHA || '[missing]'}.`)) - } - return needsUpdate -} - -function syncChromium(program) { - const requiredChromiumRef = config.getProjectRef('chrome') - let args = [ - 'sync', '--nohooks', '--revision', 'src@' + requiredChromiumRef, '--reset', - '--with_tags', '--with_branch_heads', '--upstream' - ]; - - const syncWithForce = program.init || program.force - if (syncWithForce) { - args.push('--force') - } - - const latestSyncInfoFilePath = - path.join(config.rootDir, '.brave_latest_successful_sync.json') - const latestSyncInfo = util.readJSON(latestSyncInfoFilePath, {}) - const expectedSyncInfo = { - chromiumRef: requiredChromiumRef, - gClientTimestamp: fs.statSync(config.gClientFile).mtimeMs.toString(), - } - - const chromiumNeedsUpdate = - shouldUpdateChromium(latestSyncInfo, expectedSyncInfo) - const shouldSyncChromium = chromiumNeedsUpdate || syncWithForce - if (!shouldSyncChromium && !program.sync_chromium) { - if (program.delete_unused_deps && !config.isCI) { - Log.warn( - '--delete_unused_deps is ignored for src/ dir because Chromium sync ' + - 'is required. Pass --sync_chromium to force it.') - } - return false - } - - if (program.delete_unused_deps) { - if (util.isGitExclusionExists(config.srcDir, 'brave/')) { - args.push('-D') - } else if (!config.isCI) { - Log.warn( - '--delete_unused_deps is ignored because sync has not yet added ' + - 'the exclusion for the src/brave/ directory, likely because sync ' + - 'has not previously successfully run before.') - } - } - - if (program.sync_chromium !== undefined) { - if (!program.sync_chromium) { - Log.warn( - 'Chromium needed sync but received the flag to skip performing the ' + - 'update. Working directory may not compile correctly.') - return false - } else if (!shouldSyncChromium) { - Log.warn( - 'Chromium doesn\'t need sync but received the flag to do it anyway.') - } - } - - util.runGClient(args) - util.addGitExclusion(config.srcDir, 'brave/') - util.writeJSON(latestSyncInfoFilePath, expectedSyncInfo) - - const postSyncChromiumRef = util.getGitReadableLocalRef(config.srcDir) - Log.status(`Chromium is now at ${postSyncChromiumRef || '[unknown]'}`) - return true -} - function syncBrave(program) { let args = ['sync', '--nohooks'] const syncWithForce = program.init || program.force @@ -239,11 +66,11 @@ async function RunCommand() { } if (program.init || !fs.existsSync(config.depotToolsDir)) { - maybeInstallDepotTools() + syncUtil.maybeInstallDepotTools() } if (program.init || !fs.existsSync(config.defaultGClientFile)) { - buildDefaultGClientConfig(targetOSList, targetArchList) + syncUtil.buildDefaultGClientConfig(targetOSList, targetArchList) } else if (program.target_os) { Log.warn( '--target_os is ignored. If you are attempting to sync with ' + @@ -259,7 +86,10 @@ async function RunCommand() { } Log.progressScope('gclient sync', () => { - const didSyncChromium = syncChromium(program) + const syncWithForce = program.init || program.force + + const didSyncChromium = syncUtil.syncChromium( + syncWithForce, program.sync_chromium, program.delete_unused_deps) if (!didSyncChromium || program.delete_unused_deps) { // If no Chromium sync was done, run sync inside `brave` to sync Brave DEPS. syncBrave(program) diff --git a/package.json b/package.json index 7b32ca776f4b..eca183b0d409 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "sync": "node ./build/commands/scripts/sync.js", "gclient": "node ./build/commands/scripts/gclient.js", "build": "node ./build/commands/scripts/commands.js build", + "build_chromium_release": "node ./build/commands/scripts/commands.js build_chromium_release", "gn_check": "node ./build/commands/scripts/commands.js gn_check", "versions": "node ./build/commands/scripts/commands.js versions", "upload": "node ./build/commands/scripts/commands.js upload",