Add npm run build_chromium (#21017)

* Add npm run build_chromium

* Per platfrom targets

* Add target_os & target_arch

* Fix android apk path

* Use repack-archive.py

* Use chromeVersion for output

* Change the output filename, add mac support

* Add comments

* Support linux

* Adjust build args

* Fix presubmit

* Fix review comment

* Add isChromium to config.js

* Remove changing outputDir

* Return <build_dir>

* Tune gn args

* Review fixes

* Fix non-mac builds

* Rename the script, add a warning

* Change the comment

build/commands/lib/buildChromiumRelease.js

@@ -0,0 +1,175 @@
+// Copyright (c) 2023 The Brave Authors. All rights reserved.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// A script to build and pack a Chromium release build from scratch
+// Reuses the same /src folder
+// Designed to be used on CI, but should work locally too.
+// The script includes syncing; there is no need to run npm run sync before.
+
+const config = require('./config')
+const util = require('./util')
+const path = require('path')
+const fs = require('fs-extra')
+const syncUtil = require('./syncUtils')
+const Log = require('./logging')
+
+// Use the same filename as for Brave archive.
+const getOutputFilename = () => {
+  const platform = (() => {
+    if (config.getTargetOS() === 'win')
+      return 'win32'
+    if (config.getTargetOS() === 'mac')
+      return 'darwin'
+    return config.getTargetOS()
+  })()
+  return `chromium-${config.chromeVersion}-${platform}-${config.targetArch}`
+}
+
+const chromiumConfigs = {
+  'win': {
+    buildTarget: 'mini_installer',
+    processArtifacts: () => {
+      // Repack it to reduce the size and use .zip instead of .7z.
+      input = path.join(config.outputDir, 'chrome.7z')
+      output = path.join(config.outputDir, `${getOutputFilename()}.zip`)
+      util.run('python3',
+        [
+          path.join(config.braveCoreDir, 'script', 'repack-archive.py'),
+          `--input=${input}`,
+          `--output=${output}`,
+          '--target_dir=Chrome-bin',
+        ],
+        config.defaultOptions)
+    }
+  },
+  'linux': {
+    buildTarget: 'chrome/installer/linux:stable_deb',
+    processArtifacts: () => {
+      const debArch = (() => {
+        if (config.targetArch === 'x64') return 'amd64'
+        return config.targetArch
+      })()
+      fs.moveSync(
+        path.join(config.outputDir,
+          `chromium-browser-stable_${config.chromeVersion}-1_${debArch}.deb`),
+        path.join(config.outputDir, `${getOutputFilename()}.deb`))
+    }
+  },
+  'mac': {
+    buildTarget: 'chrome',
+    extraHooks: () => {
+      Log.progressScope('download_hermetic_xcode', () => {
+        util.run('vpython3',
+          [
+            path.join(config.braveCoreDir,
+                      'build', 'mac', 'download_hermetic_xcode.py'),
+          ],
+          config.defaultOptions)
+      })
+    },
+    processArtifacts: () => {
+      util.run('zip',
+        ['-r', '-y', `${getOutputFilename()}.zip`, 'Chromium.app'],
+        { cwd: config.outputDir }
+      )
+    }
+  },
+  'android': {
+    buildTarget: 'chrome_public_apk',
+    processArtifacts: () => {
+      fs.moveSync(
+        path.join(config.outputDir, 'apks', 'ChromePublic.apk'),
+        path.join(config.outputDir, `${getOutputFilename()}.apk`))
+    }
+  },
+}
+
+// A function to make gn args to build a release Chromium build.
+// There is two primarily sources:
+// 1. Chromium perf builds: tools/mb/mb_config_expectations/chromium.perf.json
+// 2. Brave Release build configuration
+function getChromiumGnArgs() {
+  const targetOs = config.getTargetOS()
+  const targetArch = config.targetArch
+  const args = {
+    target_cpu: targetArch,
+    target_os: targetOs,
+    is_official_build: true,
+    enable_keystone_registration_framework: false,
+    ffmpeg_branding: 'Chrome',
+    enable_widevine: true,
+    ignore_missing_widevine_signing_cert: true,
+    ...config.extraGnArgs,
+  }
+
+  if (targetOs === 'android') {
+    args.debuggable_apks = false
+  } else {
+    args.enable_hangout_services_extension = true
+    args.enable_nacl = false
+  }
+
+  if (targetOs === 'mac') {
+    args.use_system_xcode = true
+  }
+
+  return args
+}
+
+function buildChromiumRelease(buildOptions = {}) {
+  if (!config.isCI && !buildOptions.force) {
+    console.error(
+      'Warning: the command resets all changes in src/ folder.\n' +
+      'src/brave stays untouched. Pass --force to continue.')
+    return 1
+  }
+  config.buildConfig = 'Release'
+  config.isChromium = true
+  config.update(buildOptions)
+
+  const chromiumConfig = chromiumConfigs[config.getTargetOS()]
+  if (chromiumConfig == undefined)
+    throw Error(`${config.getTargetOS()} is unsupported`)
+
+  syncUtil.maybeInstallDepotTools()
+  syncUtil.buildDefaultGClientConfig(
+    [config.getTargetOS()], [config.targetArch], true)
+
+  util.runGit(config.srcDir, ['clean', '-f', '-d'])
+
+
+  Log.progressScope('gclient sync', () => {
+    syncUtil.syncChromium(true, true, false)
+  })
+
+  Log.progressScope('gclient runhooks', () => {
+    util.runGClient(['runhooks'])
+  })
+
+  if (chromiumConfig.extraHooks != undefined) {
+    chromiumConfig.extraHooks()
+  }
+
+  const options = config.defaultOptions
+  const buildArgsStr = util.buildArgsToString(getChromiumGnArgs())
+  util.run('gn', ['gen', config.outputDir, '--args="' + buildArgsStr + '"'],
+    options)
+
+
+  Log.progressScope(`ninja`, () => {
+    const target = chromiumConfig.buildTarget
+    const ninjaOpts = [
+      '-C', options.outputDir || config.outputDir, target,
+      ...config.extraNinjaOpts
+    ]
+    util.run('autoninja', ninjaOpts, config.defaultOptions)
+  })
+
+  Log.progressScope('make archive', () => {
+    chromiumConfig.processArtifacts()
+  })
+}
+
+module.exports = buildChromiumRelease

build/commands/lib/config.js

@@ -107,6 +107,7 @@ const Config = function () {
   this.buildTarget = 'brave'
   this.rootDir = rootDir
   this.isUniversalBinary = false
+  this.isChromium = false
   this.scriptDir = path.join(this.rootDir, 'scripts')
   this.srcDir = path.join(this.rootDir, 'src')
   this.chromeVersion = this.getProjectVersion('chrome')
@@ -1289,6 +1290,9 @@ Object.defineProperty(Config.prototype, 'outputDir', {
     if (this.targetEnvironment) {
       buildConfigDir = buildConfigDir + "_" + this.targetEnvironment
     }
+    if (this.isChromium) {
+      buildConfigDir = buildConfigDir + "_chromium"
+    }
 
     return path.join(baseDir, buildConfigDir)
   },

build/commands/lib/syncUtils.js

@@ -0,0 +1,197 @@
+// Copyright (c) 2023 The Brave Authors. All rights reserved.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+const chalk = require('chalk')
+const config = require('./config')
+const fs = require('fs')
+const path = require('path')
+const Log = require('./logging')
+const util = require('./util')
+
+
+function maybeInstallDepotTools(options = config.defaultOptions) {
+  options.cwd = config.braveCoreDir
+
+  if (!fs.existsSync(config.depotToolsDir)) {
+    Log.progressScope('install depot_tools', () => {
+      fs.mkdirSync(config.depotToolsDir)
+      util.run(
+        'git',
+        [
+          '-C',
+          config.depotToolsDir,
+          'clone',
+          'https://chromium.googlesource.com/chromium/tools/depot_tools.git',
+          '.'
+        ],
+        options
+      )
+    })
+  }
+
+  const ninjaLogCfgPath = path.join(config.depotToolsDir, 'ninjalog.cfg');
+  if (!fs.existsSync(ninjaLogCfgPath)) {
+    // Create a ninja config to prevent (auto)ninja from calling goma_auth
+    // each time. See for details:
+    // https://chromium.googlesource.com/chromium/tools/depot_tools/+/main/ninjalog.README.md
+    const ninjaLogCfgConfig = {
+      'is-googler': false,
+      'version': 3,
+      'countdown': 10,
+      'opt-in': false,
+    };
+    fs.writeFileSync(ninjaLogCfgPath, JSON.stringify(ninjaLogCfgConfig))
+  }
+}
+
+function toGClientConfigItem(name, value, pretty = true) {
+  // Convert value to json and replace "%True%" -> True, "%False%" -> False,
+  // "%None%" -> None.
+  const pythonLikeValue =
+      JSON.stringify(value, null, pretty ? 2 : 0).replace(/"%(.*?)%"/gm, '$1')
+  return `${name} = ${pythonLikeValue}\n`
+}
+
+function buildDefaultGClientConfig(
+  targetOSList, targetArchList, onlyChromium = false) {
+  const items = [
+    {
+      managed: '%False%',
+      name: 'src',
+      url: config.chromiumRepo,
+      custom_deps: {
+        'src/third_party/WebKit/LayoutTests': '%None%',
+        'src/chrome_frame/tools/test/reference_build/chrome': '%None%',
+        'src/chrome_frame/tools/test/reference_build/chrome_win': '%None%',
+        'src/chrome/tools/test/reference_build/chrome': '%None%',
+        'src/chrome/tools/test/reference_build/chrome_linux': '%None%',
+        'src/chrome/tools/test/reference_build/chrome_mac': '%None%',
+        'src/chrome/tools/test/reference_build/chrome_win': '%None%'
+      },
+      custom_vars: {
+        'checkout_rust': '%True%',
+        'checkout_pgo_profiles': config.isBraveReleaseBuild() ? '%True%' :
+                                                                '%False%'
+      }
+    }
+  ]
+
+  if (!onlyChromium) {
+    items.push({
+      managed: '%False%',
+      name: 'src/brave',
+      // We do not use gclient to manage brave-core, so this should not
+      // actually get used.
+      url: 'https://github.com/brave/brave-core.git'
+    })
+  }
+
+  let out = toGClientConfigItem('solutions', items);
+
+  if (process.env.GIT_CACHE_PATH) {
+    out += toGClientConfigItem('cache_dir', process.env.GIT_CACHE_PATH)
+  }
+  if (targetOSList) {
+    out += toGClientConfigItem('target_os', targetOSList, false)
+  }
+  if (targetArchList) {
+    out += toGClientConfigItem('target_cpu', targetArchList, false)
+  }
+
+  fs.writeFileSync(config.defaultGClientFile, out)
+}
+
+function shouldUpdateChromium(latestSyncInfo, expectedSyncInfo) {
+  const chromiumRef = expectedSyncInfo.chromiumRef
+  const headSHA = util.runGit(config.srcDir, ['rev-parse', 'HEAD'], true)
+  const targetSHA = util.runGit(config.srcDir, ['rev-parse', chromiumRef], true)
+  const needsUpdate = targetSHA !== headSHA || (!headSHA && !targetSHA) ||
+      JSON.stringify(latestSyncInfo) !== JSON.stringify(expectedSyncInfo)
+  if (needsUpdate) {
+    const currentRef = util.getGitReadableLocalRef(config.srcDir)
+    console.log(`Chromium repo ${chalk.blue.bold('needs sync')}.\n  target is ${
+        chalk.italic(chromiumRef)} at commit ${
+        targetSHA || '[missing]'}\n  current commit is ${
+        chalk.italic(currentRef || '[unknown]')} at commit ${
+        chalk.inverse(headSHA || '[missing]')}\n  latest successful sync is ${
+        JSON.stringify(latestSyncInfo, null, 4)}`)
+  }
+  else {
+    console.log(
+        chalk.green.bold(`Chromium repo does not need sync as it is already ${
+            chalk.italic(chromiumRef)} at commit ${targetSHA || '[missing]'}.`))
+  }
+  return needsUpdate
+}
+
+function syncChromium(syncWithForce, sync_chromium, delete_unused_deps) {
+  const requiredChromiumRef = config.getProjectRef('chrome')
+  let args = [
+    'sync', '--nohooks', '--revision', 'src@' + requiredChromiumRef, '--reset',
+    '--with_tags', '--with_branch_heads', '--upstream'
+  ];
+
+  if (syncWithForce) {
+    args.push('--force')
+  }
+
+  const latestSyncInfoFilePath =
+      path.join(config.rootDir, '.brave_latest_successful_sync.json')
+  const latestSyncInfo = util.readJSON(latestSyncInfoFilePath, {})
+  const expectedSyncInfo = {
+    chromiumRef: requiredChromiumRef,
+    gClientTimestamp: fs.statSync(config.gClientFile).mtimeMs.toString(),
+  }
+
+  const chromiumNeedsUpdate =
+      shouldUpdateChromium(latestSyncInfo, expectedSyncInfo)
+  const shouldSyncChromium = chromiumNeedsUpdate || syncWithForce
+  if (!shouldSyncChromium && !sync_chromium) {
+    if (delete_unused_deps && !config.isCI) {
+      Log.warn(
+        '--delete_unused_deps is ignored for src/ dir because Chromium sync ' +
+        'is required. Pass --sync_chromium to force it.')
+    }
+    return false
+  }
+
+  if (delete_unused_deps) {
+    if (util.isGitExclusionExists(config.srcDir, 'brave/')) {
+      args.push('-D')
+    } else if (!config.isCI) {
+      Log.warn(
+          '--delete_unused_deps is ignored because sync has not yet added ' +
+          'the exclusion for the src/brave/ directory, likely because sync ' +
+          'has not previously successfully run before.')
+    }
+  }
+
+  if (sync_chromium !== undefined) {
+    if (!sync_chromium) {
+      Log.warn(
+          'Chromium needed sync but received the flag to skip performing the ' +
+          'update. Working directory may not compile correctly.')
+      return false
+    } else if (!shouldSyncChromium) {
+      Log.warn(
+          'Chromium doesn\'t need sync but received the flag to do it anyway.')
+    }
+  }
+
+  util.runGClient(args)
+  util.addGitExclusion(config.srcDir, 'brave/')
+  util.writeJSON(latestSyncInfoFilePath, expectedSyncInfo)
+
+  const postSyncChromiumRef = util.getGitReadableLocalRef(config.srcDir)
+  Log.status(`Chromium is now at ${postSyncChromiumRef || '[unknown]'}`)
+  return true
+}
+
+
+module.exports = {
+  maybeInstallDepotTools,
+  buildDefaultGClientConfig,
+  syncChromium
+}