From 2b0b35084088ef3e077bf1f84142194a1948063f Mon Sep 17 00:00:00 2001 From: Guy Sartorelli Date: Tue, 7 May 2024 13:35:43 +1200 Subject: [PATCH] Revert "ENH Process modules one at a time" This reverts commit 255e75a3a9a2aa2607a1116903524d0870488a58. --- src/Translator.php | 360 +++++++++++++++++++++++---------------------- 1 file changed, 186 insertions(+), 174 deletions(-) diff --git a/src/Translator.php b/src/Translator.php index 9eb5df2..372fe49 100644 --- a/src/Translator.php +++ b/src/Translator.php @@ -9,7 +9,6 @@ use RuntimeException; use LogicException; use Symfony\Component\Console\Formatter\OutputFormatter; -use Exception; class Translator { @@ -33,6 +32,8 @@ class Translator private string $githubToken = ''; + private array $modulePaths = []; + private array $originalJson = []; private array $originalYaml = []; @@ -47,37 +48,30 @@ public function run() { $this->outputFormatter = new OutputFormatter(true); $this->checkEnv(); - $modulePaths = $this->getModulePaths(); - $this->log('Updating translations for ' . count($modulePaths) . ' module(s)'); - try { - foreach ($modulePaths as $modulePath) { - if ($this->doTransifexPullAndUpdate) { - $this->storeJson($modulePath); - $this->storeYaml($modulePath); - $this->setJsonAndYmlFileTimes($modulePath); - $this->transifexPullSource($modulePath); - $this->mergeYaml(); - $this->removeEnglishStringsFromYamlTranslations(); - $this->cleanYaml($modulePath); - $this->mergeJson(); - $this->removeEnglishStringsFromJsonTranslations(); - } - if ($this->doCollectStrings) { - $this->collectStrings($modulePath); - } - if ($this->doTransifexPullAndUpdate) { - $this->generateJavascript($modulePath); - } - if ($this->doTransifexPush) { - $this->transifexPushSource($modulePath); - } - $this->gitCommitPushAndPullRequest($modulePath); - } - } catch (Exception $e) { - throw $e; - } finally { - $this->outputPullRequestUrls(); + $this->setModulePaths(); + if ($this->doTransifexPullAndUpdate) { + $this->log('Updating translations for ' . count($this->modulePaths) . ' module(s)'); + $this->storeJson(); + $this->storeYaml(); + $this->setJsonAndYmlFileTimes(); + $this->transifexPullSource(); + $this->mergeYaml(); + $this->removeEnglishStringsFromYamlTranslations(); + $this->cleanYaml(); + $this->mergeJson(); + $this->removeEnglishStringsFromJsonTranslations(); + } + if ($this->doCollectStrings) { + $this->collectStrings(); + } + if ($this->doTransifexPullAndUpdate) { + $this->generateJavascript(); + } + if ($this->doTransifexPush) { + $this->transifexPushSource(); } + $this->gitCommitPushAndPullRequest(); + $this->outputPullRequestUrls(); } private function checkEnv(): void @@ -150,9 +144,8 @@ private function getFrameworkMajor(): string return $matches[1]; } - private function getModulePaths(): array + private function setModulePaths(): void { - $modulePaths = []; $client = new Client(); $cmsMajor = $this->getFrameworkMajor(); $url = "https://raw.githubusercontent.com/silverstripe/supported-modules/$cmsMajor/modules.json"; @@ -193,10 +186,9 @@ private function getModulePaths(): array if (!is_numeric($branch)) { throw new RuntimeException("Branch $branch in $modulePath is not a minor or next-minor branch"); } - $modulePaths[] = $modulePath; + $this->modulePaths[] = $modulePath; } } - return $modulePaths; } /** @@ -213,16 +205,18 @@ private function getCleanBranch(string $branch): string /** * Backup local json files prior to replacing local copies with transifex */ - private function storeJson(string $modulePath): void + private function storeJson(): void { $this->log('Backing up local json files'); // Backup files prior to replacing local copies with transifex $this->originalJson = []; - $jsPath = $this->getJSLangDirectories($modulePath); - foreach ((array) $jsPath as $langDir) { - foreach (glob($langDir . '/src/*.json') as $path) { - $str = file_get_contents($path); - $this->originalJson[$path] = $this->jsonDecode($str); + foreach ($this->modulePaths as $modulePath) { + $jsPath = $this->getJSLangDirectories($modulePath); + foreach ((array) $jsPath as $langDir) { + foreach (glob($langDir . '/src/*.json') as $path) { + $str = file_get_contents($path); + $this->originalJson[$path] = $this->jsonDecode($str); + } } } $this->log('Finished backing up ' . count($this->originalJson) . ' json files'); @@ -231,13 +225,15 @@ private function storeJson(string $modulePath): void /** * Backup local yaml files in memory prior to replacing local copies with transifex */ - private function storeYaml(string $modulePath): void + private function storeYaml(): void { $this->log('Backing up local yaml files'); $this->originalYaml = []; - foreach (glob($this->getYmlLangDirectory($modulePath) . '/*.yml') as $path) { - $rawYaml = file_get_contents($path); - $this->originalYaml[$path] = Yaml::parse($rawYaml); + foreach ($this->modulePaths as $modulePath) { + foreach (glob($this->getYmlLangDirectory($modulePath) . '/*.yml') as $path) { + $rawYaml = file_get_contents($path); + $this->originalYaml[$path] = Yaml::parse($rawYaml); + } } $this->log('Finished backing up ' . count($this->originalYaml) . ' yaml files'); } @@ -245,41 +241,45 @@ private function storeYaml(string $modulePath): void /** * Set mtime to a year ago so that transifex will see these as obsolete */ - private function setJsonAndYmlFileTimes(string $modulePath): void + private function setJsonAndYmlFileTimes() { $date = date('YmdHi.s', strtotime('-1 year')); - $name = $this->getModuleName($modulePath); - $this->log("Setting file mtime to a past date for {$name}"); - $ymlLang = $this->getYmlLangDirectory($modulePath); - if ($ymlLang) { - $this->exec("find $ymlLang -type f \( -name \"*.yml\" \) -exec touch -t $date {} \;"); - } - foreach ($this->getJSLangDirectories($modulePath) as $jsLangDir) { - $this->exec("find $jsLangDir -type f \( -name \"*.json*\" \) -exec touch -t $date {} \;"); + foreach ($this->modulePaths as $modulePath) { + $name = $this->getModuleName($modulePath); + $this->log("Setting file mtime to a past date for {$name}"); + $ymlLang = $this->getYmlLangDirectory($modulePath); + if ($ymlLang) { + $this->exec("find $ymlLang -type f \( -name \"*.yml\" \) -exec touch -t $date {} \;"); + } + foreach ($this->getJSLangDirectories($modulePath) as $jsLangDir) { + $this->exec("find $jsLangDir -type f \( -name \"*.json*\" \) -exec touch -t $date {} \;"); + } } } /** * Update sources from transifex */ - private function transifexPullSource(string $modulePath): void - { - $this->log("Pulling translations for $modulePath"); - // ensure .tx/config is up to date - $contents = file_get_contents("$modulePath/.tx/config"); - if (strpos($contents, '[o:') === false) { - $this->exec('tx migrate', $modulePath); - // delete .bak files created as part of tx migrate - foreach (scandir("$modulePath/.tx") as $filename) { - if (pathinfo($filename, PATHINFO_EXTENSION) !== 'bak') { - continue; + private function transifexPullSource() + { + foreach ($this->modulePaths as $modulePath) { + $this->log("Pulling translations for $modulePath"); + // ensure .tx/config is up to date + $contents = file_get_contents("$modulePath/.tx/config"); + if (strpos($contents, '[o:') === false) { + $this->exec('tx migrate', $modulePath); + // delete .bak files created as part of tx migrate + foreach (scandir("$modulePath/.tx") as $filename) { + if (pathinfo($filename, PATHINFO_EXTENSION) !== 'bak') { + continue; + } + $this->log("Deleting $modulePath/.tx/$filename"); + unlink("$modulePath/.tx/$filename"); } - $this->log("Deleting $modulePath/.tx/$filename"); - unlink("$modulePath/.tx/$filename"); } + // pull from transifex + $this->exec("tx pull -a -s -t -f --minimum-perc={$this->txMinimumPerc}", $modulePath); } - // pull from transifex - $this->exec("tx pull -a -s -t -f --minimum-perc={$this->txMinimumPerc}", $modulePath); } /** @@ -408,23 +408,25 @@ private function isEmpty($val) /** * Tidy yaml files using symfony yaml */ - private function cleanYaml($modulePath) - { - $name = $this->getModuleName($modulePath); - $this->log("Cleaning YAML sources for {$name}"); - $num = 0; - foreach (glob($this->getYmlLangDirectory($modulePath) . '/*.yml') as $sourceFile) { - $dirty = file_get_contents($sourceFile); - $sourceData = Yaml::parse($dirty); - $this->removeBlankStrings($sourceData); - $this->recursiveKeySort($sourceData); - $cleaned = Yaml::dump($sourceData, 9999, 2); - if ($dirty !== $cleaned) { - $num++; - file_put_contents($sourceFile, $cleaned); + private function cleanYaml() + { + foreach ($this->modulePaths as $modulePath) { + $name = $this->getModuleName($modulePath); + $this->log("Cleaning YAML sources for {$name}"); + $num = 0; + foreach (glob($this->getYmlLangDirectory($modulePath) . '/*.yml') as $sourceFile) { + $dirty = file_get_contents($sourceFile); + $sourceData = Yaml::parse($dirty); + $this->removeBlankStrings($sourceData); + $this->recursiveKeySort($sourceData); + $cleaned = Yaml::dump($sourceData, 9999, 2); + if ($dirty !== $cleaned) { + $num++; + file_put_contents($sourceFile, $cleaned); + } } + $this->log("{$num} yml files cleaned"); } - $this->log("{$num} yml files cleaned"); } /** @@ -483,12 +485,16 @@ private function removeEnglishStringsFromJsonTranslations(): void } /** - * Run text collector on the given module + * Run text collector on the given modules */ - private function collectStrings(string $modulePath): void + private function collectStrings(): void { $this->log('Running i18nTextCollectorTask'); - $module = urlencode($this->getModuleName($modulePath)); + $modulesNames = []; + foreach ($this->modulePaths as $modulePath) { + $modulesNames[] = $this->getModuleName($modulePath); + } + $module = urlencode(implode(',', $modulesNames)); $site = rtrim($this->txSite, '/'); $this->exec("wget --content-on-error $site/dev/tasks/i18nTextCollectorTask?module=$module"); } @@ -496,108 +502,112 @@ private function collectStrings(string $modulePath): void /** * Push source updates to transifex */ - private function transifexPushSource(string $modulePath): void + private function transifexPushSource(): void { $this->log('Pushing updated sources to transifex'); if ($this->isDevMode) { $this->log('Not pushing to transifex because TX_DEV_MODE is enabled'); return; } - $this->exec('tx push -s', $modulePath); + foreach ($this->modulePaths as $modulePath) { + $this->exec('tx push -s', $modulePath); + } } /** - * Commit changes for the module + * Commit changes for all modules */ - private function gitCommitPushAndPullRequest(string $modulePath): void + private function gitCommitPushAndPullRequest(): void { $this->log('Committing translations to git'); - $this->log("Committing translations for $modulePath"); - // Get endpoint - $remote = $this->exec('git config --get remote.origin.url', $modulePath); - if (!preg_match('#^(https://github\.com/|git@github\.com:)([^/]+)/(.+?)(\.git)?$#', $remote, $matches)) { - throw new RuntimeException("Invalid git remote $remote"); - } - $account = $matches[2]; - $repo = $matches[3]; - $endpoint = "https://api.github.com/repos/$account/$repo/pulls"; - - // Add remote - if (!in_array('tx-ccs', explode("\n", $this->exec('git remote', $modulePath)))) { - $this->exec("git remote add tx-ccs git@github.com:creative-commoners/$repo.git", $modulePath); - } - - // Git add all changes - $jsPath = $this->getJSLangDirectories($modulePath); - $langPath = $this->getYmlLangDirectory($modulePath); - foreach (array_merge((array) $jsPath, (array) $langPath) as $path) { - if (is_dir($path)) { - $this->exec("git add $path/*", $modulePath); + foreach ($this->modulePaths as $modulePath) { + $this->log("Committing translations for $modulePath"); + // Get endpoint + $remote = $this->exec('git config --get remote.origin.url', $modulePath); + if (!preg_match('#^(https://github\.com/|git@github\.com:)([^/]+)/(.+?)(\.git)?$#', $remote, $matches)) { + throw new RuntimeException("Invalid git remote $remote"); + } + $account = $matches[2]; + $repo = $matches[3]; + $endpoint = "https://api.github.com/repos/$account/$repo/pulls"; + + // Add remote + if (!in_array('tx-ccs', explode("\n", $this->exec('git remote', $modulePath)))) { + $this->exec("git remote add tx-ccs git@github.com:creative-commoners/$repo.git", $modulePath); + } + + // Git add all changes + $jsPath = $this->getJSLangDirectories($modulePath); + $langPath = $this->getYmlLangDirectory($modulePath); + foreach (array_merge((array) $jsPath, (array) $langPath) as $path) { + if (is_dir($path)) { + $this->exec("git add $path/*", $modulePath); + } } - } - $this->exec("git add .tx/config", $modulePath); + $this->exec("git add .tx/config", $modulePath); - // Check if there's anything to commit - $status = $this->exec('git status', $modulePath); - if (strpos($status, 'nothing to commit') !== false) { - $this->log('Nothing to commit'); - return; - } + // Check if there's anything to commit + $status = $this->exec('git status', $modulePath); + if (strpos($status, 'nothing to commit') !== false) { + $this->log('Nothing to commit, continuing'); + continue; + } - // Create new branch - $currentBranch = $this->exec('git rev-parse --abbrev-ref HEAD', $modulePath); - $time = time(); - $branch = "pulls/$currentBranch/tx-$time"; - $this->exec("git checkout -b $branch", $modulePath); + // Create new branch + $currentBranch = $this->exec('git rev-parse --abbrev-ref HEAD', $modulePath); + $time = time(); + $branch = "pulls/$currentBranch/tx-$time"; + $this->exec("git checkout -b $branch", $modulePath); - // Commit changes - $title = 'TLN Update translations'; - $this->exec("git commit -m \"$title\"", $modulePath); + // Commit changes + $title = 'TLN Update translations'; + $this->exec("git commit -m \"$title\"", $modulePath); - if ($this->isDevMode) { - $this->log(implode(' ', [ - 'Not pushing changes or creating pull-request because TX_DEV_MODE is enabled.', - 'A new branch was created.' - ])); - return; - } + if ($this->isDevMode) { + $this->log(implode(' ', [ + 'Not pushing changes or creating pull-request because TX_DEV_MODE is enabled.', + 'A new branch was created.' + ])); + continue; + } - // Push changes to creative-commoners - $this->exec("git push --set-upstream tx-ccs $branch", $modulePath); + // Push changes to creative-commoners + $this->exec("git push --set-upstream tx-ccs $branch", $modulePath); - // Ensure secondary-rate limit is not exceeded - $seconds = 10; - $this->log("Sleeping $seconds seconds so that secondary rate-limit is not exceeded"); - sleep($seconds); + // Ensure secondary-rate limit is not exceeded + $seconds = 10; + $this->log("Sleeping $seconds seconds so that secondary rate-limit is not exceeded"); + sleep($seconds); - // Create pull-request via github api - // https://docs.github.com/en/rest/pulls/pulls#create-a-pull-request - $body = implode(' ', [ - 'Automated translations update generated using', - '[silverstripe/tx-translator](https://github.com/silverstripe/silverstripe-tx-translator)' - ]); - $client = new Client(); - $response = $client->request('POST', $endpoint, [ - 'headers' => [ - "Accept" => "application/vnd.github.v3+json", - "Authorization" => "token {$this->githubToken}" - ], - 'body' => $this->jsonEncode([ - "title" => $title, - "body" => $body, - "head" => "creative-commoners:$branch", - "base" => $currentBranch - ]) - ]); - $code = $response->getStatusCode(); - $responseBody = (string) $response->getBody(); - if ($code === 201) { - $pullRequestUrl = json_decode($responseBody)->html_url; - $this->log("Pull request was succesfully created at $pullRequestUrl"); - $this->pullRequestUrls[] = $pullRequestUrl; - } else { - $this->log($responseBody); - throw new RuntimeException("Pull request failed - status code was $code"); + // Create pull-request via github api + // https://docs.github.com/en/rest/pulls/pulls#create-a-pull-request + $body = implode(' ', [ + 'Automated translations update generated using', + '[silverstripe/tx-translator](https://github.com/silverstripe/silverstripe-tx-translator)' + ]); + $client = new Client(); + $response = $client->request('POST', $endpoint, [ + 'headers' => [ + "Accept" => "application/vnd.github.v3+json", + "Authorization" => "token {$this->githubToken}" + ], + 'body' => $this->jsonEncode([ + "title" => $title, + "body" => $body, + "head" => "creative-commoners:$branch", + "base" => $currentBranch + ]) + ]); + $code = $response->getStatusCode(); + $responseBody = (string) $response->getBody(); + if ($code === 201) { + $pullRequestUrl = json_decode($responseBody)->html_url; + $this->log("Pull request was succesfully created at $pullRequestUrl"); + $this->pullRequestUrls[] = $pullRequestUrl; + } else { + $this->log($responseBody); + throw new RuntimeException("Pull request failed - status code was $code"); + } } } @@ -684,16 +694,18 @@ private function getTransifexSources(string $modulePath): array } /** - * Generate javascript for the module + * Generate javascript for all modules */ - private function generateJavascript(string $modulePath): void + private function generateJavascript(): void { $this->log('Generating javascript locale files'); // Check which paths in each module require processing $count = 0; - $jsPaths = $this->getJSLangDirectories($modulePath); - foreach ((array)$jsPaths as $jsPath) { - $count += $this->generateJavascriptInDirectory($modulePath, $jsPath); + foreach ($this->modulePaths as $modulePath) { + $jsPaths = $this->getJSLangDirectories($modulePath); + foreach ((array)$jsPaths as $jsPath) { + $count += $this->generateJavascriptInDirectory($modulePath, $jsPath); + } } $this->log("Finished generating {$count} files"); }