Skip to content

Add parallel translation command #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,18 @@ To translate your language files, run the following command:
php artisan ai-translator:translate
```

To speed up translating multiple locales, you can run them in parallel. The command uses up to five processes by default:

```bash
php artisan ai-translator:translate-parallel --max-processes=5
```

Specify target locales separated by commas using the `--locale` option. For example:

```bash
php artisan ai-translator:translate-parallel --locale=ko,ja
```

This command will:

1. Recognize all language folders in your `lang` directory
Expand Down
3 changes: 2 additions & 1 deletion composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@
"guzzlehttp/guzzle": "^7.0.1",
"guzzlehttp/promises": "^2.0",
"illuminate/support": "^8.0|^9.0|^10.0|^11.0|^12.0",
"openai-php/client": "^0.10.3"
"openai-php/client": "^0.10.3",
"symfony/process": "^6.0"
},
"require-dev": {
"laravel/pint": "^1.0",
Expand Down
108 changes: 108 additions & 0 deletions src/Console/TranslateStringsParallel.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
<?php

namespace Kargnas\LaravelAiTranslator\Console;

use Symfony\Component\Process\Process;

class TranslateStringsParallel extends TranslateStrings
{
protected $signature = 'ai-translator:translate-parallel
{--s|source= : Source language to translate from (e.g. --source=en)}
{--l|locale=* : Target locales to translate (e.g. --locale=ko,ja)}
{--r|reference= : Reference languages for translation guidance (e.g. --reference=fr,es)}
{--c|chunk= : Chunk size for translation (e.g. --chunk=100)}
{--m|max-context= : Maximum number of context items to include (e.g. --max-context=1000)}
{--force-big-files : Force translation of files with more than 500 strings}
{--show-prompt : Show the whole AI prompts during translation}
{--max-processes=5 : Number of locales to translate simultaneously}
{--non-interactive : Run in non-interactive mode, using default or provided values}';

protected $description = 'Translates PHP language files in parallel for multiple locales.';

public function translate(int $maxContextItems = 100): void
{
$specifiedLocales = $this->option('locale');
$nonInteractive = $this->option('non-interactive');
$availableLocales = $this->getExistingLocales();

if (!$nonInteractive && empty($specifiedLocales)) {
$selected = $this->choiceLanguages(
$this->colors['yellow'] . 'Choose target locales for translation. Multiple selections with comma separator (e.g. "1,2")' . $this->colors['reset'],
true
);
$locales = is_array($selected) ? $selected : [$selected];
} else {
$locales = !empty($specifiedLocales)
? $this->validateAndFilterLocales($specifiedLocales, $availableLocales)
: $availableLocales;
}

if (empty($locales)) {
$this->error('No valid locales specified or found for translation.');
return;
}

$queue = [];
foreach ($locales as $locale) {
if ($locale === $this->sourceLocale || in_array($locale, config('ai-translator.skip_locales', []))) {
$this->warn('Skipping locale ' . $locale . '.');
continue;
}
$queue[] = $locale;
}

$maxProcesses = (int) ($this->option('max-processes') ?? 5);
$running = [];

while (!empty($queue) || !empty($running)) {
while (count($running) < $maxProcesses && !empty($queue)) {
$locale = array_shift($queue);
$process = new Process($this->buildLocaleCommand($locale, $maxContextItems), base_path());
$process->start();
$running[$locale] = $process;
$this->info('▶ Started translation for ' . $locale);
}

foreach ($running as $locale => $process) {
if (!$process->isRunning()) {
$this->output->write($process->getOutput());
$error = $process->getErrorOutput();
if ($error) {
$this->error($error);
}
unset($running[$locale]);
}
}

usleep(100000);
}

$this->line('\n' . $this->colors['green_bg'] . $this->colors['white'] . $this->colors['bold'] . ' All translations completed ' . $this->colors['reset']);
}

private function buildLocaleCommand(string $locale, int $maxContextItems): array
{
$cmd = [
'php',
'artisan',
'ai-translator:translate',
'--source=' . $this->sourceLocale,
'--locale=' . $locale,
'--chunk=' . $this->chunkSize,
'--max-context=' . $maxContextItems,
'--non-interactive',
];

if (!empty($this->referenceLocales)) {
$cmd[] = '--reference=' . implode(',', $this->referenceLocales);
}
if ($this->option('force-big-files')) {
$cmd[] = '--force-big-files';
}
if ($this->option('show-prompt')) {
$cmd[] = '--show-prompt';
}

return $cmd;
}
}
2 changes: 2 additions & 0 deletions src/ServiceProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
use Kargnas\LaravelAiTranslator\Console\TestTranslateCommand;
use Kargnas\LaravelAiTranslator\Console\TranslateCrowdin;
use Kargnas\LaravelAiTranslator\Console\TranslateStrings;
use Kargnas\LaravelAiTranslator\Console\TranslateStringsParallel;
use Kargnas\LaravelAiTranslator\Console\TranslateFileCommand;

class ServiceProvider extends \Illuminate\Support\ServiceProvider
Expand All @@ -26,6 +27,7 @@ public function register(): void

$this->commands([
TranslateStrings::class,
TranslateStringsParallel::class,
TranslateCrowdin::class,
TestTranslateCommand::class,
TranslateFileCommand::class,
Expand Down