Skip to content

Commit d93a2a7

Browse files
committed
fix: character encoding issues in shell command processor (google-gemini#1949)
2 parents 4f57bb3 + 12765eb commit d93a2a7

7 files changed

Lines changed: 696 additions & 9 deletions

File tree

‎package-lock.json‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/cli/src/ui/hooks/shellCommandProcessor.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,11 @@ vi.mock('os', () => ({
1919
default: {
2020
platform: () => 'linux',
2121
tmpdir: () => '/tmp',
22+
homedir: () => '/home/user',
2223
},
2324
platform: () => 'linux',
2425
tmpdir: () => '/tmp',
26+
homedir: () => '/home/user',
2527
}));
2628
vi.mock('@thacio/auditaria-cli-core');
2729
vi.mock('../utils/textUtils.js', () => ({

‎packages/cli/src/ui/hooks/shellCommandProcessor.ts‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,18 @@
66
import { t } from '@thacio/auditaria-cli-core';
77

88
import { spawn } from 'child_process';
9-
import { StringDecoder } from 'string_decoder';
9+
import { TextDecoder } from 'util';
1010
import {
1111
HistoryItemWithoutId,
1212
IndividualToolCallDisplay,
1313
ToolCallStatus,
1414
} from '../types.js';
1515
import { useCallback } from 'react';
16-
import { Config, GeminiClient } from '@thacio/auditaria-cli-core';
16+
import {
17+
Config,
18+
GeminiClient,
19+
getCachedEncodingForBuffer,
20+
} from '@thacio/auditaria-cli-core';
1721
import { type PartListUnion } from '@google/genai';
1822
import { formatMemoryUsage } from '../utils/formatters.js';
1923
import { isBinary } from '../utils/textUtils.js';
@@ -72,8 +76,8 @@ function executeShellCommand(
7276
});
7377

7478
// Use decoders to handle multi-byte characters safely (for streaming output).
75-
const stdoutDecoder = new StringDecoder('utf8');
76-
const stderrDecoder = new StringDecoder('utf8');
79+
let stdoutDecoder: TextDecoder | null = null;
80+
let stderrDecoder: TextDecoder | null = null;
7781

7882
let stdout = '';
7983
let stderr = '';
@@ -86,6 +90,12 @@ function executeShellCommand(
8690
let sniffedBytes = 0;
8791

8892
const handleOutput = (data: Buffer, stream: 'stdout' | 'stderr') => {
93+
if (!stdoutDecoder || !stderrDecoder) {
94+
const encoding = getCachedEncodingForBuffer(data);
95+
stdoutDecoder = new TextDecoder(encoding);
96+
stderrDecoder = new TextDecoder(encoding);
97+
}
98+
8999
outputChunks.push(data);
90100

91101
if (streamToUi && sniffedBytes < MAX_SNIFF_SIZE) {
@@ -102,8 +112,8 @@ function executeShellCommand(
102112

103113
const decodedChunk =
104114
stream === 'stdout'
105-
? stdoutDecoder.write(data)
106-
: stderrDecoder.write(data);
115+
? stdoutDecoder.decode(data, { stream: true })
116+
: stderrDecoder.decode(data, { stream: true });
107117
if (stream === 'stdout') {
108118
stdout += stripAnsi(decodedChunk);
109119
} else {
@@ -161,8 +171,12 @@ function executeShellCommand(
161171
abortSignal.removeEventListener('abort', abortHandler);
162172

163173
// Handle any final bytes lingering in the decoders
164-
stdout += stdoutDecoder.end();
165-
stderr += stderrDecoder.end();
174+
if (stdoutDecoder) {
175+
stdout += stdoutDecoder.decode();
176+
}
177+
if (stderrDecoder) {
178+
stderr += stderrDecoder.decode();
179+
}
166180

167181
const finalBuffer = Buffer.concat(outputChunks);
168182

‎packages/core/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@
4444
"simple-git": "^3.28.0",
4545
"strip-ansi": "^7.1.0",
4646
"undici": "^7.10.0",
47-
"ws": "^8.18.0"
47+
"ws": "^8.18.0",
48+
"chardet": "^2.1.0"
4849
},
4950
"devDependencies": {
5051
"@types/diff": "^7.0.2",

‎packages/core/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ export * from './utils/editor.js';
3535
export * from './utils/quotaErrorDetection.js';
3636
export * from './utils/fileUtils.js';
3737
export * from './utils/retry.js';
38+
export * from './utils/systemEncoding.js';
3839

3940
// Export services
4041
export * from './services/fileDiscoveryService.js';

0 commit comments

Comments
 (0)