Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -56,74 +56,82 @@ error_count=0
warning_count=0

# Check name field
NAME=$(echo "$FRONTMATTER" | grep '^name:' | sed 's/name: *//' | sed 's/^"\(.*\)"$/\1/')
# The "|| true" on each field extraction keeps grep's no-match exit status from
# killing the script under set -e; a missing field is reported below instead.
NAME=$(echo "$FRONTMATTER" | grep '^name:' | sed 's/name: *//' | sed 's/^"\(.*\)"$/\1/' || true)

if [ -z "$NAME" ]; then
echo "❌ Missing required field: name"
((error_count++))
error_count=$((error_count + 1))
else
echo "✅ name: $NAME"

# Validate name format
if ! [[ "$NAME" =~ ^[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9]$ ]]; then
echo "❌ name must start/end with alphanumeric and contain only letters, numbers, hyphens"
((error_count++))
error_count=$((error_count + 1))
fi

# Validate name length
name_length=${#NAME}
if [ $name_length -lt 3 ]; then
echo "❌ name too short (minimum 3 characters)"
((error_count++))
error_count=$((error_count + 1))
elif [ $name_length -gt 50 ]; then
echo "❌ name too long (maximum 50 characters)"
((error_count++))
error_count=$((error_count + 1))
fi

# Check for generic names
if [[ "$NAME" =~ ^(helper|assistant|agent|tool)$ ]]; then
echo "⚠️ name is too generic: $NAME"
((warning_count++))
warning_count=$((warning_count + 1))
fi
fi

# Check description field
DESCRIPTION=$(echo "$FRONTMATTER" | grep '^description:' | sed 's/description: *//')
# The description may span multiple lines (unquoted prose plus <example> blocks),
# so capture everything from "description:" until the next top-level agent key.
DESCRIPTION=$(echo "$FRONTMATTER" | awk '
/^description:/ { in_description=1; sub(/^description:[[:space:]]*/, ""); print; next }
/^(name|model|color|tools):/ { in_description=0 }

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any key outside this list that follows description: is absorbed into it: permissionMode: default + maxTurns: 5 gives 112 reported characters for a 76-character description. Stopping at any top-level key is not the fix either, since it cuts plugin-dev's own agents at their unindented Context: lines (1170 → 255). The shape the product parses is a description: |- block scalar.

in_description { print }
')

if [ -z "$DESCRIPTION" ]; then
echo "❌ Missing required field: description"
((error_count++))
error_count=$((error_count + 1))
else
desc_length=${#DESCRIPTION}
echo "✅ description: ${desc_length} characters"

if [ $desc_length -lt 10 ]; then
echo "⚠️ description too short (minimum 10 characters recommended)"
((warning_count++))
warning_count=$((warning_count + 1))
elif [ $desc_length -gt 5000 ]; then
echo "⚠️ description very long (over 5000 characters)"
((warning_count++))
warning_count=$((warning_count + 1))
fi

# Check for example blocks
if ! echo "$DESCRIPTION" | grep -q '<example>'; then
echo "⚠️ description should include <example> blocks for triggering"
((warning_count++))
warning_count=$((warning_count + 1))
fi

# Check for "Use this agent when" pattern
if ! echo "$DESCRIPTION" | grep -qi 'use this agent when'; then
echo "⚠️ description should start with 'Use this agent when...'"
((warning_count++))
warning_count=$((warning_count + 1))
fi
fi

# Check model field
MODEL=$(echo "$FRONTMATTER" | grep '^model:' | sed 's/model: *//')
MODEL=$(echo "$FRONTMATTER" | grep '^model:' | sed 's/model: *//' || true)

if [ -z "$MODEL" ]; then
echo "❌ Missing required field: model"
((error_count++))
error_count=$((error_count + 1))
else
echo "✅ model: $MODEL"

Expand All @@ -133,17 +141,17 @@ else
;;
*)
echo "⚠️ Unknown model: $MODEL (valid: inherit, sonnet, opus, haiku)"
((warning_count++))
warning_count=$((warning_count + 1))
;;
esac
fi

# Check color field
COLOR=$(echo "$FRONTMATTER" | grep '^color:' | sed 's/color: *//')
COLOR=$(echo "$FRONTMATTER" | grep '^color:' | sed 's/color: *//' || true)

if [ -z "$COLOR" ]; then
echo "❌ Missing required field: color"
((error_count++))
error_count=$((error_count + 1))
else
echo "✅ color: $COLOR"

Expand All @@ -153,13 +161,13 @@ else
;;
*)
echo "⚠️ Unknown color: $COLOR (valid: blue, cyan, green, yellow, magenta, red)"
((warning_count++))
warning_count=$((warning_count + 1))
;;
esac
fi

# Check tools field (optional)
TOOLS=$(echo "$FRONTMATTER" | grep '^tools:' | sed 's/tools: *//')
TOOLS=$(echo "$FRONTMATTER" | grep '^tools:' | sed 's/tools: *//' || true)

if [ -n "$TOOLS" ]; then
echo "✅ tools: $TOOLS"
Expand All @@ -173,23 +181,23 @@ echo "Checking system prompt..."

if [ -z "$SYSTEM_PROMPT" ]; then
echo "❌ System prompt is empty"
((error_count++))
error_count=$((error_count + 1))
else
prompt_length=${#SYSTEM_PROMPT}
echo "✅ System prompt: $prompt_length characters"

if [ $prompt_length -lt 20 ]; then
echo "❌ System prompt too short (minimum 20 characters)"
((error_count++))
error_count=$((error_count + 1))
elif [ $prompt_length -gt 10000 ]; then
echo "⚠️ System prompt very long (over 10,000 characters)"
((warning_count++))
warning_count=$((warning_count + 1))
fi

# Check for second person
if ! echo "$SYSTEM_PROMPT" | grep -q "You are\|You will\|Your"; then
echo "⚠️ System prompt should use second person (You are..., You will...)"
((warning_count++))
warning_count=$((warning_count + 1))
fi

# Check for structure
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#!/bin/bash
# Regression tests for validate-agent.sh (anthropics/claude-code#83803):
# - warnings must not abort the run: under `set -e`, `((x++))` returns nonzero
# when x was 0, so the first warning killed the script with exit 1
# - multi-line descriptions (prose plus <example> blocks) must not be
# false-flagged as missing examples
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
VALIDATOR="$SCRIPT_DIR/validate-agent.sh"
PLUGIN_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
TMP_DIR="$(mktemp -d)" || exit 1
trap 'rm -rf "$TMP_DIR"' EXIT

failures=0

check() {
local label="$1" expected="$2" file="$3"
local actual=0
bash "$VALIDATOR" "$file" > "$TMP_DIR/out.txt" 2>&1 || actual=$?
if [ "$actual" -eq "$expected" ]; then
echo "PASS: $label (exit $actual)"
else
echo "FAIL: $label (expected exit $expected, got $actual)"
cat "$TMP_DIR/out.txt"
failures=$((failures + 1))
fi
}

# The plugin's own agents are valid and must pass.
for agent in "$PLUGIN_ROOT"/agents/*.md; do

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These three files fail the product's parser (claude plugin validate, 2.1.268 and 2.1.270: "YAML frontmatter failed to parse"). Asserting exit 0 here pins the validator to accepting files the runtime drops.

check "own agent $(basename "$agent")" 0 "$agent"
done

# A valid agent whose fields only trigger warnings must still exit 0,
# and must reach the summary line instead of aborting at the first warning.
cat > "$TMP_DIR/warning-agent.md" <<'EOF'
---
name: warning-agent
description: A valid description that has no example blocks and no trigger phrase
model: sonnet
color: blue
---

You are a test agent. Your job is to exist so the validator has something to warn about.
EOF
check "valid agent with warnings" 0 "$TMP_DIR/warning-agent.md"
if ! grep -q "Validation passed" "$TMP_DIR/out.txt"; then

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Exit code plus this line is not enough to see the description fix regress: with the single-line extraction restored, 5/5 still pass while the false <example> warning fires. A fixture with a multi-line description asserting that warning is absent would close it.

echo "FAIL: summary line missing (script aborted before finishing)"
failures=$((failures + 1))
fi

# An invalid agent (bad name, missing color) must still fail with exit 1.
cat > "$TMP_DIR/invalid-agent.md" <<'EOF'
---
name: x
description: A valid description for an otherwise invalid agent file
model: sonnet
---

You are a test agent with an invalid name and no color field.
EOF
check "invalid agent" 1 "$TMP_DIR/invalid-agent.md"

echo ""
if [ "$failures" -gt 0 ]; then
echo "$failures test(s) failed"
exit 1
fi
echo "All tests passed"