name: Test Structured Outputs (Optimized) # This workflow uses EXPLICIT prompts that tell Claude exactly what to return. # This makes tests fast, deterministic, and focuses on testing OUR code, not Claude's reasoning. # # NOTE: Disabled until Agent SDK structured outputs feature is released # The --json-schema flag is not yet available in public Claude Code releases on: # Disabled - uncomment when feature is released # push: # branches: [main] # pull_request: workflow_dispatch: permissions: contents: read jobs: test-basic-types: name: Test Basic Type Conversions runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Test with explicit values id: test uses: ./base-action with: # EXPLICIT: Tell Claude exactly what to return - no reasoning needed prompt: | Run this command: echo "test" Then return EXACTLY these values: - text_field: "hello" - number_field: 42 - boolean_true: true - boolean_false: false json_schema: | { "type": "object", "properties": { "text_field": {"type": "string"}, "number_field": {"type": "number"}, "boolean_true": {"type": "boolean"}, "boolean_false": {"type": "boolean"} }, "required": ["text_field", "number_field", "boolean_true", "boolean_false"] } anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} allowed_tools: "Bash" - name: Verify outputs run: | # Test string pass-through if [ "${{ steps.test.outputs.text_field }}" != "hello" ]; then echo "❌ String: expected 'hello', got '${{ steps.test.outputs.text_field }}'" exit 1 fi # Test number → string conversion if [ "${{ steps.test.outputs.number_field }}" != "42" ]; then echo "❌ Number: expected '42', got '${{ steps.test.outputs.number_field }}'" exit 1 fi # Test boolean → "true" conversion if [ "${{ steps.test.outputs.boolean_true }}" != "true" ]; then echo "❌ Boolean true: expected 'true', got '${{ steps.test.outputs.boolean_true }}'" exit 1 fi # Test boolean → "false" conversion if [ "${{ steps.test.outputs.boolean_false }}" != "false" ]; then echo "❌ Boolean false: expected 'false', got '${{ steps.test.outputs.boolean_false }}'" exit 1 fi echo "✅ All basic type conversions correct" test-complex-types: name: Test Arrays and Objects runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Test complex types id: test uses: ./base-action with: # EXPLICIT: No file reading, no analysis prompt: | Run: echo "ready" Return EXACTLY: - items: ["apple", "banana", "cherry"] - config: {"key": "value", "count": 3} - empty_array: [] json_schema: | { "type": "object", "properties": { "items": { "type": "array", "items": {"type": "string"} }, "config": {"type": "object"}, "empty_array": {"type": "array"} }, "required": ["items", "config", "empty_array"] } anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} allowed_tools: "Bash" - name: Verify JSON stringification run: | # Arrays should be JSON stringified ITEMS='${{ steps.test.outputs.items }}' if ! echo "$ITEMS" | jq -e '. | length == 3' > /dev/null; then echo "❌ Array not properly stringified: $ITEMS" exit 1 fi # Objects should be JSON stringified CONFIG='${{ steps.test.outputs.config }}' if ! echo "$CONFIG" | jq -e '.key == "value"' > /dev/null; then echo "❌ Object not properly stringified: $CONFIG" exit 1 fi # Empty arrays should work EMPTY='${{ steps.test.outputs.empty_array }}' if ! echo "$EMPTY" | jq -e '. | length == 0' > /dev/null; then echo "❌ Empty array not properly stringified: $EMPTY" exit 1 fi echo "✅ All complex types JSON stringified correctly" test-edge-cases: name: Test Edge Cases runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Test edge cases id: test uses: ./base-action with: prompt: | Run: echo "test" Return EXACTLY: - zero: 0 - empty_string: "" - negative: -5 - decimal: 3.14 json_schema: | { "type": "object", "properties": { "zero": {"type": "number"}, "empty_string": {"type": "string"}, "negative": {"type": "number"}, "decimal": {"type": "number"} }, "required": ["zero", "empty_string", "negative", "decimal"] } anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} allowed_tools: "Bash" - name: Verify edge cases run: | # Zero should be "0", not empty or falsy if [ "${{ steps.test.outputs.zero }}" != "0" ]; then echo "❌ Zero: expected '0', got '${{ steps.test.outputs.zero }}'" exit 1 fi # Empty string should be empty (not "null" or missing) if [ "${{ steps.test.outputs.empty_string }}" != "" ]; then echo "❌ Empty string: expected '', got '${{ steps.test.outputs.empty_string }}'" exit 1 fi # Negative numbers should work if [ "${{ steps.test.outputs.negative }}" != "-5" ]; then echo "❌ Negative: expected '-5', got '${{ steps.test.outputs.negative }}'" exit 1 fi # Decimals should preserve precision if [ "${{ steps.test.outputs.decimal }}" != "3.14" ]; then echo "❌ Decimal: expected '3.14', got '${{ steps.test.outputs.decimal }}'" exit 1 fi echo "✅ All edge cases handled correctly" test-name-sanitization: name: Test Output Name Sanitization runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Test special characters in field names id: test uses: ./base-action with: prompt: | Run: echo "test" Return EXACTLY: {test-result: "passed", item_count: 10} json_schema: | { "type": "object", "properties": { "test-result": {"type": "string"}, "item_count": {"type": "number"} }, "required": ["test-result", "item_count"] } anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} allowed_tools: "Bash" - name: Verify sanitized names work run: | # Hyphens should be preserved (GitHub Actions allows them) if [ "${{ steps.test.outputs.test-result }}" != "passed" ]; then echo "❌ Hyphenated name failed" exit 1 fi # Underscores should work if [ "${{ steps.test.outputs.item_count }}" != "10" ]; then echo "❌ Underscore name failed" exit 1 fi echo "✅ Name sanitization works" test-execution-file-structure: name: Test Execution File Format runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Run with structured output id: test uses: ./base-action with: prompt: "Run: echo 'complete'. Return: {done: true}" json_schema: | { "type": "object", "properties": { "done": {"type": "boolean"} }, "required": ["done"] } anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} allowed_tools: "Bash" - name: Verify execution file contains structured_output run: | FILE="${{ steps.test.outputs.execution_file }}" # Check file exists if [ ! -f "$FILE" ]; then echo "❌ Execution file missing" exit 1 fi # Check for structured_output field if ! jq -e '.[] | select(.type == "result") | .structured_output' "$FILE" > /dev/null; then echo "❌ No structured_output in execution file" cat "$FILE" exit 1 fi # Verify the actual value DONE=$(jq -r '.[] | select(.type == "result") | .structured_output.done' "$FILE") if [ "$DONE" != "true" ]; then echo "❌ Wrong value in execution file" exit 1 fi echo "✅ Execution file format correct" test-summary: name: Summary runs-on: ubuntu-latest needs: - test-basic-types - test-complex-types - test-edge-cases - test-name-sanitization - test-execution-file-structure if: always() steps: - name: Generate Summary run: | echo "# Structured Output Tests (Optimized)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "Fast, deterministic tests using explicit prompts" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "| Test | Result |" >> $GITHUB_STEP_SUMMARY echo "|------|--------|" >> $GITHUB_STEP_SUMMARY echo "| Basic Types | ${{ needs.test-basic-types.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY echo "| Complex Types | ${{ needs.test-complex-types.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY echo "| Edge Cases | ${{ needs.test-edge-cases.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY echo "| Name Sanitization | ${{ needs.test-name-sanitization.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY echo "| Execution File | ${{ needs.test-execution-file-structure.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY # Check if all passed ALL_PASSED=${{ needs.test-basic-types.result == 'success' && needs.test-complex-types.result == 'success' && needs.test-edge-cases.result == 'success' && needs.test-name-sanitization.result == 'success' && needs.test-execution-file-structure.result == 'success' }} if [ "$ALL_PASSED" = "true" ]; then echo "" >> $GITHUB_STEP_SUMMARY echo "## ✅ All Tests Passed" >> $GITHUB_STEP_SUMMARY else echo "" >> $GITHUB_STEP_SUMMARY echo "## ❌ Some Tests Failed" >> $GITHUB_STEP_SUMMARY exit 1 fi