name: Test Structured Outputs on: pull_request: workflow_dispatch: workflow_call: permissions: contents: read jobs: test-basic-types: name: Test Basic Type Conversions runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Test with explicit values id: test uses: ./base-action with: prompt: | Run this command: echo "test" Then return EXACTLY these values: - text_field: "hello" - number_field: 42 - boolean_true: true - boolean_false: false anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} claude_args: | --allowedTools Bash --json-schema '{"type":"object","properties":{"text_field":{"type":"string"},"number_field":{"type":"number"},"boolean_true":{"type":"boolean"},"boolean_false":{"type":"boolean"}},"required":["text_field","number_field","boolean_true","boolean_false"]}' - name: Verify outputs run: | # Parse the structured_output JSON OUTPUT='${{ steps.test.outputs.structured_output }}' # Test string pass-through TEXT_FIELD=$(echo "$OUTPUT" | jq -r '.text_field') if [ "$TEXT_FIELD" != "hello" ]; then echo "❌ String: expected 'hello', got '$TEXT_FIELD'" exit 1 fi # Test number → string conversion NUMBER_FIELD=$(echo "$OUTPUT" | jq -r '.number_field') if [ "$NUMBER_FIELD" != "42" ]; then echo "❌ Number: expected '42', got '$NUMBER_FIELD'" exit 1 fi # Test boolean → "true" conversion BOOLEAN_TRUE=$(echo "$OUTPUT" | jq -r '.boolean_true') if [ "$BOOLEAN_TRUE" != "true" ]; then echo "❌ Boolean true: expected 'true', got '$BOOLEAN_TRUE'" exit 1 fi # Test boolean → "false" conversion BOOLEAN_FALSE=$(echo "$OUTPUT" | jq -r '.boolean_false') if [ "$BOOLEAN_FALSE" != "false" ]; then echo "❌ Boolean false: expected 'false', got '$BOOLEAN_FALSE'" exit 1 fi echo "✅ All basic type conversions correct" test-complex-types: name: Test Arrays and Objects runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Test complex types id: test uses: ./base-action with: prompt: | Run: echo "ready" Return EXACTLY: - items: ["apple", "banana", "cherry"] - config: {"key": "value", "count": 3} - empty_array: [] anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} claude_args: | --allowedTools Bash --json-schema '{"type":"object","properties":{"items":{"type":"array","items":{"type":"string"}},"config":{"type":"object"},"empty_array":{"type":"array"}},"required":["items","config","empty_array"]}' - name: Verify JSON stringification run: | # Parse the structured_output JSON OUTPUT='${{ steps.test.outputs.structured_output }}' # Arrays should be JSON stringified if ! echo "$OUTPUT" | jq -e '.items | length == 3' > /dev/null; then echo "❌ Array not properly formatted" echo "$OUTPUT" | jq '.items' exit 1 fi # Objects should be JSON stringified if ! echo "$OUTPUT" | jq -e '.config.key == "value"' > /dev/null; then echo "❌ Object not properly formatted" echo "$OUTPUT" | jq '.config' exit 1 fi # Empty arrays should work if ! echo "$OUTPUT" | jq -e '.empty_array | length == 0' > /dev/null; then echo "❌ Empty array not properly formatted" echo "$OUTPUT" | jq '.empty_array' exit 1 fi echo "✅ All complex types handled correctly" test-edge-cases: name: Test Edge Cases runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Test edge cases id: test uses: ./base-action with: prompt: | Run: echo "test" Return EXACTLY: - zero: 0 - empty_string: "" - negative: -5 - decimal: 3.14 anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} claude_args: | --allowedTools Bash --json-schema '{"type":"object","properties":{"zero":{"type":"number"},"empty_string":{"type":"string"},"negative":{"type":"number"},"decimal":{"type":"number"}},"required":["zero","empty_string","negative","decimal"]}' - name: Verify edge cases run: | # Parse the structured_output JSON OUTPUT='${{ steps.test.outputs.structured_output }}' # Zero should be "0", not empty or falsy ZERO=$(echo "$OUTPUT" | jq -r '.zero') if [ "$ZERO" != "0" ]; then echo "❌ Zero: expected '0', got '$ZERO'" exit 1 fi # Empty string should be empty (not "null" or missing) EMPTY_STRING=$(echo "$OUTPUT" | jq -r '.empty_string') if [ "$EMPTY_STRING" != "" ]; then echo "❌ Empty string: expected '', got '$EMPTY_STRING'" exit 1 fi # Negative numbers should work NEGATIVE=$(echo "$OUTPUT" | jq -r '.negative') if [ "$NEGATIVE" != "-5" ]; then echo "❌ Negative: expected '-5', got '$NEGATIVE'" exit 1 fi # Decimals should preserve precision DECIMAL=$(echo "$OUTPUT" | jq -r '.decimal') if [ "$DECIMAL" != "3.14" ]; then echo "❌ Decimal: expected '3.14', got '$DECIMAL'" exit 1 fi echo "✅ All edge cases handled correctly" test-name-sanitization: name: Test Output Name Sanitization runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Test special characters in field names id: test uses: ./base-action with: prompt: | Run: echo "test" Return EXACTLY: {test-result: "passed", item_count: 10} anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} claude_args: | --allowedTools Bash --json-schema '{"type":"object","properties":{"test-result":{"type":"string"},"item_count":{"type":"number"}},"required":["test-result","item_count"]}' - name: Verify sanitized names work run: | # Parse the structured_output JSON OUTPUT='${{ steps.test.outputs.structured_output }}' # Hyphens should be preserved in the JSON TEST_RESULT=$(echo "$OUTPUT" | jq -r '.["test-result"]') if [ "$TEST_RESULT" != "passed" ]; then echo "❌ Hyphenated name failed: expected 'passed', got '$TEST_RESULT'" exit 1 fi # Underscores should work ITEM_COUNT=$(echo "$OUTPUT" | jq -r '.item_count') if [ "$ITEM_COUNT" != "10" ]; then echo "❌ Underscore name failed: expected '10', got '$ITEM_COUNT'" exit 1 fi echo "✅ Name sanitization works" test-execution-file-structure: name: Test Execution File Format runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Run with structured output id: test uses: ./base-action with: prompt: "Run: echo 'complete'. Return: {done: true}" anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} claude_args: | --allowedTools Bash --json-schema '{"type":"object","properties":{"done":{"type":"boolean"}},"required":["done"]}' - name: Verify execution file contains structured_output run: | FILE="${{ steps.test.outputs.execution_file }}" # Check file exists if [ ! -f "$FILE" ]; then echo "❌ Execution file missing" exit 1 fi # Check for structured_output field if ! jq -e '.[] | select(.type == "result") | .structured_output' "$FILE" > /dev/null; then echo "❌ No structured_output in execution file" cat "$FILE" exit 1 fi # Verify the actual value DONE=$(jq -r '.[] | select(.type == "result") | .structured_output.done' "$FILE") if [ "$DONE" != "true" ]; then echo "❌ Wrong value in execution file" exit 1 fi echo "✅ Execution file format correct" test-summary: name: Summary runs-on: ubuntu-latest needs: - test-basic-types - test-complex-types - test-edge-cases - test-name-sanitization - test-execution-file-structure if: always() steps: - name: Generate Summary run: | echo "# Structured Output Tests (Optimized)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "Fast, deterministic tests using explicit prompts" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "| Test | Result |" >> $GITHUB_STEP_SUMMARY echo "|------|--------|" >> $GITHUB_STEP_SUMMARY echo "| Basic Types | ${{ needs.test-basic-types.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY echo "| Complex Types | ${{ needs.test-complex-types.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY echo "| Edge Cases | ${{ needs.test-edge-cases.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY echo "| Name Sanitization | ${{ needs.test-name-sanitization.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY echo "| Execution File | ${{ needs.test-execution-file-structure.result == 'success' && '✅ PASS' || '❌ FAIL' }} |" >> $GITHUB_STEP_SUMMARY # Check if all passed ALL_PASSED=${{ needs.test-basic-types.result == 'success' && needs.test-complex-types.result == 'success' && needs.test-edge-cases.result == 'success' && needs.test-name-sanitization.result == 'success' && needs.test-execution-file-structure.result == 'success' }} if [ "$ALL_PASSED" = "true" ]; then echo "" >> $GITHUB_STEP_SUMMARY echo "## ✅ All Tests Passed" >> $GITHUB_STEP_SUMMARY else echo "" >> $GITHUB_STEP_SUMMARY echo "## ❌ Some Tests Failed" >> $GITHUB_STEP_SUMMARY exit 1 fi