feat: add AI flaky test analyzer (Extension)

.github/actions/analyze-flaky-tests/action.yml

@@ -0,0 +1,79 @@
+name: Analyze Flaky Tests with AI
+description: 'Analyzes flaky test failures using Claude AI and posts findings as Slack thread replies.'
+
+inputs:
+  github-token:
+    description: 'GitHub token with repo and actions:read access'
+    required: true
+  claude-api-key:
+    description: 'Anthropic API key for Claude'
+    required: true
+  slack-bot-token:
+    description: 'Slack Bot Token with chat:write scope'
+    required: true
+  slack-channel-id:
+    description: 'Slack channel ID to post findings to'
+    required: true
+  slack-thread-ts:
+    description: 'Slack thread timestamp to reply to (from the flaky test report)'
+    required: true
+  failures-json:
+    description: 'JSON array of test failures to analyze (from the flaky test report)'
+    required: true
+  target-owner:
+    description: 'Owner of the repository containing the tests'
+    required: false
+    default: 'MetaMask'
+  target-repo:
+    description: 'Repository containing the tests'
+    required: false
+    default: 'metamask-extension'
+  github-tools-repository:
+    description: 'The GitHub repository containing the GitHub tools.'
+    required: false
+    default: ${{ github.action_repository }}
+  github-tools-ref:
+    description: 'The SHA of the action to use.'
+    required: false
+    default: ${{ github.action_ref }}
+
+runs:
+  using: composite
+  steps:
+    - name: Checkout GitHub tools repository
+      uses: actions/checkout@v6
+      with:
+        repository: ${{ inputs.github-tools-repository }}
+        ref: ${{ inputs.github-tools-ref }}
+        path: ./github-tools
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: ./github-tools/.nvmrc
+        cache-dependency-path: ./github-tools/yarn.lock
+        cache: yarn
+
+    - name: Enable Corepack
+      working-directory: ./github-tools
+      shell: bash
+      run: corepack enable
+
+    - name: Install dependencies
+      working-directory: ./github-tools
+      shell: bash
+      run: yarn --immutable
+
+    - name: Run AI analysis
+      env:
+        GITHUB_TOKEN: ${{ inputs.github-token }}
+        E2E_CLAUDE_API_KEY: ${{ inputs.claude-api-key }}
+        SLACK_BOT_TOKEN: ${{ inputs.slack-bot-token }}
+        SLACK_CHANNEL_ID: ${{ inputs.slack-channel-id }}
+        SLACK_THREAD_TS: ${{ inputs.slack-thread-ts }}
+        TARGET_OWNER: ${{ inputs.target-owner }}
+        TARGET_REPO: ${{ inputs.target-repo }}
+        FAILURES_JSON: ${{ inputs.failures-json }}
+      working-directory: ./github-tools
+      shell: bash
+      run: yarn ts-node --swc .github/scripts/analyze-flaky-tests/index.ts

.github/actions/flaky-test-report/action.yml

@@ -12,8 +12,17 @@ inputs:
     description: 'GitHub token with repo and actions:read access'
     required: true
   slack-webhook-flaky-tests:
-    description: 'Slack webhook URL for flaky test reports'
-    required: true
+    description: 'Slack webhook URL for flaky test reports (legacy fallback)'
+    required: false
+    default: ''
+  slack-bot-token:
+    description: 'Slack Bot Token with chat:write scope (preferred over webhook for thread support)'
+    required: false
+    default: ''
+  slack-channel-id:
+    description: 'Slack channel ID to post the report to (required when using slack-bot-token)'
+    required: false
+    default: ''
   github-tools-repository:
     description: 'The GitHub repository containing the GitHub tools. Defaults to the GitHub tools action repositor, and usually does not need to be changed.'
     required: false
@@ -23,6 +32,17 @@ inputs:
     required: false
     default: ${{ github.action_ref }}
 
+outputs:
+  thread_ts:
+    description: 'Slack thread timestamp of the posted report (only available when using slack-bot-token)'
+    value: ${{ steps.report.outputs.thread_ts }}
+  has_failures:
+    description: 'Whether any test failures were found (true/false)'
+    value: ${{ steps.report.outputs.has_failures }}
+  failures_json:
+    description: 'JSON array of the top 10 test failures for downstream analysis'
+    value: ${{ steps.report.outputs.failures_json }}
+
 runs:
   using: composite
   steps:
@@ -51,11 +71,14 @@ runs:
       run: yarn --immutable
 
     - name: Run flaky test report script
+      id: report
       env:
         REPOSITORY: ${{ inputs.repository }}
         WORKFLOW_ID: ${{ inputs.workflow-id }}
         GITHUB_TOKEN: ${{ inputs.github-token }}
         SLACK_WEBHOOK_FLAKY_TESTS: ${{ inputs.slack-webhook-flaky-tests }}
+        SLACK_BOT_TOKEN: ${{ inputs.slack-bot-token }}
+        SLACK_CHANNEL_ID: ${{ inputs.slack-channel-id }}
       working-directory: ./github-tools
       shell: bash
       run: node .github/scripts/create-flaky-test-report.mjs