最好的 AI 编程 agent 怎么选：在自己的仓库上试一周

#!/usr/bin/env bash
# Score a coding-agent trial from GitHub's own records, one row per issue.
# Usage: ./agent-trial-score.sh OWNER/REPO 'ISSUE SEARCH' [CI_CHECK]
#   ISSUE SEARCH  selects every issue you handed to the agent, e.g.
#                 'label:trial-a' (GitHub issue search syntax)
#   CI_CHECK      exact name of the check run that is your test job, e.g. tests.
#                 Without it the first_ci column stays "unset".
set -euo pipefail

repo="${1:?usage: $0 OWNER/REPO 'ISSUE SEARCH' [CI_CHECK]}"
query="${2:?usage: $0 OWNER/REPO 'ISSUE SEARCH' [CI_CHECK]}"
check="${3:-}"

# Any failed read stops the script: a number built on a failed call is worse than none.
die() { echo "$1: stopping without a score" >&2; exit 2; }

issues=$(gh issue list -R "$repo" --state all --search "$query" --limit 100 \
  --json number,createdAt) || die "could not list issues for $repo"

if [ "$(jq length <<<"$issues")" -eq 0 ]; then
  echo "no issues match '$query' in $repo: nothing to score" >&2
  exit 1
fi
# The list call returns at most 100 issues; a silently cut-off list would skew the score.
[ "$(jq length <<<"$issues")" -lt 100 ] || die "100 or more issues match '$query'; narrow the search"

# Comments or reviews left by human accounts (GitHub user type "User").
humans() {
  local ids
  ids=$(gh api --paginate "$1?per_page=100" -q '.[] | select(.user.type == "User") | .id') || return 1
  grep -c . <<<"$ids" || true
}

rows=$(mktemp)
trap 'rm -f "$rows"' EXIT

while read -r issue; do
  i=$(jq -r .number <<<"$issue")
  notes=$(humans "repos/$repo/issues/$i/comments") || die "could not read comments on issue $i"

  # The last PR in this repository that was set to close the issue, merged or not.
  n=$(gh api graphql -F owner="${repo%/*}" -F name="${repo#*/}" -F number="$i" -f query='
    query($owner: String!, $name: String!, $number: Int!) {
      repository(owner: $owner, name: $name) { issue(number: $number) {
        closedByPullRequestsReferences(first: 20, includeClosedPrs: true) {
          nodes { number repository { nameWithOwner } } } } } }' \
    -q "[.data.repository.issue.closedByPullRequestsReferences.nodes[]
        | select(.repository.nameWithOwner == \"$repo\") | .number] | last // empty") \
    || die "could not read linked PRs of issue $i"
  link=closes
  if [ -z "$n" ]; then
    # No closing link: fall back to the last PR in this repository that mentions the issue.
    mentions=$(gh api --paginate "repos/$repo/issues/$i/timeline?per_page=100" -q ".[]
        | select(.event == \"cross-referenced\" and .source.issue.pull_request != null
                 and .source.issue.repository.full_name == \"$repo\") | .source.issue.number") \
      || die "could not read the timeline of issue $i"
    n=$(tail -n 1 <<<"$mentions")
    link=mention
  fi
  if [ -z "$n" ]; then
    printf '%s\t-\t-\tNO_PR\t-\t-\t-\t%s\t-\n' "$i" "$notes" >>"$rows"
    continue
  fi

  pr=$(gh pr view "$n" -R "$repo" --json state,mergedAt,mergedBy) || die "could not read PR $n"
  result=$(jq -r .state <<<"$pr")
  hours=-
  if [ "$result" = MERGED ]; then
    hours=$(jq -rn --arg a "$(jq -r .createdAt <<<"$issue")" --arg b "$(jq -r .mergedAt <<<"$pr")" \
      '(($b|fromdate) - ($a|fromdate)) / 3600 | . * 10 | round / 10')
  fi

  shas=$(gh api --paginate "repos/$repo/pulls/$n/commits?per_page=100" -q '.[].sha') \
    || die "could not read commits of PR $n"
  commits=$(grep -c . <<<"$shas" || true)
  first_sha=$(head -n 1 <<<"$shas")

  # Your test job on the first commit still on the branch: pass, fail, pending or none.
  # A skipped or missing run is "none", never "pass".
  first_ci=unset
  if [ -n "$check" ] && [ -n "$first_sha" ]; then
    first_ci=$(gh api --paginate "repos/$repo/commits/$first_sha/check-runs?per_page=100" \
      -q ".check_runs[] | select(.name == \"$check\") | .conclusion // \"pending\"" \
      | sort -u | tr '\n' ' ') || die "could not read check runs of PR $n"
    case " $first_ci" in
      *" pending "*) first_ci=pending ;;
      *" failure "*|*" timed_out "*|*" cancelled "*|*" action_required "*|*" startup_failure "*) first_ci=fail ;;
      *" success "*) first_ci=pass ;;
      *) first_ci=none ;;
    esac
  fi

  pr_notes=$(humans "repos/$repo/issues/$n/comments") || die "could not read comments on PR $n"
  reviews=$(humans "repos/$repo/pulls/$n/reviews") || die "could not read reviews on PR $n"
  merged_by=$(jq -r '.mergedBy.login // "-"' <<<"$pr")
  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
    "$i" "$n" "$link" "$result" "$hours" "$first_ci" "$commits" "$((notes + pr_notes + reviews))" "$merged_by" >>"$rows"
done < <(jq -c '.[]' <<<"$issues")

printf 'issue\tpr\tlink\tresult\thours\tfirst_ci\tcommits\thuman_notes\tmerged_by\n'
cat "$rows"

awk -F'\t' -v check="$check" '
  $4 == "MERGED" { m++; if ($6 == "pass") p++; if ($6 == "fail") f++; if ($8 == 0) q++ }
  $4 == "NO_PR"  { none++ }
  $4 == "CLOSED" { c++ }
  $4 == "OPEN"   { o++ }
  END {
    printf "\nissues handed over: %d  merged: %d  no PR: %d  PR closed unmerged: %d  PR still open: %d\n", NR, m, none, c, o
    if (m == 0) { print "no merged PRs: no merge-time numbers"; exit }
    if (check != "") printf "\"%s\" on the first commit of merged PRs: %d pass, %d fail, %d other\n", check, p, f, m - p - f
    printf "merged with no human comment or review: %d of %d\n", q, m
  }' "$rows"
# Median without gawk-only functions: sort first, then pick the middle.
awk -F'\t' '$4 == "MERGED" { print $5 }' "$rows" | sort -n | awk '
  { h[NR] = $1 }
  END {
    if (NR == 0) exit
    med = (NR % 2) ? h[(NR + 1) / 2] : (h[NR / 2] + h[NR / 2 + 1]) / 2
    printf "median hours from issue to merge: %.1f (of %d merged)\n", med, NR
  }'

./agent-trial-score.sh OWNER/REPO 'label:trial-a' tests

gh api repos/OWNER/REPO/commits/COMMIT_SHA/check-runs -q '.check_runs[] | "\(.name)\t\(.conclusion)"'

$ ./agent-trial-score.sh OWNER/trial-copy-a 'label:trial-a' tests
issue	pr	link	result	hours	first_ci	commits	human_notes	merged_by
6	7	mention	MERGED	0	fail	3	0	xqliu
3	-	-	NO_PR	-	-	-	0	-
2	5	closes	CLOSED	-	none	1	1	-
1	4	closes	MERGED	0	none	1	0	xqliu

issues handed over: 4  merged: 2  no PR: 1  PR closed unmerged: 1  PR still open: 0
"tests" on the first commit of merged PRs: 0 pass, 1 fail, 1 other
merged with no human comment or review: 2 of 2
median hours from issue to merge: 0.0 (of 2 merged)
exit=0

$ FAIL_ON='issues/6/comments' PATH=stub:$PATH ./agent-trial-score.sh OWNER/trial-copy-a 'label:trial-a' tests
gh: Resource not accessible (HTTP 403)
could not read comments on issue 6: stopping without a score
exit=2

$ FAIL_ON='check-runs' PATH=stub:$PATH ./agent-trial-score.sh OWNER/trial-copy-a 'label:trial-a' tests
gh: Resource not accessible (HTTP 403)
could not read check runs of PR 7: stopping without a score
exit=2

$ ./agent-trial-score.sh orbi-build/no-such-repo 'label:x'
GraphQL: Could not resolve to a Repository with the name 'orbi-build/no-such-repo'. (repository)
could not list issues for orbi-build/no-such-repo: stopping without a score
exit=2

$ ./agent-trial-score.sh orbi-build/orbi 'label:no-such-label-xyz'
no issues match 'label:no-such-label-xyz' in orbi-build/orbi: nothing to score
exit=1

issue	pr	link	result	hours	first_ci	commits	human_notes	merged_by
…
2164	2166	closes	MERGED	1.4	none	2	0	app/orbi-build
…
2143	2150	closes	MERGED	3.6	fail	5	0	app/orbi-build
…
2141	-	-	NO_PR	-	-	-	0	-
2139	2146	closes	MERGED	2.2	fail	3	1	app/orbi-build
…
2127	2129	closes	MERGED	1	fail	3	0	app/orbi-build
2125	-	-	NO_PR	-	-	-	1	-
2123	2124	closes	MERGED	1	none	2	0	app/orbi-build
2120	-	-	NO_PR	-	-	-	1	-
…

issues handed over: 26  merged: 23  no PR: 3  PR closed unmerged: 0  PR still open: 0
"tests" on the first commit of merged PRs: 18 pass, 3 fail, 2 other
merged with no human comment or review: 19 of 23
median hours from issue to merge: 1.1 (of 23 merged)
exit=0