wandb List Experiments
This is a human/operator audit guide, not a live advisor or student skill. Run the script below to fetch all experiment PRs from the advisor branch and organize them into 3 files.
The BASE_BRANCH should be set to the advisor branch (e.g. noam). Check the $ADVISOR_BRANCH env var or the PR base branch.
Code
import subprocess, json, os, re
from datetime import datetime
BASE_BRANCH = os.environ.get("ADVISOR_BRANCH", "noam")
def get_repo_info():
r = subprocess.run(["gh", "repo", "view", "--json", "owner,name"], capture_output=True, text=True)
if r.returncode:
raise RuntimeError(r.stderr)
info = json.loads(r.stdout)
return info["owner"]["login"], info["name"]
def fetch_issue_comments(owner, name, number):
"""Fetch all PR conversation comments for a PR, across every REST page."""
comments = []
page = 1
while True:
endpoint = f"repos/{owner}/{name}/issues/{number}/comments?per_page=100&page={page}"
r = subprocess.run(["gh", "api", endpoint], capture_output=True, text=True)
if r.returncode:
raise RuntimeError(r.stderr)
batch = json.loads(r.stdout)
comments.extend((comment.get("body") or "") for comment in batch)
if len(batch) < 100:
return comments
page += 1
def fetch():
"""Fetch PRs with complete comments via GraphQL plus REST overflow pages."""
owner, name = get_repo_info()
prs = []
cursor = None
while True:
after = f', after: "{cursor}"' if cursor else ""
query = f'''{{
repository(owner: "{owner}", name: "{name}") {{
pullRequests(first: 100, baseRefName: "{BASE_BRANCH}", states: [OPEN, CLOSED, MERGED]{after}) {{
pageInfo {{ hasNextPage endCursor }}
nodes {{
number title body state mergedAt url
labels(first: 10) {{ nodes {{ name }} }}
comments(first: 100) {{
pageInfo {{ hasNextPage }}
nodes {{ body }}
}}
}}
}}
}}
}}'''
r = subprocess.run(["gh", "api", "graphql", "-f", f"query={query}"], capture_output=True, text=True)
if r.returncode:
raise RuntimeError(r.stderr)
data = json.loads(r.stdout)["data"]["repository"]["pullRequests"]
for node in data["nodes"]:
first_comments = [c["body"] or "" for c in node["comments"]["nodes"]]
comments = (
fetch_issue_comments(owner, name, node["number"])
if node["comments"]["pageInfo"]["hasNextPage"]
else first_comments
)
prs.append({
"number": node["number"],
"title": node["title"],
"body": node["body"] or "",
"state": node["state"],
"mergedAt": node["mergedAt"],
"url": node["url"],
"labels": [{"name": l["name"]} for l in node["labels"]["nodes"]],
"comments": comments,
})
if not data["pageInfo"]["hasNextPage"]:
break
cursor = data["pageInfo"]["endCursor"]
return prs
def extract_section(text, header):
"""Extract a markdown section (# or ##) by header name."""
match = re.search(rf'(^#{{1,2}} {header}.*?)(?=^#{{1,2}} |\Z)', text, re.MULTILINE | re.DOTALL)
return match.group(1).strip() if match else None
def find_results_text(pr):
"""Search body then comments (newest first) for the Results section."""
body = pr.get("body", "") or ""
section = extract_section(body, "Results")
if section and "_To be filled" not in section:
return section
for comment in reversed(pr.get("comments", [])):
section = extract_section(comment, "Results")
if section and "_To be filled" not in section:
return section
return None
def extract_val_loss(text):
if not text:
return None
patterns = [
r'val[/_]loss[:\s]*\*?\*?(\d+\.\d+)',
r'\*\*val/loss\*\*[:\s|]*\*?\*?(\d+\.\d+)',
r'val/loss[:\s|]+(\d+\.\d+)',
]
for p in patterns:
m = re.search(p, text, re.IGNORECASE)
if m:
return float(m.group(1))
return None
def has_results(pr):
return find_results_text(pr) is not None
def labels_str(pr):
return ", ".join(l["name"] for l in pr.get("labels", []))
def hypothesis_oneline(body):
"""Extract first sentence of hypothesis."""
sec = extract_section(body or "", "Hypothesis")
if not sec:
return ""
text = re.sub(r'^#{1,2} Hypothesis\n', '', sec).strip()
m = re.match(r'(.+?[.!])\s', text)
return m.group(1) if m else text[:120]
# --- Fetch ---
prs = fetch()
merged = sorted([p for p in prs if p.get("mergedAt")], key=lambda p: p["mergedAt"])
with_results = [p for p in prs if has_results(p) and not p.get("mergedAt")]
no_results = [p for p in prs if not has_results(p)]
ts = datetime.now().strftime("%Y-%m-%d_%H-%M")
outdir = "experiment_log"
os.makedirs(outdir, exist_ok=True)
# --- File 1: Merged winners (chronological) ---
merged_path = f"{outdir}/merged_{ts}.md"
with open(merged_path, "w") as f:
f.write(f"# Merged Experiments ({len(merged)} winners)\n\n")
f.write("These PRs beat baseline and were merged, in chronological order.\n\n")
for pr in merged:
body = pr.get("body", "") or ""
hyp = extract_section(body, "Hypothesis") or ""
results = find_results_text(pr) or ""
val = extract_val_loss(results)
val_str = f" | val_loss: {val}" if val else ""
f.write(f"## #{pr['number']}: {pr['title']}{val_str}\n\n")
if hyp:
f.write(f"{hyp}\n\n")
if results:
f.write(f"{results}\n\n")
f.write("---\n\n")
# --- File 2: Compact results table ---
table_path = f"{outdir}/results_table_{ts}.md"
with open(table_path, "w") as f:
f.write(f"# Experiment Results Table\n\n")
f.write(f"Total: {len(prs)} PRs | Merged: {len(merged)} | ")
f.write(f"Ran (not merged): {len(with_results)} | Never ran: {len(no_results)}\n\n")
f.write("## Merged\n\n")
f.write("| PR | Title | val_loss |\n|---|---|---|\n")
for pr in merged:
results = find_results_text(pr)
val = extract_val_loss(results)
val_str = f"{val:.4f}" if val else "—"
f.write(f"| #{pr['number']} | {pr['title']} | {val_str} |\n")
f.write(f"\n## Ran but not merged ({len(with_results)})\n\n")
f.write("| PR | Title | val_loss | Hypothesis (short) |\n|---|---|---|---|\n")
for pr in sorted(with_results, key=lambda p: p["number"], reverse=True):
body = pr.get("body", "") or ""
results = find_results_text(pr)
val = extract_val_loss(results)
val_str = f"{val:.4f}" if val else "—"
hyp = hypothesis_oneline(body)
f.write(f"| #{pr['number']} | {pr['title']} | {val_str} | {hyp[:80]} |\n")
f.write(f"\n## Never ran ({len(no_results)})\n\n")
f.write("| PR | Title | Hypothesis (short) |\n|---|---|---|\n")
for pr in sorted(no_results, key=lambda p: p["number"], reverse=True):
hyp = hypothesis_oneline(pr.get("body", ""))
f.write(f"| #{pr['number']} | {pr['title']} | {hyp[:80]} |\n")
# --- File 3: Full details (only PRs that ran) ---
full_path = f"{outdir}/full_details_{ts}.md"
with open(full_path, "w") as f:
f.write(f"# Full Experiment Details ({len(merged) + len(with_results)} experiments)\n\n")
for pr in merged + sorted(with_results, key=lambda p: p["number"], reverse=True):
body = pr.get("body", "").strip() or "_No description._"
state = "MERGED" if pr.get("mergedAt") else "CLOSED"
f.write(f"# #{pr['number']}: {pr['title']} [{state}]\n\n{body}\n\n")
results = find_results_text(pr)
if results and results not in body:
f.write(f"{results}\n\n")
f.write("---\n\n")
print(f"Saved {len(prs)} PRs ({len(merged)} merged, {len(with_results)} ran, {len(no_results)} never ran):")
print(f" {merged_path} — merged winners with hypothesis + results")
print(f" {table_path} — compact table of all experiments")
print(f" {full_path} — full PR bodies (only experiments that ran)")