final_summary.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408
  1. #!/usr/bin/env python3
  2. from __future__ import annotations
  3. import json
  4. import os
  5. import glob
  6. import pathlib
  7. import urllib.error
  8. import urllib.parse
  9. import urllib.request
  10. import datetime
  11. from typing import Any, Dict, Iterable, List, Tuple
  12. # ----------------------- Constants -----------------------
  13. ICON_WIN = "https://raw.githubusercontent.com/EgoistDeveloper/operating-system-logos/master/src/32x32/WIN.png"
  14. ICON_MAC = "https://raw.githubusercontent.com/EgoistDeveloper/operating-system-logos/master/src/32x32/MAC.png"
  15. ICON_IOS = "https://raw.githubusercontent.com/EgoistDeveloper/operating-system-logos/master/src/32x32/IOS.png"
  16. ICON_AND = "https://raw.githubusercontent.com/EgoistDeveloper/operating-system-logos/master/src/32x32/AND.png"
  17. ICON_CPP = "https://raw.githubusercontent.com/isocpp/logos/master/cpp_logo.png"
  18. ICON_PM = "https://avatars.githubusercontent.com/u/96267164?s=32"
  19. ALIGN_4_COLS = "|:--|:--:|:--:|:--:|\n" # reused in Validation/Tests/Build matrix sections
  20. FAMILIES = ("windows-msvc", "windows-mingw", "macos", "ios", "android")
  21. VALIDATION_ORDER = {"LF line endings": 0, "JSON": 1, "Markdown": 2}
  22. TESTS_ORDER = {"Clang Latest": 0, "GCC Latest": 1, "Clang Oldest": 2, "GCC Oldest": 3}
  23. # ----------------------- Helpers -----------------------
  24. def env(name: str, default: str = "") -> str:
  25. v = os.getenv(name)
  26. return v if v is not None else default
  27. def now_utc() -> datetime.datetime:
  28. # Sonar: avoid utcnow(); use tz-aware now()
  29. return datetime.datetime.now(datetime.timezone.utc)
  30. def hms_from_ms(ms: int) -> str:
  31. s = max(0, round(ms / 1000))
  32. hh = f"{s // 3600:02d}"
  33. mm = f"{(s % 3600) // 60:02d}"
  34. ss = f"{s % 60:02d}"
  35. return f"{hh}:{mm}:{ss}"
  36. def parse_iso8601(s: str | None) -> datetime.datetime | None:
  37. if not s:
  38. return None
  39. try:
  40. return datetime.datetime.fromisoformat(s.replace("Z", "+00:00"))
  41. except Exception:
  42. return None
  43. def gh_api(path: str, method: str = "GET", data: Any = None, token: str | None = None,
  44. params: Dict[str, Any] | None = None) -> Any:
  45. base = "https://api.github.com"
  46. url = f"{base}{path}"
  47. if params:
  48. url = f"{url}?{urllib.parse.urlencode(params)}"
  49. headers = {
  50. "Accept": "application/vnd.github+json",
  51. "User-Agent": "final-summary-script",
  52. }
  53. if token:
  54. headers["Authorization"] = f"Bearer {token}"
  55. req = urllib.request.Request(url, method=method, headers=headers)
  56. payload = None
  57. if data is not None:
  58. payload = json.dumps(data).encode("utf-8")
  59. req.add_header("Content-Type", "application/json")
  60. try:
  61. with urllib.request.urlopen(req, payload, timeout=60) as r:
  62. raw = r.read()
  63. return {} if not raw else json.loads(raw.decode("utf-8"))
  64. except urllib.error.HTTPError as e:
  65. msg = e.read().decode("utf-8", errors="replace")
  66. print(f"[WARN] GitHub API {method} {url} -> {e.code}: {msg}")
  67. raise
  68. except Exception as e:
  69. print(f"[WARN] GitHub API {method} {url} -> {e}")
  70. raise
  71. def read_json_file(p: str) -> Any:
  72. try:
  73. with open(p, "r", encoding="utf-8") as f:
  74. return json.load(f)
  75. except FileNotFoundError:
  76. return None
  77. except Exception as e:
  78. print(f"[WARN] Cannot read JSON '{p}': {e}")
  79. return None
  80. def write_json_file(p: str, data: Any) -> None:
  81. pathlib.Path(p).parent.mkdir(parents=True, exist_ok=True)
  82. with open(p, "w", encoding="utf-8") as f:
  83. json.dump(data, f, ensure_ascii=False, indent=2)
  84. def append_summary(md: str) -> None:
  85. summary_path = env("GITHUB_STEP_SUMMARY") or "SUMMARY.md"
  86. with open(summary_path, "a", encoding="utf-8") as f:
  87. f.write(md)
  88. def status_icon(status: str) -> str:
  89. return {
  90. "success": "✅",
  91. "failure": "❌",
  92. "cancelled": "🚫",
  93. "timed_out": "⌛",
  94. "skipped": "⏭",
  95. "neutral": "⚠️",
  96. "action_required": "⚠️",
  97. }.get(status, "❓")
  98. def family_title_and_icon(fam: str) -> Tuple[str, str]:
  99. match fam:
  100. case "windows-msvc": return "Windows (MSVC)", ICON_WIN
  101. case "windows-mingw": return "Windows (MinGW)", ICON_WIN
  102. case "macos": return "macOS", ICON_MAC
  103. case "ios": return "iOS", ICON_IOS
  104. case "android": return "Android", ICON_AND
  105. case _: return fam, ICON_PM
  106. # ----------------------- 1) Collect validation & tests -----------------------
  107. def _job_duration(job: Dict[str, Any]) -> str:
  108. st = parse_iso8601(job.get("started_at"))
  109. en = parse_iso8601(job.get("completed_at"))
  110. if st and en:
  111. return hms_from_ms(int((en - st).total_seconds() * 1000))
  112. return ""
  113. def _test_pretty_name(name: str) -> str:
  114. pretty = name.replace("Test (", "").removesuffix(")")
  115. low = pretty.lower()
  116. if "gcc-latest" in low: return "GCC Latest"
  117. if "gcc-oldest" in low: return "GCC Oldest"
  118. if "clang-latest" in low: return "Clang Latest"
  119. if "clang-oldest" in low: return "Clang Oldest"
  120. return pretty
  121. def _rows_for_job(j: Dict[str, Any]) -> List[Dict[str, Any]]:
  122. rows: List[Dict[str, Any]] = []
  123. dur = _job_duration(j)
  124. name = j.get("name") or ""
  125. # Build matrix
  126. if name.startswith("Build "):
  127. pretty = name.replace("Build (", "").removesuffix(")")
  128. rows.append({
  129. "group": "builds",
  130. "name": pretty,
  131. "status": j.get("conclusion") or "neutral",
  132. "duration": dur,
  133. "url": j.get("html_url"),
  134. })
  135. # Code validation
  136. if name == "Validate Code":
  137. mapping = {
  138. "Validate JSON": "JSON",
  139. "Validate Markdown": "Markdown",
  140. "Ensure LF line endings": "LF line endings",
  141. }
  142. for st in (j.get("steps") or []):
  143. stname = st.get("name")
  144. if stname in mapping:
  145. rows.append({
  146. "group": "validation",
  147. "name": mapping[stname],
  148. "status": st.get("conclusion") or "skipped",
  149. "duration": dur,
  150. "url": j.get("html_url"),
  151. })
  152. # Tests matrix
  153. if name.startswith("Test "):
  154. pretty = _test_pretty_name(name)
  155. steps = j.get("steps") or []
  156. test_step = next((s for s in steps if s.get("name") == "Test"), None)
  157. status = (test_step.get("conclusion") if test_step else None) or j.get("conclusion") or "neutral"
  158. rows.append({
  159. "group": "tests",
  160. "name": pretty,
  161. "status": status,
  162. "duration": dur,
  163. "url": j.get("html_url"),
  164. })
  165. return rows
  166. def collect_validation_and_tests() -> None:
  167. token = env("GITHUB_TOKEN")
  168. repo_full = env("GITHUB_REPOSITORY") # "owner/repo"
  169. run_id = env("GITHUB_RUN_ID")
  170. if not (token and repo_full and run_id):
  171. print("[INFO] Missing GITHUB_TOKEN / GITHUB_REPOSITORY / GITHUB_RUN_ID; skipping GH API collect.")
  172. return
  173. owner, repo = repo_full.split("/", 1)
  174. r = gh_api(f"/repos/{owner}/{repo}/actions/runs/{run_id}/jobs",
  175. method="GET", token=token, params={"per_page": 100})
  176. jobs = r.get("jobs") or []
  177. rows: List[Dict[str, Any]] = []
  178. for j in jobs:
  179. rows.extend(_rows_for_job(j))
  180. pathlib.Path("partials").mkdir(parents=True, exist_ok=True)
  181. write_json_file("partials/validation.json", rows)
  182. # ----------------------- 2) Compose Summary -----------------------
  183. def _load_primary_items() -> List[Dict[str, Any]]:
  184. files = [p for p in glob.glob("partials/**/*.json", recursive=True)
  185. if not any(p.endswith(x) for x in ("validation.json", "source.json"))
  186. and not pathlib.Path(p).name.startswith("installer-")]
  187. items: List[Dict[str, Any]] = []
  188. for p in files:
  189. data = read_json_file(p)
  190. if isinstance(data, dict) and "family" in data:
  191. items.append(data)
  192. return items
  193. def _load_installer_map() -> Dict[str, str]:
  194. inst_map: Dict[str, str] = {}
  195. for p in glob.glob("partials/installer-*.json"):
  196. obj = read_json_file(p) or {}
  197. plat, url = obj.get("platform"), obj.get("installer_url")
  198. if plat and url:
  199. inst_map[plat] = url
  200. return inst_map
  201. def _durations_map(val_rows: List[Dict[str, Any]]) -> Dict[str, str]:
  202. return {r["name"]: r.get("duration", "-") for r in val_rows if r.get("group") == "builds"}
  203. def _render_family_table(fam: str, rows: List[Dict[str, Any]],
  204. inst_map: Dict[str, str], dur_map: Dict[str, str]) -> None:
  205. title, icon = family_title_and_icon(fam)
  206. append_summary(f'### <img src="{icon}" width="22"/> {title}\n\n')
  207. cols_arch = ["| Architecture |"]
  208. cols_stats = ["| Cache statistic |"]
  209. cols_time = ["| Build time |"]
  210. cols_down = ["| Download |"]
  211. for it in rows:
  212. plat = it.get("platform", "")
  213. arch = it.get("arch", "")
  214. hits = it.get("hits", "")
  215. total = it.get("total", "")
  216. rate = it.get("rate", "")
  217. hms = dur_map.get(plat, "-")
  218. main = it.get("artifact_url") or ""
  219. dbg = it.get("debug_symbols_url") or ""
  220. aab = it.get("aab_url") or ""
  221. inst = inst_map.get(plat, "")
  222. cols_arch.append(f" {arch} |")
  223. cols_stats.append(f" {rate} ({hits} / {total}) |")
  224. cols_time.append(f" {hms} |")
  225. dl_parts = []
  226. if inst: dl_parts.append(f"[Installer]({inst})")
  227. if dbg: dl_parts.append(f"[Debug symbols]({dbg})")
  228. if main: dl_parts.append(f"[Archive]({main})")
  229. if aab: dl_parts.append(f"[AAB]({aab})")
  230. dl = "<br/>".join(dl_parts) if dl_parts else "—"
  231. cols_down.append(f" {dl} |")
  232. count = len(rows)
  233. align = "|:--|" + ":--:|" * count
  234. append_summary("".join(cols_arch) + "\n")
  235. append_summary(align + "\n")
  236. append_summary("".join(cols_stats) + "\n")
  237. append_summary("".join(cols_time) + "\n")
  238. append_summary("".join(cols_down) + "\n\n")
  239. def _render_validation_section(val_rows: List[Dict[str, Any]]) -> None:
  240. rows = sorted((r for r in val_rows if r.get("group") == "validation"),
  241. key=lambda r: (VALIDATION_ORDER.get(r.get("name"), 999), r.get("name", "")))
  242. if not rows:
  243. return
  244. append_summary("### 🔍 Validation\n")
  245. append_summary("| Check | Status | Time | Logs |\n")
  246. append_summary(ALIGN_4_COLS)
  247. for r in rows:
  248. icon = status_icon(r.get("status", ""))
  249. dur = r.get("duration") or "-"
  250. url = r.get("url") or ""
  251. logs = f"[Logs]({url})" if url else "—"
  252. append_summary(f"| {r.get('name','')} | {icon} | {dur} | {logs} |\n")
  253. append_summary("\n")
  254. def _render_tests_section(val_rows: List[Dict[str, Any]]) -> None:
  255. rows = sorted((r for r in val_rows if r.get("group") == "tests"),
  256. key=lambda r: (TESTS_ORDER.get(r.get("name"), 999), r.get("name", "")))
  257. if not rows:
  258. return
  259. append_summary("### 🧪 Tests\n")
  260. append_summary("| Matrix | Status | Time | Logs |\n")
  261. append_summary(ALIGN_4_COLS)
  262. for r in rows:
  263. icon = status_icon(r.get("status", ""))
  264. dur = r.get("duration") or "-"
  265. url = r.get("url") or ""
  266. logs = f"[Logs]({url})" if url else "—"
  267. append_summary(f"| {r.get('name','')} | {icon} | {dur} | {logs} |\n")
  268. append_summary("\n")
  269. def _render_build_matrix_section(val_rows: List[Dict[str, Any]]) -> None:
  270. rows = sorted((r for r in val_rows if r.get("group") == "builds"),
  271. key=lambda r: r.get("name", ""))
  272. if not rows:
  273. return
  274. append_summary("### 🚦 Build matrix\n")
  275. append_summary("| Platform | Status | Time | Logs |\n")
  276. append_summary(ALIGN_4_COLS)
  277. for r in rows:
  278. icon = status_icon(r.get("status", ""))
  279. dur = r.get("duration") or "-"
  280. url = r.get("url") or ""
  281. logs = f"[Logs]({url})" if url else "—"
  282. append_summary(f"| {r.get('name','')} | {icon} | {dur} | {logs} |\n")
  283. append_summary("\n")
  284. def compose_summary() -> None:
  285. # Source code section
  286. src_json = read_json_file("partials/source.json")
  287. if src_json and src_json.get("source_url"):
  288. append_summary("\n\n")
  289. append_summary(
  290. f'### <img src="{ICON_CPP}" width="20"/> Source code - [Download]({src_json["source_url"]})\n\n\n'
  291. )
  292. items = _load_primary_items()
  293. inst_map = _load_installer_map()
  294. val_rows: List[Dict[str, Any]] = read_json_file("partials/validation.json") or []
  295. dur_map = _durations_map(val_rows)
  296. # Family tables
  297. for fam in FAMILIES:
  298. fam_rows = [x for x in items if x.get("family") == fam]
  299. if fam_rows:
  300. _render_family_table(fam, fam_rows, inst_map, dur_map)
  301. # Validation, Tests, Build matrix
  302. _render_validation_section(val_rows)
  303. _render_tests_section(val_rows)
  304. _render_build_matrix_section(val_rows)
  305. # ----------------------- 3) Delete partial artifacts -----------------------
  306. def delete_partial_artifacts() -> None:
  307. token = env("GITHUB_TOKEN")
  308. repo_full = env("GITHUB_REPOSITORY")
  309. run_id = env("GITHUB_RUN_ID")
  310. if not (token and repo_full and run_id):
  311. print("[INFO] Missing env for deleting artifacts; skipping.")
  312. return
  313. owner, repo = repo_full.split("/", 1)
  314. r = gh_api(f"/repos/{owner}/{repo}/actions/runs/{run_id}/artifacts",
  315. token=token, params={"per_page": 100})
  316. arts = r.get("artifacts") or []
  317. for a in arts:
  318. name = a.get("name", "")
  319. if name.startswith("partial-json-"):
  320. aid = a.get("id")
  321. print(f"Deleting artifact {name} (id={aid})")
  322. gh_api(f"/repos/{owner}/{repo}/actions/artifacts/{aid}",
  323. method="DELETE", token=token)
  324. # ----------------------- Main -----------------------
  325. def main() -> None:
  326. pathlib.Path("partials").mkdir(parents=True, exist_ok=True)
  327. try:
  328. collect_validation_and_tests()
  329. except Exception as e:
  330. print(f"[WARN] collect_validation_and_tests failed: {e}")
  331. try:
  332. compose_summary()
  333. except Exception as e:
  334. print(f"[WARN] compose_summary failed: {e}")
  335. try:
  336. delete_partial_artifacts()
  337. except Exception as e:
  338. print(f"[WARN] delete_partial_artifacts failed: {e}")
  339. if __name__ == "__main__":
  340. main()