notofonts/scripts/gather.py

from github import Github, Repository
from gftools.utils import download_file
from zipfile import ZipFile
from pathlib import Path
import tempfile
import os
import json
from io import BytesIO
from pybars import Compiler, strlist
import re
import subprocess


TESTING = False
SPECIAL_REPOS = [
    "notofonts.github.io",
    "overview",
    ".github",
    ".allstar",
    "notobuilder",
    "noto-data-dev",
    "noto-docs",
    "noto-project-template",
    "notoglot-mini",
]


def fonts_from_zip(zipfile, dst=None):
    """Unzip fonts. If not dst is given unzip as BytesIO objects"""
    fonts = []
    for filename in zipfile.namelist():
        if filename.endswith(".ttf") or filename.endswith(".otf"):
            if dst:
                target = os.path.join(dst, filename)
                zipfile.extract(filename, dst)
                fonts.append(target)
            else:
                fonts.append(BytesIO(zipfile.read(filename)))
    return fonts


def tree_has_new_files():
    ls = subprocess.run(["git", "ls-files", "--others"], capture_output=True)
    return ls.stdout


print("Fetching existing repos")
g = Github(os.environ["GITHUB_TOKEN"])
org = g.get_organization("notofonts")
org_repos = org.get_repos()
org_names = [r.name for r in org_repos]

subprocess.run(["git", "config", "user.name", "actions-user"])
subprocess.run(["git", "config", "user.email", "[email protected]"])

to_push = []

fontrepos = json.load(open("fontrepos.json"))
if os.path.exists("state.json"):
    state = json.load(open("state.json"))
else:
    state = {}

results = {}

for repo_name in org_names:
    if repo_name in SPECIAL_REPOS:
        continue
    repo = g.get_repo("notofonts/" + repo_name)
    if repo.archived:
        continue
    if repo_name not in fontrepos:
        print("Unknown repo %s; is it missing from fontrepos?" % repo_name)
        continue

    print(f"Gathering data for {repo_name}")

    repo = g.get_repo("notofonts/" + repo_name)
    results[repo_name] = {
        "title": repo.description,
        "tier": fontrepos[repo_name].get("tier", 3),
        "gh_url": "https://notofonts.github.io/" + repo_name,
        "repo_url": "https://www.github.com/notofonts/" + repo_name,
    }

    # Get issues
    results[repo_name]["issues"] = []
    for issue in repo.get_issues():
        results[repo_name]["issues"].append(
            {"title": issue.title, "number": issue.number, "url": issue.html_url}
        )

    if repo_name not in state:
        state[repo_name] = {}

    # Check for new releases
    releases = repo.get_releases()
    for release in sorted(
        releases, key=lambda r: r.published_at.isoformat() if r.published_at else ""
    ):
        m = re.match(r"^(.*)-(v[\d.]+)", release.tag_name)
        if not m:
            print(f"Unparsable release {release.tag_name} in {repo_name}")
            continue
        if release.draft:
            continue
        family, version = m[1], m[2]
        family = re.sub(r"([a-z])([A-Z])", r"\1 \2", family)
        if release.tag_name in state[repo_name].get("known_releases", []):
            continue
        assets = release.get_assets()
        if not assets:
            continue
        latest_asset = assets[0]
        state[repo_name].setdefault("known_releases", []).append(release.tag_name)
        family_thing = (
            state[repo_name].setdefault("families", {}).setdefault(family, {})
        )

        body = release.body
        if not body:
            tag_sha = repo.get_git_ref("tags/" + release.tag_name).object.sha
            try:
                body = repo.get_git_tag(tag_sha).message
            except Exception as e:
                print("Couldn't retrieve release message for %s" % release.tag_name)

        family_thing["latest_release"] = {
            "url": release.html_url,
            "version": version,
            "notes": body,
        }

        if release.published_at:
            family_thing["latest_release"][
                "published"
            ] = release.published_at.isoformat()

        try:
            z = ZipFile(download_file(latest_asset.browser_download_url))
            family_thing["files"] = []
            with tempfile.TemporaryDirectory() as tmpdir:
                fonts = fonts_from_zip(z, tmpdir)
                for font in fonts:
                    newpath = Path("fonts/") / Path(font).relative_to(tmpdir)
                    os.makedirs(newpath.parent, exist_ok=True)
                    family_thing["files"].append(str(newpath))
                    os.rename(font, newpath)
                if tree_has_new_files() and not TESTING:
                    # Add it and tag it
                    subprocess.run(["git", "add", "."])
                    subprocess.run(["git", "commit", "-m", "Add " + release.tag_name])
                    subprocess.run(["git", "tag", release.tag_name])
                    to_push.append(release.tag_name)
        except Exception as e:
            print("Couldn't fetch download for %s" % latest_asset.browser_download_url)

            # Tweet about the new release or something
    results[repo_name]["families"] = state[repo_name].get("families", {})

subprocess.run(["git", "push"])
for tag in to_push:
    subprocess.run(["git", "push", "origin", tag])

# Save state
json.dump(state, open("state.json", "w"), indent=True, sort_keys=True)

for result in results.values():
    for family in result.get("families", {}).values():
        newfiles = {"unhinted": [], "hinted": [], "full": []}
        for file in sorted(family.get("files", [])):
            if "unhinted" in file:
                newfiles["unhinted"].append(file)
            elif "hinted" in file:
                newfiles["hinted"].append(file)
            elif "full" in file:
                newfiles["full"].append(file)
        family["files"] = newfiles

json.dump(results, open("docs/noto.json", "w"), indent=True, sort_keys=True)