Added 52 upstream qmk keymaps automatically

[precondition]

I created a list of keymap candidates with this multi-command:

cd ~/qmk_firwmare | ls **/keymaps/**/readme.md | sed "/default\/readme.md/d" | xargs grep -E -c '!\[.*\]\(.*\.(jpe?g|png)( ".*")?\)' | grep "1$"  | sed "s/1$//" > candidate_keymap_folders.txt

Next, I filtered that list to only get keymaps whose folder names are real GitHub usernames which I stored as a List[Tuple[str, str]] in kbs_authors.py

import json
import requests
import subprocess
from typing import Tuple, List
import re
from kbs_authors import kbs_authors # needs to be generated first
import os
from math import floor
from itertools import compress

path_pattern = re.compile("^keyboards/(.*)/keymaps/(.*)/$", flags=re.MULTILINE|re.UNICODE)

with open("candidate_keymap_folders.txt", "r") as f:
    folder_paths: str = f.read()
    kbs_names: List[Tuple[str, str]] = re.findall(path_pattern, folder_paths)

# Run the below code once to filter keymaps whose folder names is a real GitHub username.
# (Some keymaps are named "french" etc. so we can't assume the folder name to always be that of the author)
#
# def is_real_github_username(kb_username) -> bool:
    # keyboard_name, username = kb_username
    # response = requests.get(f"https://github.com/{username}/qmk_firmware/tree/master/keyboards/{keyboard_name}/keymaps/{username}")
    # print(f"{response=}")
    # return bool(response)

# filtered_kbs_names = filter(is_real_github_username, kbs_names)
# with open("kbs_authors.py", "w") as f:
    # f.write("kbs_authors = " + str(list(filtered_kbs_names)))

def get_folder_path(keyboard_name, username) -> str:
    return f"/home/vern/qmk_firmware/keyboards/{keyboard_name}/keymaps/{username}/"

def get_image_link(kb_username) -> str:
    keyboard_name, username = kb_username
    path = get_folder_path(keyboard_name, username)
    md_image_pattern = re.compile(r'!\[.*\]\((.*\.[a-z]+)(?: ".*")?\)')
    with open(path + "readme.md", "r") as readme_file:
        image_link = re.findall(md_image_pattern, readme_file.read())[0]
    return image_link

def get_layout_macro_name(kb_info, keyboard_name, username) -> str:
    # Note: `kb_info["keyboard_folder"]` isn't always the same as `keyboard_name`
    # For example, the keyboard folder for the Atreus is said to be "atreus/astar"
    with open(get_folder_path(keyboard_name, username) + "keymap.c", "r") as f:
        keymapdotc = f.read()
    # Important to go through the longest layout macro names first because there are
    # many layout macro names that start the same 
    # (e.g. LAYOUT_60_ansi, LAYOUT_60_ansi_split_rshift)
    layout_macros = sorted(kb_info["layouts"].keys(), key=len, reverse=True)
    for possible_layout in layout_macros:
        if possible_layout in keymapdotc:
            return possible_layout
    return tuple(kb_info["layouts"].keys())[0]

def get_key_count(kb_info, keyboard_name, username) -> int:
    layout = get_layout_macro_name(kb_info, keyboard_name, username)
    return kb_info["layouts"][layout]["key_count"]

def get_OS(readme: str) -> List[str]:
    windows_pattern = re.compile("Windows", flags=re.IGNORECASE)
    mac_pattern = re.compile("Mac(?:OS)?", flags=re.IGNORECASE)
    linux_pattern = re.compile("(?:GNU[\+/ ])?Linux", flags=re.IGNORECASE)
    os_patterns = [windows_pattern, mac_pattern, linux_pattern]
    os_categories = ["Windows", "MacOS", "GNU+Linux"]
    os_mask = [bool(re.search(pattern, readme)) for pattern in os_patterns]
    return list(compress(os_categories, os_mask))

def get_stagger(kb_info, layout) -> str:
    # Assume ortholinearity, change assumption if decimal x or y values are found.
    stagger_type = "ortholinear"

    # Dactyl, Dactyl Manuform, Tractyl, Pterodactyl, ...
    if "actyl" in kb_info["keyboard_name"]:
        return "depth"

    last_x: int = 0
    last_y: float = 0
    # I hate how frequently QMK uses the word "layout" as a key in the JSON
    for keyboard_key in kb_info["layouts"][layout]["layout"]: 
        x: int = keyboard_key[ "x" ]
        y: float = keyboard_key[ "y" ]
        if not isinstance(x, int):
            stagger_type = "row"
        # if the current keyboard_key is to the left of the last one,
        # that has to mean that it is on another row.
        if floor(x) >= last_x:
            if y != last_y:
                stagger_type = "columnar"
        last_x = x
        last_y = y

    return stagger_type

def has_home_row_mods(keymapdotc) -> bool:
    # Every alpha layout I have ever come across had A *and* S on their home row 
    # with the exception of AZERTY which only has S on its home row.
    mt_a_pattern = re.compile("MT\(\s*MOD_[A-Z]+,\s*[A-Z][A-Z]_A\)|(L|R)?\w{2,3}_T\([A-Z][A-Z]_A\)")
    mt_s_pattern = re.compile("MT\(\s*MOD_[A-Z]+,\s*[A-Z][A-Z]_S\)|(L|R)?\w{2,3}_T\([A-Z][A-Z]_S\)")
    return bool(re.search(mt_a_pattern, keymapdotc)) and bool(re.search(mt_s_pattern, keymapdotc)) 

headers = ["url", "author", "image", "keyCount", "firmware", "keyboard", "stagger", "split", "languages", "summary", "OS", "writeup", "hasLetterOnThumb", "layerCount", "imageDate", "hasOuterPinkyColumns", "hasExtraIndexColumns", "hasVerticalCombos", "hasHomeRowMods"]
def get_keymapdb_entry(kb_username) -> dict:
    print(f"Adding {kb_username=} to the database.")

    # Set up variables
    keyboard_name, username = kb_username
    entry: dict = {}
    rev: str = ""
    if os.path.isdir(f"/home/vern/qmk_firmware/keyboards/{keyboard_name}/rev1/"):
        rev = "/rev1"
    qmk_info_output = subprocess.check_output(["qmk", "info", "-kb", keyboard_name + rev, "-m", "-f", "json"])
    kb_info: dict = json.loads(qmk_info_output)
    layout: str = get_layout_macro_name(kb_info, keyboard_name, username)
    print(f"The layout macro for this board is {layout}.")
    path = get_folder_path(keyboard_name, username)
    with open(path + "readme.md", "r") as readme_file:
        readme: str = readme_file.read()
    with open(path + "keymap.c", "r") as keymapdotc_file:
        keymapdotc: str = keymapdotc_file.read()

    # Populate the db fields
    entry["url"] = f"https://github.com/{username}/qmk_firmware/tree/master/keyboards/{keyboard_name}/keymaps/{username}"
    entry["author"] = username
    entry["image"] = get_image_link(kb_username)
    entry["keyCount"] = get_key_count(kb_info, keyboard_name, username)
    entry["firmware"] = "QMK"
    entry["keyboard"] = kb_info["keyboard_name"]
    entry["stagger"] = get_stagger(kb_info, layout)
    if "split" in kb_info and "enabled" in kb_info["split"]:
        entry["split"] = kb_info["split"]["enabled"]
    else:
        entry["split"] = False
    entry["languages"] = ["English"]
    # Use the title of the keymap readme for the summary (but strip the md header #)
    entry["summary"] = re.sub("^# ", "", readme[:readme.find("\n")])
    entry["OS"] = get_OS(readme)
    entry["writeup"] = f"https://github.com/{username}/qmk_firmware/tree/master/keyboards/{keyboard_name}/keymaps/{username}/readme.md"
    entry["hasLetterOnThumb"] = False # might be wrong
    # Remove any commented out layer templates before counting the number of layers
    # This technique doesn't work for old keymaps which don't use any layout macros like maxr1998's
    entry["layerCount"] = re.sub("/?\*.*LAYOUT", "", keymapdotc).count("LAYOUT")
    assert entry["layerCount"] > 0
    entry["imageDate"] = "idk"
    # TODO rules.mk options
    entry["hasOuterPinkyColumns"] = True # most likely but might be wrong
    entry["hasExtraIndexColumns"] = False # not likely but might be wrong
    entry["hasVerticalCombos"] = False # not likely but might be wrong
    entry["hasHomeRowMods"] = has_home_row_mods(keymapdotc)
    assert set(entry.keys()) == set(headers), f"{set(entry.keys())=}\n{ set(headers)=}"
    return entry

keymapdb = tuple(map(get_keymapdb_entry, kbs_authors))
print(keymapdb)

data = {"headers": headers, "keymaps": keymapdb}
with open("new_data.json", "w") as new_data_f:
    new_data_f.write(json.dumps(data, sort_keys=True, indent=2))

PS: Don't judge the code too much, 'tis just a "quick" script I wrote

[precondition]

I haven't manually reviewed the keymaps yet. Many keymaps are uninteresting (is a keymap for a macro pad, is just a slightly modified standard US QWERTY keyboard layout, ...) and need to be removed but it's hard to programmatically define what is "uninteresting" :p