search

neuropoly / intranet.neuro.polymtl.ca

NeuroPoly's lab manual
https://intranet.neuro.polymtl.ca
4 stars 7 forks source link

Port to sphinx #20

Closed kousu closed 2 years ago

kousu commented 2 years ago

Port this wiki to be built by sphinx and hosted on Github pages, for all the same reasons as https://github.com/neuropoly/neuropoly-docs/issues/20.

kousu commented 2 years ago

Loose plan

kousu commented 2 years ago 
#!/usr/bin/env python3
"""
Convert a Gitbook v2 repo's table of contents to MyST-parser compatible sphinx tables of contents (toctrees).

Usage:

    gitbook2mysttoctrees.py path/to/gitbook-repo
"""

import sys, os
from os.path import splitext, relpath, dirname
from itertools import islice, chain
from tempfile import NamedTemporaryFile
from subprocess import check_call as system
from textwrap import dedent

from pprint import pprint

count=0  # to report a summary

def insert_toctree(fname, files):

    global count

    # find the page title,
    # write a MyST toctree just after it

    dir = dirname(fname)

    with open(fname) as input, NamedTemporaryFile("w", dir=os.path.dirname(fname), delete=False) as output:
        # putting the scratch file into the same dir should make renaming it zero-cost
        try:
            for line in input:
                print(line, file=output, end="")
                if line.startswith("# "): # page title
                    break

            print(file=output)
            print("```{toctree}", file=output)
            print(":hidden:", file=output)
            for page in files:
                page = splitext(relpath(page.strip(), start=dir))[0]
                print(page, file=output)
                count+=1
            print("```", file=output)

            for line in input:
                print(line, file=output, end="")
            os.rename(output.name, fname)
        except:
            os.unlink(output.name)
            raise

def parse(fd, fname):
    """
    parse a markdown list
    """

    indents = [0]
    files = [fname]
    tables_of_contents = [[]]

    for line in fd:
        if line.lstrip()[:1] != "*": continue # skip lines that aren't list elements

        # split the indent off from the contents
        I, line = line.split("*", 1)

        I = len(I) # examine the depth of the current list element
        if indents[-1] < I:
            # entering a submenu
            # adjust the stacks
            indents.append(I)
            files.append(tables_of_contents[-1][-1]) # wait if this is true, do we even need to track files at all?
            tables_of_contents.append([])
        while indents[-1] > I:
            # we're ending a submenu, so pop
            indents.pop()
            yield files.pop(), tables_of_contents.pop()
            #import time; time.sleep(5)

        # lazily parse markdown links
        # this is probably
        text, link = line.split("](", 1)
        text = text[text.index("[")+1:]
        link = link[:link.rindex(")")]

        # regardless of whether we entered, stayed in, or left a submenu,
        # record the current line to the current
        tables_of_contents[-1].append(link)

    # at the end, we should have gotten back to the start
    assert len(files) == len(tables_of_contents) == 1 and files[0] == fname
    yield files.pop(), tables_of_contents.pop()

if __name__ == '__main__':
    if len(sys.argv) == 2:
        os.chdir(sys.argv[1])
    with open("SUMMARY.md") as fd:
        for itoc, (fname, toctree) in enumerate(parse(fd, "SUMMARY.md")):
            if fname == "SUMMARY.md":
                 # special case: Gitbook stores its ToC in the top level SUMMARY.md separate file
                 # redirect its contents into the top level README.md instead
                fname = "README.md"
                toctree[toctree.index("README.md")] = "self" # 'self' makes sphinx take the title from the page without recursing back onto itself
            insert_toctree(fname, toctree)
            system(["git", "add", fname])

    system(["git", "rm", "--quiet", "SUMMARY.md"])

    print(f"Spread {count} pages across {itoc+1} toctrees.")
    print(dedent("""
      To see the changes made, run:

         git diff --staged -- '*.md'

      If you wish to undo them, run:

         git reset HEAD -- '*.md' && git checkout -- '*.md'
      """))
kousu commented 2 years ago 
$ cat gitbook2mystadmonitions.sh 
#!/usr/bin/env bash

find "${1:-./}" -name '*.md' | xargs \
sed -i -e 's/{% hint style="info" %}/```{note}/' \
    -e 's/{% hint style="warning" %}/```{warning}/' \
    -e 's/{% hint style="danger" %}/```{warning}/' \
    -e 's/{% endhint %}/```/'

produces:

 ### **Create Job Script**

-{% hint style="info" %}
+```{note}
 **Example**

 ```bash
@@ -51,7 +51,7 @@ sct_testing
 echo "Script ended at:"
 date

-{% endhint %} +```

kousu commented 2 years ago 
$ cat gitbook2mysttabs.sh 
#!/usr/bin/env bash

# this ports to sphinx_panels: https://sphinx-panels.readthedocs.io/en/sphinx-book-theme/index.html#installation
# requires adding sphinx_panels to conf.py::extensions, and sphinx-panels to setup.py
#
# NB: using four ticks ```` to make sure any embedded code blocks are safe
# if there's embedded code blocks with their own embedded code blocks we might have a problem
find "${1:-./}" -name '*.md' | xargs \
sed -i \
    -e 's/{% tabs %}//' \
    -e 's/{% tab title="\(.*\)" %}/````{tabbed} \1/' \
    -e 's/{% endtab %}/````/' \
    -e 's/{% endtabs %}//'

exit 0

# this alternate version ports to https://sphinx-tabs.readthedocs.io/en/latest/
# requires adding sphinx_tabs.tabs to conf.py::extensions, and sphinx-tabs to setup.py
find "${1:-./}" -name '*.md' | xargs \
sed -i \
    -e 's/{% tabs %}/````{tabs}/' \
    -e 's/{% tab title="\(.*\)" %}/```{tab} \1/' \
    -e 's/{% endtab %}/```/' \
    -e 's/{% endtabs %}/````/'
kousu commented 2 years ago 
$ cat ./gitbook2mystemojiheaders.sh 
#!/usr/bin/env bash

# wrap emojis in page titles in a <span> tag so we can style them
# this is a bad hack but it's what we've got for now

find "${1:-./}" -name '*.md' | xargs \
  perl -CSAD -i -pe 's/^# ([\x{2000}-\x{1FFFF}]+) /# <span>\1<\/span> /'

this produces

diff --git a/onboarding/README.md b/onboarding/README.md
index 803dae9..3cdfc81 100644
--- a/onboarding/README.md
+++ b/onboarding/README.md
@@ -1,4 +1,4 @@
-# 👋 Onboarding
+# <span>👋</span> Onboarding

 Welcome to NeuroPoly Lab!

diff --git a/practical-information/README.md b/practical-information/README.md
index cb0d493..65663c8 100644
--- a/practical-information/README.md
+++ b/practical-information/README.md
@@ -1,4 +1,4 @@
-# 📎 Practical Information
+# <span>📎</span> Practical Information

 This section provides useful information for applicants to NeuroPoly Lab and newly arrived: VISA, moving to Montreal, scholarship available, etc.

diff --git a/practical-information/moving-to-montreal.md b/practical-information/moving-to-montreal.md
index 4e54665..36db2c4 100644
--- a/practical-information/moving-to-montreal.md
+++ b/practical-information/moving-to-montreal.md
@@ -1,4 +1,4 @@
-# 🇨🇦  Living in Montreal
+# <span>🇨🇦</span>  Living in Montreal
kousu commented 2 years ago 
$ cat ./gitbookembeds2mystlinks.sh 
#!/usr/bin/env bash

# defang proprietary {% embed %} tag by turning it into a markdown link
find "${1:-./}" -name '*.md' | xargs \
    perl -CSAD -i -pe 's/{% embed url="(.+)" %}/[\1\](\1)/'

produces

diff --git a/software-development/deep-learning/README.md b/software-development/deep-learning/README.md
index 25e26d6..e9ec267 100644
--- a/software-development/deep-learning/README.md
+++ b/software-development/deep-learning/README.md
@@ -2,11 +2,11 @@

 <U+200B>[http://scikit-learn.org/](http://scikit-learn.org/)

-{% embed url="https://github.com/Lasagne/Lasagne" %}
+[https://github.com/Lasagne/Lasagne](https://github.com/Lasagne/Lasagne)

-{% embed url="https://github.com/fchollet/keras" %}
+[https://github.com/fchollet/keras](https://github.com/fchollet/keras)

-{% embed url="http://nilearn.github.io/" %}
+[http://nilearn.github.io/](http://nilearn.github.io/)
kousu commented 2 years ago

And this does all of them together:

s$ cat gitbook2myst.sh 
#!/usr/bin/env bash

set -e

./gitbook2mystadmonitions.sh
./gitbook2mystemojiheaders.sh 
./gitbook2mysttabs.sh
./gitbookembeds2mystlinks.sh
./gitbook2mysttoctrees.py

However this won't work unless you manually edit in

diff --git a/conf.py b/conf.py
index dce1e7f..65c9afe 100644
--- a/conf.py
+++ b/conf.py
@@ -33,6 +33,7 @@ root_doc = 'README'
 # ones.
 extensions = [
         'myst_parser', # there's also myst_nb, which supports embedding Jupyter notebooks, but is heavier.
+        'sphinx_panels',
 ]

 # Add any paths that contain templates here, relative to this directory.
diff --git a/setup.py b/setup.py
index ce37a9a..72dc84e 100644
--- a/setup.py
+++ b/setup.py
@@ -19,6 +19,7 @@ setup(
         "sphinx": [
             "myst-parser",
             "sphinx-book-theme",
+            "sphinx-panels",
             # pinned because of this bug https://github.com/pydata/pydata-sphinx-theme/pull/509
             # and that the patched sphinx-book-theme isn't out yet: https://github.com/executablebooks/sphinx-book-theme/issues/428#issuecomment-966021270
             "sphinx~=4.2.0", # TODO: unpin when the next sphinx-book-theme is released