fix relative links

[Byron]

Supersedes #421 .

The story behind this is that at some point I pushed some commits into #421 to fix CI. While at it, I also addressed some other issues I noticed and upgraded dependencies. These where lost when the PR was replaced by one force-pushed commit (see the Details section). This PR takes all new changes from this PR, selected by hand and visual comparison, so the authorship is maintained by a trailer line instead.

```patch From 9e5eff17dc31ea496c3d8973f77eade8ab869d3a Mon Sep 17 00:00:00 2001 From: Filipp Date: Tue, 28 Mar 2023 11:00:26 +0300 Subject: [PATCH] Fix relative links check the FIXMEs --- src/generator/lib/util.py | 34 +++++++++++++----- src/generator/templates/api/lib/lib.mako | 2 +- src/generator/templates/api/lib/mbuild.mako | 10 +++--- src/generator/templates/api/lib/rbuild.mako | 4 +-- src/generator/templates/api/lib/schema.mako | 6 ++-- src/rust/preproc/Cargo.toml | 1 + src/rust/preproc/src/main.rs | 39 +++++++++++++++++++-- 7 files changed, 74 insertions(+), 22 deletions(-) diff --git a/src/generator/lib/util.py b/src/generator/lib/util.py index 0ddc7616b1f..7be208b5a94 100644 --- a/src/generator/lib/util.py +++ b/src/generator/lib/util.py @@ -1,6 +1,7 @@ import os import re import subprocess +import urllib import inflect from dataclasses import dataclass @@ -17,6 +18,7 @@ flags=re.IGNORECASE | re.MULTILINE) re_find_replacements = re.compile(r"\{[/\+]?\w+\*?\}") +re_relative_links = re.compile(r"\]\s*\([^h]") HTTP_METHODS = set(("OPTIONS", "GET", "POST", "PUT", "DELETE", "HEAD", "TRACE", "CONNECT", "PATCH")) @@ -132,19 +134,34 @@ def rust_doc_comment(s): def has_markdown_codeblock_with_indentation(s): return re_spaces_after_newline.search(s) != None - -def preprocess(s): - p = subprocess.Popen([os.environ['PREPROC']], close_fds=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) +def preprocess(base_url, s): + if base_url is None: + printl(f"WARNING {s} has no base_url") + p = subprocess.Popen( + [os.environ['PREPROC']], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + env={"URL_BASE": base_url or ""} + ) + res = p.communicate(s.encode('utf-8')) + exitcode = p.wait(timeout=1) + if exitcode != 0: + raise ValueError(f"Child process exited with non-zero code {exitcode}") return res[0].decode('utf-8') +def has_relative_links(s): + return re_relative_links.search(s) is not None # runs the preprocessor in case there is evidence for code blocks using indentation -def rust_doc_sanitize(s): - if has_markdown_codeblock_with_indentation(s): - return preprocess(s) - else: - return s +def rust_doc_sanitize(base_url): + def fixer(s): + if has_markdown_codeblock_with_indentation(s) or has_relative_links(s): + return preprocess(base_url, s) + else: + return s + return fixer # rust comment filter @@ -1173,6 +1190,5 @@ def string_impl(p): "string": lambda x: x }.get(p.get("format", p["type"]), lambda x: f"{x}.to_string()") - if __name__ == '__main__': raise AssertionError('For import only') diff --git a/src/generator/templates/api/lib/lib.mako b/src/generator/templates/api/lib/lib.mako index 22eb880e5e0..f7f407a164d 100644 --- a/src/generator/templates/api/lib/lib.mako +++ b/src/generator/templates/api/lib/lib.mako @@ -334,7 +334,7 @@ You can read the full text at the repository's [license file][repo-license]. #[derive(PartialEq, Eq, Hash)] pub enum Scope { % for url, scope in auth.oauth2.scopes.items(): - ${scope.description | rust_doc_sanitize, rust_doc_comment} + ${scope.description | rust_doc_sanitize(documentationLink), rust_doc_comment} ${scope_url_to_variant(name, url, fully_qualified=False)}, % if not loop.last: diff --git a/src/generator/templates/api/lib/mbuild.mako b/src/generator/templates/api/lib/mbuild.mako index f6fd75d157d..23e3c8d4db1 100644 --- a/src/generator/templates/api/lib/mbuild.mako +++ b/src/generator/templates/api/lib/mbuild.mako @@ -66,7 +66,7 @@ parts = get_parts(part_prop) %>\ % if 'description' in m: -${m.description | rust_doc_sanitize, rust_doc_comment} +${m.description | rust_doc_sanitize(documentationLink), rust_doc_comment} /// % endif % if m.get('supportsMediaDownload', False): @@ -90,7 +90,7 @@ ${m.description | rust_doc_sanitize, rust_doc_comment} /// It is not used directly, but through a [`${rb_type(resource)}`] instance. /// % if part_desc: -${part_desc | rust_doc_sanitize, rust_doc_comment} +${part_desc | rust_doc_sanitize(documentationLink), rust_doc_comment} /// % if m.get('scopes'): /// # Scopes @@ -248,7 +248,7 @@ ${self._setter_fn(resource, method, m, p, part_prop, ThisType, c)}\ # end part description %>\ % if 'description' in p: - ${p.description | rust_doc_sanitize, rust_doc_comment, indent_all_but_first_by(1)} + ${p.description | rust_doc_sanitize(documentationLink), rust_doc_comment, indent_all_but_first_by(1)} % endif % if is_repeated_property(p): /// @@ -272,7 +272,7 @@ ${self._setter_fn(resource, method, m, p, part_prop, ThisType, c)}\ % endif % if part_desc: /// - ${part_desc | rust_doc_sanitize, rust_doc_comment, indent_all_but_first_by(1)} + ${part_desc | rust_doc_sanitize(documentationLink), rust_doc_comment, indent_all_but_first_by(1)} % endif pub fn ${mangle_ident(setter_fn_name(p))}(mut self, ${value_name}: ${InType}) -> ${ThisType} { % if p.get('repeated', False): @@ -898,7 +898,7 @@ if enable_resource_parsing \ } % for p in media_params: - ${p.description | rust_doc_sanitize, rust_doc_comment, indent_all_but_first_by(1)} + ${p.description | rust_doc_sanitize(documentationLink), rust_doc_comment, indent_all_but_first_by(1)} /// % for item_name, item in p.info.items(): /// * *${split_camelcase_s(item_name)}*: ${isinstance(item, (list, tuple)) and put_and(enclose_in("'", item)) or str(item)} diff --git a/src/generator/templates/api/lib/rbuild.mako b/src/generator/templates/api/lib/rbuild.mako index 696f3e6b66f..bd13253981c 100644 --- a/src/generator/templates/api/lib/rbuild.mako +++ b/src/generator/templates/api/lib/rbuild.mako @@ -80,7 +80,7 @@ impl${rb_params} ${ThisType} { % if 'description' in m: /// Create a builder to help you perform the following task: /// - ${m.description | rust_doc_sanitize, rust_doc_comment, indent_all_but_first_by(1)} + ${m.description | rust_doc_sanitize(documentationLink), rust_doc_comment, indent_all_but_first_by(1)} % endif % if required_props: /// @@ -91,7 +91,7 @@ impl${rb_params} ${ThisType} { arg_prefix = "/// * `" + p.name + "` - " %>\ ${arg_prefix}${p.get('description', "No description provided.") - | remove_empty_lines, prefix_all_but_first_with(' ' * SPACES_PER_TAB + '///' + ' ' * (len(arg_prefix) - len('///')))} + | rust_doc_sanitize(documentationLink), remove_empty_lines, prefix_all_but_first_with(' ' * SPACES_PER_TAB + '///' + ' ' * (len(arg_prefix) - len('///')))} % endfor % endif pub fn ${mangle_ident(a)}${type_params}(&self${method_args}) -> ${RType}${mb_tparams} { diff --git a/src/generator/templates/api/lib/schema.mako b/src/generator/templates/api/lib/schema.mako index 55e2c5fdcf0..7673938e5bf 100644 --- a/src/generator/templates/api/lib/schema.mako +++ b/src/generator/templates/api/lib/schema.mako @@ -13,7 +13,7 @@ % if properties: ${struct} { % for pn, p in items(properties): - ${p.get('description', 'no description provided') | rust_doc_sanitize, rust_doc_comment, indent_all_but_first_by(1)} + ${p.get('description', 'no description provided') | rust_doc_sanitize(documentationLink), rust_doc_comment, indent_all_but_first_by(1)} % if pn != mangle_ident(pn): #[serde(rename="${pn}")] % endif @@ -36,7 +36,7 @@ ${struct}(pub ${to_rust_type(schemas, s.id, NESTED_TYPE_SUFFIX, s, allow_optiona %>\ pub enum ${et} { % for p in s.variant.map: - ${p.get('description', 'no description provided') | rust_doc_sanitize, rust_doc_comment, indent_all_but_first_by(1)} + ${p.get('description', 'no description provided') | rust_doc_sanitize(documentationLink), rust_doc_comment, indent_all_but_first_by(1)} % if variant_type(p) != p.type_value: #[serde(rename="${p.type_value}")] % endif @@ -80,7 +80,7 @@ ${struct} { _never_set: Option } s_type = s.id %>\ -<%block filter="rust_doc_sanitize, rust_doc_comment">\ +<%block filter="rust_doc_sanitize(documentationLink), rust_doc_comment">\ ${doc(s, c)}\ #[serde_with::serde_as(crate = "::client::serde_with")] diff --git a/src/rust/preproc/Cargo.toml b/src/rust/preproc/Cargo.toml index 81ce97c34ce..34aa4713a37 100644 --- a/src/rust/preproc/Cargo.toml +++ b/src/rust/preproc/Cargo.toml @@ -16,3 +16,4 @@ test = false [dependencies] pulldown-cmark-to-cmark = "10.0.0" pulldown-cmark = "0.9.0" +url = "2.3.1" diff --git a/src/rust/preproc/src/main.rs b/src/rust/preproc/src/main.rs index d5be8af81a7..f6c3d756cb2 100644 --- a/src/rust/preproc/src/main.rs +++ b/src/rust/preproc/src/main.rs @@ -1,9 +1,9 @@ extern crate pulldown_cmark; extern crate pulldown_cmark_to_cmark; -use pulldown_cmark::Parser; +use pulldown_cmark::{CowStr, Parser, Tag}; use pulldown_cmark_to_cmark::cmark; -use std::io::{self, Read, Write}; +use std::{io::{self, Read, Write}, ops::Not}; fn main() { let md = { @@ -13,6 +13,33 @@ fn main() { }; let mut output = String::with_capacity(2048); + let url_base = std::env::var("URL_BASE").unwrap_or_default(); + + let url_base = url_base + .strip_suffix('/') + .map(String::from) + .unwrap_or(url_base); + + let url_root = if url_base.is_empty() { + String::new() + } else { + let parsed = url::Url::parse(&url_base).unwrap(); + let scheme = parsed.scheme(); + let host = parsed.host_str().unwrap_or_default(); + let port = parsed.port().map(|p|format!(":{p}")).unwrap_or_default(); + format!("{scheme}://{host}{port}") + }; + + fn fix_url<'a>(root: &str, base: &str, url: CowStr<'a>) -> CowStr<'a> { + if url.starts_with('/') { + format!("{root}{url}").into() + } else if url.starts_with("..") { + format!("{base}/{url}").into() + } else { + url + } + } + cmark( Parser::new_ext(&md, pulldown_cmark::Options::all()).map(|e| { use pulldown_cmark::Event::*; @@ -23,9 +50,17 @@ fn main() { CodeBlock(pulldown_cmark::CodeBlockKind::Indented) => Start(CodeBlock( pulldown_cmark::CodeBlockKind::Fenced("text".into()), )), + Link(lt, url, title) => Start(Link( + lt.clone(), + fix_url(&url_root, &url_base, url.clone()), + title.clone(), + )), _ => e, } } + End(Tag::Link(lt, url, title)) => { + End(Tag::Link(lt, fix_url(&url_root, &url_base, url), title)) + } _ => e, } }), ```

[Byron]

Everything works with cargo-check and cargo doc, even though there are a lot of warnings.