[RFE] Generate man pages for etcd, etcdctl, etcd.conf

[ingvagabund]

Given some users are still reading man pages before hitting etcd -h or reading docs, it make sense to generate the man pages automatically (or into a hand-crafted template).

The etcd.conf can be generated based on [1]

[1] https://github.com/coreos/etcd/blob/master/Documentation/op-guide/configuration.md

[ingvagabund]

There are some online man pages (most likely picked from the help):

http://manpages.org/etcd http://manpages.org/etcdctl

[heyitsanthony]

I tried generating etcdctl man pages from the README with go-md2man; the output was not very good and broken at times. /cc @joshix

[ingvagabund]

It would be great to generate the man pages right from the go code. At best, extend the urfare/cli, resp. pflag to provide such functionality. When generating the man pages, you can than include the etcdctl/ctlv2/ctl.go file and just call Manpage() (or GetMD() or general GetFORMAT()) over the cli.NewApp() object. Checking the last time the pflag it was not a trivial task to implement such an extension.

[ingvagabund]

FYI with the patch below applied I am able to generate man pages for etcdctl of both versions (i.e. v2 and v3).:

From 634797b70ae901d5909c7e128a0240b6fe20ed02 Mon Sep 17 00:00:00 2001
From: Jan Chaloupka <jchaloup@redhat.com>
Date: Mon, 6 Nov 2017 23:22:52 +0100
Subject: [PATCH] hack to generate man pages

---
 cmd/vendor/github.com/urfave/cli/flag.go |  4 +--
 etcdctl/ctlv2/ctl.go                     | 61 ++++++++++++++++++++++++++++++++
 etcdctl/ctlv3/ctl_nocov.go               | 15 +++++---
 3 files changed, 74 insertions(+), 6 deletions(-)

diff --git a/cmd/vendor/github.com/urfave/cli/flag.go b/cmd/vendor/github.com/urfave/cli/flag.go
index f8a28d1..9787fe1 100644
--- a/cmd/vendor/github.com/urfave/cli/flag.go
+++ b/cmd/vendor/github.com/urfave/cli/flag.go
@@ -752,7 +752,7 @@ func prefixedNames(fullName, placeholder string) string {
    parts := strings.Split(fullName, ",")
    for i, name := range parts {
        name = strings.Trim(name, " ")
-       prefixed += prefixFor(name) + name
+       prefixed += "\\fB" + prefixFor(name) + name + "\\fP"
        if placeholder != "" {
            prefixed += " " + placeholder
        }
@@ -828,7 +828,7 @@ func stringifyFlag(f Flag) string {
    usageWithDefault := strings.TrimSpace(fmt.Sprintf("%s%s", usage, defaultValueString))

    return withEnvHint(fv.FieldByName("EnvVar").String(),
-       fmt.Sprintf("%s\t%s", prefixedNames(fv.FieldByName("Name").String(), placeholder), usageWithDefault))
+       fmt.Sprintf("%s\n\t\t\t%s", prefixedNames(fv.FieldByName("Name").String(), placeholder), usageWithDefault))
 }

 func stringifyIntSliceFlag(f IntSliceFlag) string {
diff --git a/etcdctl/ctlv2/ctl.go b/etcdctl/ctlv2/ctl.go
index e949b06..1c7a7cf 100644
--- a/etcdctl/ctlv2/ctl.go
+++ b/etcdctl/ctlv2/ctl.go
@@ -42,6 +42,67 @@ func Start(apiv string) {
            "   Set environment variable ETCDCTL_API=3 to use v3 API or ETCDCTL_API=2 to use v2 API."
    }

+   cli.AppHelpTemplate = `.TH "ETCD" "1" " etcd User Manuals" "Etcd contributors" "Nov 2017"  ""
+.SH NAME:
+{{.Name}} - {{.Usage}}
+
+{{if .Version}}
+.SH VERSION:
+   {{.Version}}
+{{end}}
+
+.SH USAGE:
+   {{.HelpName}} {{if .VisibleFlags}}[global options]{{end}}{{if .Commands}} command [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}
+   {{if .Commands}}
+
+.SH COMMANDS:
+{{range .Commands}}{{if not .HideHelp}}\fB{{ .Name }}\fP{{ "\n\t\t\t" }}{{.Usage}}{{ "\n" }}
+
+{{end}}{{end}}{{end}}{{if .VisibleFlags}}
+
+.SH GLOBAL OPTIONS:
+{{range .VisibleFlags}}{{ . }}
+
+{{end}}{{end}}
+
+.SH SEE ALSO
+{{range .Commands}}{{if not .HideHelp}}{{if ne .Name "help" }}\fBetcdctl-{{ .Name }}(1)\fP,
+{{end}}{{end}}{{end}}
+`
+
+   cli.CommandHelpTemplate = `.TH "ETCD" "1" " etcd User Manuals" "Etcd contributors" "Nov 2017"  ""
+.SH NAME:
+   {{.HelpName}} - {{.Usage}}
+
+.SH USAGE:
+   {{.HelpName}}{{if .VisibleFlags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}
+
+{{if .VisibleFlags}}
+.SH OPTIONS:
+   {{range .VisibleFlags}}{{.}}
+   {{end}}{{end}}
+`
+
+
+   cli.SubcommandHelpTemplate = `.TH "ETCD" "1" " etcd User Manuals" "Etcd contributors" "Nov 2017"  ""
+.SH NAME:
+   {{.HelpName}} - {{.Usage}}
+
+.SH USAGE:
+   {{.HelpName}} command{{if .VisibleFlags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}
+
+.SH COMMANDS:
+{{range .VisibleCategories}}{{if .Name}}
+{{.Name}}:{{end}}{{range .VisibleCommands}}
+\fB{{ .Name }}\fP{{"\n\t\t\t"}}{{.Usage}}{{ "\n" }}{{end}}
+{{end}}
+
+{{if .VisibleFlags}}
+.SH OPTIONS:
+{{range .VisibleFlags}}{{.}}
+{{end}}{{end}}
+`
+
    app.Flags = []cli.Flag{
        cli.BoolFlag{Name: "debug", Usage: "output cURL commands which can be used to reproduce the request"},
        cli.BoolFlag{Name: "no-sync", Usage: "don't synchronize cluster information before sending request"},
diff --git a/etcdctl/ctlv3/ctl_nocov.go b/etcdctl/ctlv3/ctl_nocov.go
index 52751fe..9d048ba 100644
--- a/etcdctl/ctlv3/ctl_nocov.go
+++ b/etcdctl/ctlv3/ctl_nocov.go
@@ -16,13 +16,20 @@

 package ctlv3

-import "github.com/coreos/etcd/etcdctl/ctlv3/command"
+//import "github.com/coreos/etcd/etcdctl/ctlv3/command"
+import "github.com/spf13/cobra"

 func Start() {
    rootCmd.SetUsageFunc(usageFunc)
    // Make help just show the usage
    rootCmd.SetHelpTemplate(`{{.UsageString}}`)
-   if err := rootCmd.Execute(); err != nil {
-       command.ExitWithError(command.ExitError, err)
-   }
+   //if err := rootCmd.Execute(); err != nil {
+   //  command.ExitWithError(command.ExitError, err)
+   //}
+        header := &cobra.GenManHeader{
+                Title: "etcdctl3",
+                Section: "1",
+        }
+
+        cobra.GenManTree(rootCmd, header, "")
 }
-- 
2.7.5

Unfortunately, github.com/urfave/cli/flag.go does not allow a simple way to override the default output format as it implements the Stringer interface. The github.com/spf13/cobra on the other hand is very friendly and the man generating abilities are quite straightforward.

Can't say the same for the etcd binary itself as it uses the flags package to handle the flags. It is considerable to replace the flags with the github.com/spf13/cobra.

[ingvagabund]

@heyitsanthony any plans to generate https://github.com/coreos/etcd/blob/master/etcdmain/help.go via github.com/spf13/cobra?

EDIT: or via github.com/urfave/cli? Once done we could generate https://github.com/coreos/etcd/blob/master/Documentation/op-guide/configuration.md with it as well (or any middle product that is easy to convert to md) and keep all the flags and their description at one place. What about that? Checking the git history of the etcdmain/help.go it looks pretty wild. Easy to forget about a flag, it gets old pretty fast.

[ingvagabund]

Checking the etcdmain/config.go it should not be hard to mimic the functionality of the flag.FlagSet and store the individual flags "just for man/md generating". E.g.

type MyFlagSet {
    origfs * flag.FlagSet
    // arbitrary relevent data, e.g. `github.com/spf13/cobra` or `github.com/urfave/cli` internally
    // to generate the man page one will just use functionality of the `cobra`, resp. `cli`
}

func InitFlagSet(fs * flag.FlagSet) {
    return MyFlagSet{origfs: fs}
}

func (fs*MyFlagSet) StringVar(p *string, name string, value string, usage string) {
    // store the (p, name, value, usage) tuple for later
    fs.origings.StringVar(p, name, value, usage)
}

// Analogically for `Var`, `UintVar`, `Int64Var`, `DurationVar`, etc.

Then in the code just:

fs := InitFlagSet(cfg.FlagSet)

Then, on an appropriate place (e.g. when MAN=true env is set) the --help will print the man page instead.

[ingvagabund]

With https://gist.github.com/ingvagabund/3cad6cd9658705d415a6779010b53f0a and the patch above it's possible to generate the etcd.1 man page:

From 81519130b0abec199ddc9e3559e64884742b1bf5 Mon Sep 17 00:00:00 2001
From: Jan Chaloupka <jchaloup@redhat.com>
Date: Tue, 7 Nov 2017 14:04:01 +0100
Subject: [PATCH] hack etcdmain to generate etcd.1

---
 etcdmain/config.go       |  25 ++++----
 etcdmain/fake_flagset.go | 157 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 171 insertions(+), 11 deletions(-)
 create mode 100644 etcdmain/fake_flagset.go

diff --git a/etcdmain/config.go b/etcdmain/config.go
index b873220..4770334 100644
--- a/etcdmain/config.go
+++ b/etcdmain/config.go
@@ -118,14 +118,14 @@ func newConfig() *config {
        ),
    }

-   fs := cfg.FlagSet
-   fs.Usage = func() {
+   fs := InitFlagSet(cfg.FlagSet)
+   cfg.FlagSet.Usage = func() {
        fmt.Fprintln(os.Stderr, usageline)
    }

    fs.StringVar(&cfg.configFile, "config-file", "", "Path to the server configuration file")

-   // member
+   fs.AddGroup("member")
    fs.Var(cfg.CorsInfo, "cors", "Comma-separated white list of origins for CORS (cross-origin resource sharing).")
    fs.StringVar(&cfg.Dir, "data-dir", cfg.Dir, "Path to the data directory.")
    fs.StringVar(&cfg.WalDir, "wal-dir", cfg.WalDir, "Path to the dedicated wal directory.")
@@ -139,7 +139,7 @@ func newConfig() *config {
    fs.UintVar(&cfg.ElectionMs, "election-timeout", cfg.ElectionMs, "Time (in milliseconds) for an election to timeout.")
    fs.Int64Var(&cfg.QuotaBackendBytes, "quota-backend-bytes", cfg.QuotaBackendBytes, "Raise alarms when backend size exceeds the given quota. 0 means use the default quota.")

-   // clustering
+   fs.AddGroup("clustering")
    fs.Var(flags.NewURLsValue(embed.DefaultInitialAdvertisePeerURLs), "initial-advertise-peer-urls", "List of this member's peer URLs to advertise to the rest of the cluster.")
    fs.Var(flags.NewURLsValue(embed.DefaultAdvertiseClientURLs), "advertise-client-urls", "List of this member's client URLs to advertise to the public.")
    fs.StringVar(&cfg.Durl, "discovery", cfg.Durl, "Discovery URL used to bootstrap the cluster.")
@@ -160,7 +160,7 @@ func newConfig() *config {
    fs.BoolVar(&cfg.StrictReconfigCheck, "strict-reconfig-check", cfg.StrictReconfigCheck, "Reject reconfiguration requests that would cause quorum loss.")
    fs.BoolVar(&cfg.EnableV2, "enable-v2", true, "Accept etcd V2 client requests.")

-   // proxy
+   fs.AddGroup("proxy")
    fs.Var(cfg.proxy, "proxy", fmt.Sprintf("Valid values include %s", strings.Join(cfg.proxy.Values, ", ")))
    if err := cfg.proxy.Set(proxyFlagOff); err != nil {
        // Should never happen.
@@ -172,7 +172,7 @@ func newConfig() *config {
    fs.UintVar(&cfg.ProxyWriteTimeoutMs, "proxy-write-timeout", cfg.ProxyWriteTimeoutMs, "Time (in milliseconds) for a write to timeout.")
    fs.UintVar(&cfg.ProxyReadTimeoutMs, "proxy-read-timeout", cfg.ProxyReadTimeoutMs, "Time (in milliseconds) for a read to timeout.")

-   // security
+   fs.AddGroup("security")
    fs.StringVar(&cfg.ClientTLSInfo.CAFile, "ca-file", "", "DEPRECATED: Path to the client server TLS CA file.")
    fs.StringVar(&cfg.ClientTLSInfo.CertFile, "cert-file", "", "Path to the client server TLS cert file.")
    fs.StringVar(&cfg.ClientTLSInfo.KeyFile, "key-file", "", "Path to the client server TLS key file.")
@@ -186,28 +186,31 @@ func newConfig() *config {
    fs.StringVar(&cfg.PeerTLSInfo.TrustedCAFile, "peer-trusted-ca-file", "", "Path to the peer server TLS trusted CA file.")
    fs.BoolVar(&cfg.PeerAutoTLS, "peer-auto-tls", false, "Peer TLS using generated certificates")

-   // logging
+   fs.AddGroup("logging")
    fs.BoolVar(&cfg.Debug, "debug", false, "Enable debug-level logging for etcd.")
    fs.StringVar(&cfg.LogPkgLevels, "log-package-levels", "", "Specify a particular log level for each etcd package (eg: 'etcdmain=CRITICAL,etcdserver=DEBUG').")
    fs.StringVar(&cfg.logOutput, "log-output", "default", "Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd.")

-   // unsafe
+   fs.AddGroup("unsafe")
    fs.BoolVar(&cfg.ForceNewCluster, "force-new-cluster", false, "Force to create a new one member cluster.")

-   // version
+   fs.AddGroup("version")
    fs.BoolVar(&cfg.printVersion, "version", false, "Print the version and exit.")

    fs.IntVar(&cfg.AutoCompactionRetention, "auto-compaction-retention", 0, "Auto compaction retention for mvcc key value store in hour. 0 means disable auto compaction.")

-   // pprof profiler via HTTP
+   fs.AddGroup("profiling")
    fs.BoolVar(&cfg.EnablePprof, "enable-pprof", false, "Enable runtime profiling data via HTTP server. Address is at client URL + \"/debug/pprof/\"")

    // additional metrics
    fs.StringVar(&cfg.Metrics, "metrics", cfg.Metrics, "Set level of detail for exported metrics, specify 'extensive' to include histogram metrics")

-   // auth
+   fs.AddGroup("auth")
    fs.StringVar(&cfg.AuthToken, "auth-token", cfg.AuthToken, "Specify auth token specific options.")

+   fs.GenMan()
+   os.Exit(0)
+
    // ignored
    for _, f := range cfg.ignored {
        fs.Var(&flags.IgnoredFlag{Name: f}, f, "")

[ingvagabund]

ping :)

[gyuho]

@ingvagabund Sorry for delay. Could you send a PR with your patch? /cc @joelegasse