generate/main.go

package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"

	"github.com/cockroachdb/docs/generate/extract"
	"github.com/spf13/cobra"
)

var cmdFuncs *cobra.Command

func main() {
	var (
		inputPath  string
		outputPath string
	)

	read := func() io.Reader {
		var r io.Reader = os.Stdin
		if inputPath != "" {
			f, err := os.Open(inputPath)
			if err != nil {
				log.Fatal(err)
			}
			defer f.Close()
			b, err := ioutil.ReadAll(f)
			if err != nil {
				log.Fatal(err)
			}
			r = bytes.NewReader(b)
		}
		return r
	}

	write := func(b []byte) {
		var w io.Writer = os.Stdout
		if outputPath != "" {
			f, err := os.Create(outputPath)
			if err != nil {
				log.Fatal(err)
			}
			defer f.Close()
			w = f
		}
		if _, err := w.Write(b); err != nil {
			log.Fatal(err)
		}
	}

	var addr string

	cmdBNF := &cobra.Command{
		Use:   "bnf",
		Short: "Write EBNF to stdout from sql.y",
		Run: func(cmd *cobra.Command, args []string) {
			b, err := runBNF(addr)
			if err != nil {
				log.Fatal(err)
			}
			write(b)
		},
	}

	cmdBNF.Flags().StringVar(&addr, "addr", "https://raw.githubusercontent.com/cockroachdb/cockroach/master/pkg/sql/parser/sql.y", "Location of sql.y file. Can also specify a local file.")

	var (
		topStmt string
		descend bool
		inline  []string
	)

	cmdParse := &cobra.Command{
		Use:   "reduce",
		Short: "Reduces and simplify an EBNF file to a smaller grammar",
		Long:  "Reads from stdin, writes to stdout.",
		Run: func(cmd *cobra.Command, args []string) {
			b, err := runParse(read(), inline, topStmt, descend, true, nil, nil)
			if err != nil {
				log.Fatal(err)
			}
			write(b)
		},
	}

	cmdParse.Flags().StringVar(&topStmt, "stmt", "stmt_block", "Name of top-level statement.")
	cmdParse.Flags().BoolVar(&descend, "descend", true, "Descend past -stmt.")
	cmdParse.Flags().StringSliceVar(&inline, "inline", nil, "List of statements to inline.")

	cmdRR := &cobra.Command{
		Use:   "rr",
		Short: "Generate railroad diagram from stdin, writes to stdout",
		Run: func(cmd *cobra.Command, args []string) {
			b, err := runRR("", read(), "")
			if err != nil {
				log.Fatal(err)
			}
			write(b)
		},
	}

	cmdBody := &cobra.Command{
		Use:   "body",
		Short: "Extract HTML <body> contents from stdin, writes to stdout",
		Run: func(cmd *cobra.Command, args []string) {
			s, err := extract.InnerTag(read(), "body")
			if err != nil {
				log.Fatal(err)
			}
			write([]byte(s))
		},
	}

	var (
		baseDir     string
		filter      string
		invertMatch bool
		printBNF    bool
		railroadJar string
		maxWorkers  int
	)

	rootCmd := &cobra.Command{
		Use:   "generate",
		Short: "Generate SVG diagrams from SQL grammar",
		Long:  `With no arguments, generates SQL diagrams for all statements.`,
		Run: func(cmd *cobra.Command, args []string) {
			bnf, err := runBNF(addr)
			if err != nil {
				log.Fatal(err)
			}
			br := func() io.Reader {
				return bytes.NewReader(bnf)
			}

			if railroadJar != "" {
				_, err := os.Stat(railroadJar)
				if err != nil {
					log.Printf("%s not found, falling back to website\n", railroadJar)
					log.Println("Cockroach Labs employees can find it on the Google Drive.")
					railroadJar = ""
				}
			}

			if filter == "" {
				filter = ".*"
			}
			filterRE, err := regexp.Compile(filter)
			if err != nil {
				log.Fatal(err)
			}

			var wg sync.WaitGroup

			if filterRE.MatchString("stmt_block") == !invertMatch {
				wg.Add(1)
				go func() {
					defer wg.Done()
					const name = "stmt_block"
					if filter != "" && filter != name {
						return
					}
					g, err := runParse(br(), nil, name, true, true, nil, nil)
					if err != nil {
						log.Fatal(err)
					}
					if printBNF {
						fmt.Printf("%s:\n\n%s\n", name, g)
						return
					}
					rr, err := runRR("stmt_block", bytes.NewReader(g), railroadJar)
					if err != nil {
						log.Fatal(err)
					}
					body, err := extract.InnerTag(bytes.NewReader(rr), "body")
					body = strings.SplitN(body, "<hr/>", 2)[0]
					body += `<p>generated by <a href="http://www.bottlecaps.de/rr/ui">Railroad Diagram Generator</a></p>`
					body = fmt.Sprintf("<div>%s</div>", body)
					if err != nil {
						log.Fatal(err)
					}
					if err := ioutil.WriteFile(filepath.Join(baseDir, "grammar.html"), []byte(body), 0644); err != nil {
						log.Fatal(err)
					}
				}()
			}

			specs := []stmtSpec{
				// TODO(mjibson): improve SET filtering
				// TODO(mjibson): improve SELECT display
				{
					name:   "add_column",
					stmt:   "alter_onetable_stmt",
					inline: []string{"alter_table_cmds", "alter_table_cmd", "column_def"},
					match:  []*regexp.Regexp{regexp.MustCompile("'ADD' ('COLUMN')? ?('IF' 'NOT' 'EXISTS')? ?name")},
					replace: map[string]string{
						" | 'ALTER' opt_column name alter_column_default | 'ALTER' opt_column name 'DROP' 'NOT' 'NULL' | 'DROP' opt_column 'IF' 'EXISTS' name opt_drop_behavior | 'DROP' opt_column name opt_drop_behavior | 'ADD' table_constraint opt_validate_behavior | 'VALIDATE' 'CONSTRAINT' name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' name opt_drop_behavior | 'DROP' 'CONSTRAINT' name opt_drop_behavior": "",
						"relation_expr": "table_name",
						"col_qual_list": "( col_qualification | )"},
					unlink: []string{"table_name"},
				},
				{
					name:    "add_constraint",
					stmt:    "alter_onetable_stmt",
					replace: map[string]string{"relation_expr": "table_name", "alter_table_cmds": "'ADD' 'CONSTRAINT' name constraint_elem"},
					unlink:  []string{"table_name"},
				},
				{
					name:   "alter_column",
					stmt:   "alter_onetable_stmt",
					inline: []string{"alter_table_cmds", "alter_table_cmd", "opt_column", "alter_column_default"},
					match:  []*regexp.Regexp{regexp.MustCompile("'ALTER' 'COLUMN'")},
					replace: map[string]string{
						"( ( ',' ( 'ADD' column_def | 'ADD' 'IF' 'NOT' 'EXISTS' column_def | 'ADD' 'COLUMN' column_def | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_def | 'ALTER' ( 'COLUMN' |  ) name ( 'SET' 'DEFAULT' a_expr | 'DROP' 'DEFAULT' ) | 'ALTER' ( 'COLUMN' |  ) name 'DROP' 'NOT' 'NULL' | 'DROP' ( 'COLUMN' |  ) 'IF' 'EXISTS' name opt_drop_behavior | 'DROP' ( 'COLUMN' |  ) name opt_drop_behavior | 'ADD' table_constraint opt_validate_behavior | 'VALIDATE' 'CONSTRAINT' name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' name opt_drop_behavior | 'DROP' 'CONSTRAINT' name opt_drop_behavior ) ) )*": "",
						"relation_expr": "table_name"},
					unlink: []string{"table_name"},
				},
				{
					name:   "alter_table",
					stmt:   "alter_onetable_stmt",
					inline: []string{"alter_table_cmds", "alter_table_cmd", "column_def", "opt_drop_behavior", "alter_column_default", "opt_column", "opt_set_data", "table_constraint"},
					replace: map[string]string{
						"'VALIDATE' 'CONSTRAINT' name": "",
						"opt_validate_behavior":        "",
						"relation_expr":                "table_name"},
					unlink:  []string{"table_name"},
					nosplit: true,
				},
				{
					name:    "alter_view",
					stmt:    "alter_rename_view_stmt",
					inline:  []string{"opt_transaction"},
					replace: map[string]string{"relation_expr": "view_name", "qualified_name": "name"}, unlink: []string{"view_name", "name"},
				},
				{
					name:   "backup",
					stmt:   "backup_stmt",
					inline: []string{"targets", "table_pattern_list", "name_list", "opt_as_of_clause", "opt_incremental", "opt_with_options"},
					match:  []*regexp.Regexp{regexp.MustCompile("'BACKUP'")},
					replace: map[string]string{
						"non_reserved_word_or_sconst":                     "destination",
						"'AS' 'OF' 'SYSTEM' 'TIME' a_expr_const":          "'AS OF SYSTEM TIME' timestamp",
						"'INCREMENTAL' 'FROM' string_or_placeholder_list": "'INCREMENTAL FROM' full_backup_location ( | ',' incremental_backup_location ( ',' incremental_backup_location )* )",
						"'WITH' 'OPTIONS' '(' kv_option_list ')'":         "",
					},
					unlink: []string{"destination", "timestamp", "full_backup_location", "incremental_backup_location"},
				},
				{
					name:    "begin_transaction",
					stmt:    "begin_stmt",
					inline:  []string{"opt_transaction", "begin_transaction", "transaction_mode", "transaction_iso_level", "transaction_user_priority", "user_priority", "iso_level", "transaction_mode_list", "opt_comma", "transaction_read_mode"},
					exclude: []*regexp.Regexp{regexp.MustCompile("'START'")},
					replace: map[string]string{
						"'ISOLATION' 'LEVEL'": "'ISOLATION LEVEL'",
						//" | transaction_read_mode":                                                                      "",
						//" transaction_read_mode":                                                                        "",
						"'READ' 'UNCOMMITTED' | 'READ' 'COMMITTED' | 'SNAPSHOT' | 'REPEATABLE' 'READ' | 'SERIALIZABLE'": "'SNAPSHOT' | 'SERIALIZABLE'",
						"'READ' 'UNCOMMITTED'": "'SNAPSHOT'",
						"'READ' 'COMMITTED'":   "'SNAPSHOT'",
						"'REPEATABLE' 'READ'":  "'SERIALIZABLE'",
					},
				},
				{
					name:    "check_column_level",
					stmt:    "stmt_block",
					replace: map[string]string{"stmt_list": "'CREATE' 'TABLE' table_name '(' column_name column_type 'CHECK' '(' check_expr ')' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"},
					unlink:  []string{"table_name", "column_name", "column_type", "check_expr", "column_constraints", "table_constraints"},
				},
				{
					name:    "check_table_level",
					stmt:    "stmt_block",
					replace: map[string]string{"stmt_list": "'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' name | ) 'CHECK' '(' check_expr ')' ( table_constraints | ) ')'"},
					unlink:  []string{"table_name", "check_expr", "table_constraints"},
				},
				{
					name:    "column_def",
					stmt:    "column_def",
					replace: map[string]string{"col_qual_list": " ( | ( col_qualification ( col_qualification )* ) )"},
				},
				{name: "col_qualification", stmt: "col_qualification", inline: []string{"col_qualification_elem"}},
				{
					name:   "commit_transaction",
					stmt:   "commit_stmt",
					inline: []string{"opt_transaction"},
					match:  []*regexp.Regexp{regexp.MustCompile("'COMMIT'|'END'")},
				},
				{name: "cancel_job", stmt: "cancel_job_stmt", replace: map[string]string{"a_expr": "job_id"}, unlink: []string{"job_id"}},
				{name: "cancel_query", stmt: "cancel_query_stmt", replace: map[string]string{"a_expr": "query_id"}, unlink: []string{"query_id"}},
				{
					name:    "create_database_stmt",
					inline:  []string{"opt_with", "opt_encoding_clause", "opt_equal"},
					replace: map[string]string{"non_reserved_word_or_sconst": "encoding", "opt_template_clause": "", "opt_lc_collate_clause": "", "opt_lc_ctype_clause": ""},
					unlink:  []string{"name", "encoding"},
				},
				{
					name:   "create_index_stmt",
					inline: []string{"opt_storing", "storing", "opt_unique", "opt_name", "index_params", "index_elem", "opt_asc_desc", "name_list"},
					replace: map[string]string{
						"'INDEX' ( name": "'INDEX' ( index_name",
						"'EXISTS' name":  "'EXISTS' index_name",
						"qualified_name": "table_name",
						" name":          " column_name",
						//"',' name":       "',' column_name",
						//"( name (":       "( column_name (",
					},
					unlink:  []string{"index_name", "table_name", "column_name"},
					nosplit: true,
				},
				{name: "create_table_as_stmt", inline: []string{"opt_column_list", "name_list"}},
				{name: "create_table_stmt", inline: []string{"opt_table_elem_list", "table_elem_list", "table_elem"}},
				{
					name:    "create_view_stmt",
					inline:  []string{"opt_column_list"},
					replace: map[string]string{"any_name": "view_name", "name_list": "column_list"},
					relink:  map[string]string{"view_name": "any_name", "column_list": "name_list"},
				},
				{
					name:    "create_user_stmt",
					inline:  []string{"opt_with", "opt_password"},
					replace: map[string]string{"'SCONST'": "password", "name": "user_name"},
					unlink:  []string{"password", "user_name"},
				},
				{
					name:    "default_value_column_level",
					stmt:    "stmt_block",
					replace: map[string]string{"stmt_list": "'CREATE' 'TABLE' table_name '(' column_name column_type 'DEFAULT' default_value ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"},
					unlink:  []string{"table_name", "column_name", "column_type", "default_value", "table_constraints"},
				},
				{name: "delete_stmt", inline: []string{"relation_expr_opt_alias", "where_clause", "returning_clause", "target_list", "target_elem"}},
				{
					name:   "drop_column",
					stmt:   "alter_onetable_stmt",
					inline: []string{"alter_table_cmds", "alter_table_cmd", "opt_column", "opt_drop_behavior"},
					match:  []*regexp.Regexp{regexp.MustCompile("'DROP' 'COLUMN'")},
					replace: map[string]string{
						"( ( ',' ( 'ADD' column_def | 'ADD' 'IF' 'NOT' 'EXISTS' column_def | 'ADD' 'COLUMN' column_def | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_def | 'ALTER' ( 'COLUMN' |  ) name alter_column_default | 'ALTER' ( 'COLUMN' |  ) name 'DROP' 'NOT' 'NULL' | 'DROP' ( 'COLUMN' |  ) 'IF' 'EXISTS' name ( 'CASCADE' | 'RESTRICT' |  ) | 'DROP' ( 'COLUMN' |  ) name ( 'CASCADE' | 'RESTRICT' |  ) | 'ADD' table_constraint opt_validate_behavior | 'VALIDATE' 'CONSTRAINT' name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' name ( 'CASCADE' | 'RESTRICT' |  ) | 'DROP' 'CONSTRAINT' name ( 'CASCADE' | 'RESTRICT' |  ) ) ) )*": "",
						"relation_expr": "table_name"},
					unlink: []string{"table_name"},
				},
				{
					name:   "drop_constraint",
					stmt:   "alter_onetable_stmt",
					inline: []string{"alter_table_cmds", "alter_table_cmd"},
					match:  []*regexp.Regexp{regexp.MustCompile("relation_expr 'DROP' 'CONSTRAINT'")},
					replace: map[string]string{
						"( ( ',' ( 'ADD' column_def | 'ADD' 'IF' 'NOT' 'EXISTS' column_def | 'ADD' 'COLUMN' column_def | 'ADD' 'COLUMN' 'IF' 'NOT' 'EXISTS' column_def | 'ALTER' opt_column name alter_column_default | 'ALTER' opt_column name 'DROP' 'NOT' 'NULL' | 'DROP' opt_column 'IF' 'EXISTS' name opt_drop_behavior | 'DROP' opt_column name opt_drop_behavior | 'ADD' table_constraint opt_validate_behavior | 'VALIDATE' 'CONSTRAINT' name | 'DROP' 'CONSTRAINT' 'IF' 'EXISTS' name opt_drop_behavior | 'DROP' 'CONSTRAINT' name opt_drop_behavior ) ) )*": "",
						"relation_expr":     "table_name",
						"opt_drop_behavior": ""},
					unlink: []string{"table_name"},
				},
				{
					name:   "drop_database",
					stmt:   "drop_database_stmt",
					inline: []string{"opt_drop_behavior"},
					match:  []*regexp.Regexp{regexp.MustCompile("'DROP' 'DATABASE'")},
				},
				{
					name:    "drop_index",
					stmt:    "drop_index_stmt",
					match:   []*regexp.Regexp{regexp.MustCompile("'DROP' 'INDEX'")},
					inline:  []string{"opt_drop_behavior", "table_name_with_index_list", "table_name_with_index"},
					replace: map[string]string{"qualified_name": "table_name", "'@' unrestricted_name": "'@' index_name"},
					unlink:  []string{"table_name", "index_name"},
				},
				{name: "drop_stmt", inline: []string{"table_name_list", "any_name", "qualified_name_list", "qualified_name"}},
				{
					name:    "drop_table",
					stmt:    "drop_table_stmt",
					inline:  []string{"opt_drop_behavior", "table_name_list"},
					match:   []*regexp.Regexp{regexp.MustCompile("'DROP' 'TABLE'")},
					replace: map[string]string{"any_name": "table_name"},
					relink:  map[string]string{"table_name": "any_name"},
				},
				{
					name:    "drop_user_stmt",
					inline:  []string{"name_list"},
					replace: map[string]string{"name": "user_name"},
					unlink:  []string{"user_name"},
				},
				{
					name:    "drop_view",
					stmt:    "drop_view_stmt",
					inline:  []string{"opt_drop_behavior", "table_name_list"},
					match:   []*regexp.Regexp{regexp.MustCompile("'DROP' 'VIEW'")},
					replace: map[string]string{"any_name": "view_name"},
					relink:  map[string]string{"view_name": "any_name"},
				},
				{
					name:   "explain_stmt",
					inline: []string{"explain_option_list"},
					replace: map[string]string{
						"explain_option_name": "( | 'EXPRS' | 'METADATA' | 'QUALIFY' | 'VERBOSE' | 'TYPES' )",
					},
				},
				{name: "family_def", inline: []string{"opt_name", "name_list"}},
				{
					name:    "grant_stmt",
					inline:  []string{"privileges", "privilege_list", "privilege", "targets", "grantee_list", "table_pattern_list", "name_list"},
					replace: map[string]string{"table_pattern": "table_name", "'DATABASE' ( ( name ) ( ( ',' name": "'DATABASE' ( ( database_name ) ( ( ',' database_name", "'TO' ( ( name ) ( ( ',' name": "'TO' ( ( user_name ) ( ( ',' user_name"},
					unlink:  []string{"table_name", "database_name", "user_name"},
					nosplit: true,
				},
				{
					name:    "foreign_key_column_level",
					stmt:    "stmt_block",
					replace: map[string]string{"stmt_list": "'CREATE' 'TABLE' table_name '(' column_name column_type 'REFERENCES' parent_table ( '(' ref_column_name ')' | ) ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"},
					unlink:  []string{"table_name", "column_name", "column_type", "parent_table", "table_constraints"},
				},
				{
					name:    "foreign_key_table_level",
					stmt:    "stmt_block",
					replace: map[string]string{"stmt_list": "'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' name | ) 'FOREIGN KEY' '(' ( fk_column_name ( ',' fk_column_name )* ) ')' 'REFERENCES' parent_table ( '(' ( ref_column_name ( ',' ref_column_name )* ) ')' | ) ( table_constraints | ) ')'"},
					unlink:  []string{"table_name", "column_name", "parent_table", "table_constraints"},
				},
				{name: "index_def", inline: []string{"opt_storing", "storing", "index_params", "opt_name"}},
				{name: "import_table", stmt: "import_stmt"},
				{
					name:   "insert_stmt",
					inline: []string{"insert_target", "insert_rest", "returning_clause"},
					match:  []*regexp.Regexp{regexp.MustCompile("'INSERT'")},
				},
				{name: "iso_level"},
				{
					name:    "import_stmt",
					inline:  []string{"string_or_placeholder_list", "opt_with_options", "kv_option_list"},
					replace: map[string]string{"any_name": "table_name", "'USING' string_or_placeholder": "'USING' create_table_file", "string_or_placeholder": "file_to_import", "import_data_format": "'CSV'"},
					unlink:  []string{"table_name", "create_table_file", "file_to_import"},
					exclude: []*regexp.Regexp{regexp.MustCompile("'OPTIONS'")},
					match:   []*regexp.Regexp{regexp.MustCompile("'WITH'")},
				},
				{name: "interleave", stmt: "create_table_stmt", inline: []string{"opt_interleave"}, replace: map[string]string{"any_name": "table_name", "opt_table_elem_list": "table_definition", "name_list": "interleave_prefix", " name": " parent_table"}, unlink: []string{"table_name", "table_definition", "parent_table", "child_columns"}},
				{
					name:    "not_null_column_level",
					stmt:    "stmt_block",
					replace: map[string]string{"stmt_list": "'CREATE' 'TABLE' table_name '(' column_name column_type 'NOT NULL' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"},
					unlink:  []string{"table_name", "column_name", "column_type", "table_constraints"},
				},
				{name: "opt_interleave", replace: map[string]string{"name_list": "interleave_prefix"}, unlink: []string{"interleave_prefix"}},
				{name: "pause_job", stmt: "pause_stmt", replace: map[string]string{"a_expr": "job_id"}, unlink: []string{"job_id"}},
				{
					name:    "primary_key_column_level",
					stmt:    "stmt_block",
					replace: map[string]string{"stmt_list": "'CREATE' 'TABLE' table_name '(' column_name column_type 'PRIMARY KEY' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"},
					unlink:  []string{"table_name", "column_name", "column_type", "table_constraints"},
				},
				{
					name:    "primary_key_table_level",
					stmt:    "stmt_block",
					replace: map[string]string{"stmt_list": "'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' name | ) 'PRIMARY KEY' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')'"},
					unlink:  []string{"table_name", "column_name", "table_constraints"},
				},
				{name: "release_savepoint", stmt: "release_stmt", inline: []string{"savepoint_name"}},
				{name: "rename_column", stmt: "alter_rename_table_stmt", inline: []string{"opt_column"}, match: []*regexp.Regexp{regexp.MustCompile("'ALTER' 'TABLE' .* 'RENAME' ('COLUMN'|name)")}, replace: map[string]string{"relation_expr": "table_name", "name 'TO'": "current_name 'TO'"}, unlink: []string{"table_name", "current_name"}},
				{name: "rename_database", stmt: "alter_rename_database_stmt", match: []*regexp.Regexp{regexp.MustCompile("'ALTER' 'DATABASE'")}},
				{
					name:    "rename_index",
					stmt:    "alter_rename_index_stmt",
					match:   []*regexp.Regexp{regexp.MustCompile("'ALTER' 'INDEX'")},
					inline:  []string{"table_name_with_index"},
					replace: map[string]string{"qualified_name": "table_name", "'@' unrestricted_name": "'@' index_name"},
					unlink:  []string{"table_name", "index_name"},
				},
				{name: "rename_table", stmt: "alter_rename_table_stmt", match: []*regexp.Regexp{regexp.MustCompile("'ALTER' 'TABLE' .* 'RENAME' 'TO'")}, replace: map[string]string{"relation_expr": "current_name", "qualified_name": "new_name"}, unlink: []string{"current_name"}, relink: map[string]string{"new_name": "name"}},
				{name: "reset_session_stmt"},
				{name: "reset_csetting_stmt"},
				{
					name:    "restore",
					stmt:    "restore_stmt",
					inline:  []string{"targets", "table_pattern_list", "name_list", "opt_as_of_clause", "opt_with_options"},
					match:   []*regexp.Regexp{regexp.MustCompile("'RESTORE'")},
					exclude: []*regexp.Regexp{regexp.MustCompile("'RESTORE' 'DATABASE'")},
					replace: map[string]string{
						"non_reserved_word_or_sconst":            "destination",
						"'AS' 'OF' 'SYSTEM' 'TIME' a_expr_const": "",
						"string_or_placeholder_list":             "full_backup_location ( | incremental_backup_location ( ',' incremental_backup_location )*)",
						"'WITH' 'OPTIONS'":                       "'WITH OPTIONS'",
					},
					unlink: []string{"destination", "timestamp", "full_backup_location", "incremental_backup_location"},
				},
				{name: "resume_job", stmt: "resume_stmt", replace: map[string]string{"a_expr": "job_id"}, unlink: []string{"job_id"}},
				{
					name:   "revoke_stmt",
					inline: []string{"privileges", "privilege_list", "privilege", "targets", "grantee_list"},
					replace: map[string]string{
						"table_pattern_list":       "table_name ( ',' table_name )*",
						"name_list":                "database_name ( ',' database_name )*",
						"'FROM' name ( ( ',' name": "'FROM' user_name ( ( ',' user_name"},
					unlink: []string{"table_name", "database_name", "user_name"},
				},
				{
					name:    "rollback_transaction",
					stmt:    "rollback_stmt",
					inline:  []string{"opt_to_savepoint"},
					match:   []*regexp.Regexp{regexp.MustCompile("'ROLLBACK'")},
					replace: map[string]string{"'TRANSACTION'": "", "'TO'": "'TO' 'SAVEPOINT'", "savepoint_name": "cockroach_restart"},
					unlink:  []string{"cockroach_restart"},
				},
				{name: "savepoint_stmt", inline: []string{"savepoint_name"}},
				{
					name:   "select_stmt",
					inline: []string{"select_no_parens", "simple_select", "opt_sort_clause", "select_limit", "opt_all_clause", "distinct_clause", "target_list", "from_clause", "where_clause", "group_clause", "having_clause"},
					replace: map[string]string{"'SELECT' ( 'ALL' |  ) ( target_elem ( ',' target_elem )* ) ( 'FROM' from_list opt_as_of_clause |  ) ( 'WHERE' a_expr |  ) ( 'GROUP' 'BY' expr_list |  ) ( 'HAVING' a_expr |  ) window_clause | 'SELECT' ( 'DISTINCT' ) ( target_elem ( ',' target_elem )* ) ( 'FROM' from_list opt_as_of_clause |  ) ( 'WHERE' a_expr |  ) ( 'GROUP' 'BY' expr_list |  ) ( 'HAVING' a_expr |  ) window_clause | values_clause | 'TABLE' relation_expr | ": "",
						"select_clause 'UNION' all_or_distinct select_clause | select_clause 'INTERSECT' all_or_distinct select_clause | select_clause 'EXCEPT' all_or_distinct select_clause": "select_clause ( | ( ( 'UNION' | 'INTERSECT' | 'EXCEPT' ) all_or_distinct select_clause ) )",
						"select_clause sort_clause | select_clause ( sort_clause |  ) ( limit_clause offset_clause | offset_clause limit_clause | limit_clause | offset_clause )":              "select_clause ( 'ORDER BY' sortby_list | ) ( 'LIMIT' limit_val | ) ( 'OFFSET' offset_val | )",
						"all_or_distinct select_clause": "all_or_distinct 'SELECT ...'",
						"| select_with_parens":          "",
						"all_or_distinct":               "( 'ALL' | )",
						"select_clause":                 "'SELECT' ( 'DISTINCT' | ) ( target_elem ('AS' col_label | ) ( ',' target_elem ('AS' col_label | ) )* ) 'FROM' ( table_ref ( '@' index_name | ) ( ',' table_ref ( '@' index_name | ) )* ) ('AS OF SYSTEM TIME' timestamp | ) ( 'WHERE' a_expr |  ) ( 'GROUP BY' expr_list ( 'HAVING' a_expr |  ) |  ) "},
					unlink:  []string{"index_name"},
					nosplit: true,
				},
				{
					name:   "set_var",
					stmt:   "set_session_stmt",
					inline: []string{"set_session_stmt", "set_rest_more", "generic_set", "var_list"},
					exclude: []*regexp.Regexp{
						regexp.MustCompile("'SET' . 'TRANSACTION'"),
						regexp.MustCompile("'SET' 'TRANSACTION'"),
						regexp.MustCompile("'SET' 'SESSION' var_name"),
						regexp.MustCompile("'SET' 'SESSION' 'TRANSACTION'"),
						regexp.MustCompile("'SET' 'SESSION' 'CHARACTERISTICS'"),
						regexp.MustCompile("'SET' 'CLUSTER'"),
					},
					replace: map[string]string{
						"'=' 'DEFAULT'":  "'=' 'DEFAULT' | 'SET' 'TIME' 'ZONE' ( var_value | 'DEFAULT' | 'LOCAL' )",
						"'SET' var_name": "'SET' ( 'SESSION' | ) var_name",
					},
				},
				{
					name:   "set_cluster_setting",
					stmt:   "set_csetting_stmt",
					inline: []string{"var_value", "ctext_expr"},
					replace: map[string]string{
						"a_expr": "var_value",
						"| 'SET' 'CLUSTER' 'SETTING' var_name '=' 'ON'":  "",
						"| 'SET' 'CLUSTER' 'SETTING' var_name 'TO' 'ON'": "",
					},
				},
				{
					name:   "set_transaction",
					stmt:   "set_transaction_stmt",
					inline: []string{"transaction_mode_list", "transaction_mode", "transaction_iso_level", "transaction_user_priority", "transaction_read_mode", "iso_level", "user_priority", "opt_comma"},
					// match: []*regexp.Regexp{regexp.MustCompile("'SET' 'TRANSACTION'")},
					// exclude: []*regexp.Regexp{regexp.MustCompile("'READ'")},
					// replace: map[string]string{"'ISOLATION' 'LEVEL'": "'ISOLATION LEVEL'"},
				},
				{
					name:   "show_var",
					stmt:   "show_session_stmt",
					inline: []string{"session_var"},
					exclude: []*regexp.Regexp{
						regexp.MustCompile("'DATABASE'"),
						regexp.MustCompile("'NAMES'"),
						regexp.MustCompile("'SESSION_USER"),
						regexp.MustCompile("'TIME' 'ZONE'"),
					},
					replace: map[string]string{"identifier": "var_name"},
				},
				{
					name: "show_cluster_setting",
					stmt: "show_csettings_stmt",
					// match: []*regexp.Regexp{
					// 	regexp.MustCompile("'SHOW'.* 'CLUSTER'"),
					// },
					// exclude: []*regexp.Regexp{
					// 	regexp.MustCompile("'SHOW' 'CLUSTER' 'SETTING' 'ALL'"),
					// },
				},
				{
					name:    "show_columns",
					stmt:    "show_columns_stmt",
					replace: map[string]string{"var_name": "table_name"},
					unlink:  []string{"table_name"},
				},
				{
					name:    "show_constraints",
					stmt:    "show_constraints_stmt",
					replace: map[string]string{"var_name": "table_name"},
					unlink:  []string{"table_name"},
				},
				{
					name:    "show_create_table",
					stmt:    "show_create_table_stmt",
					replace: map[string]string{"var_name": "table_name"},
					unlink:  []string{"table_name"},
				},
				{
					name:    "show_create_view",
					stmt:    "show_create_view_stmt",
					replace: map[string]string{"var_name": "view_name"},
					unlink:  []string{"view_name"},
				},
				{
					name: "show_databases",
					stmt: "show_databases_stmt",
				},
				{
					name:    "show_backup",
					stmt:    "show_backup_stmt",
					match:   []*regexp.Regexp{regexp.MustCompile("'SHOW' 'BACKUP'")},
					replace: map[string]string{"string_or_placeholder": "location"},
					unlink:  []string{"location"},
				},
				{
					name:   "show_grants",
					stmt:   "show_grants_stmt",
					inline: []string{"on_privilege_target_clause", "targets", "for_grantee_clause", "grantee_list", "table_pattern_list", "name_list"},
					match:  []*regexp.Regexp{regexp.MustCompile("'SHOW' 'GRANTS'")},
					replace: map[string]string{
						"table_pattern":                "table_name",
						"'DATABASE' name ( ( ',' name": "'DATABASE' database_name ( ( ',' database_name",
						"'FOR' name ( ( ',' name":      "'FOR' user_name ( ( ',' user_name",
					},
					unlink: []string{"table_name", "database_name", "user_name"},
				},
				{
					name:    "show_index",
					stmt:    "show_indexes_stmt",
					replace: map[string]string{"var_name": "table_name"},
					unlink:  []string{"table_name"},
				},
				{name: "show_jobs", stmt: "show_jobs_stmt"},
				{name: "show_keys", stmt: "show_stmt", match: []*regexp.Regexp{regexp.MustCompile("'SHOW' 'KEYS'")}},
				{name: "show_queries", stmt: "show_queries_stmt"},
				{name: "show_sessions", stmt: "show_sessions_stmt"},
				{name: "show_tables", stmt: "show_tables_stmt"},
				{name: "show_trace", stmt: "show_trace_stmt"},
				{name: "show_transaction", stmt: "show_stmt", match: []*regexp.Regexp{regexp.MustCompile("'SHOW' 'TRANSACTION'")}},
				{name: "show_users", stmt: "show_users_stmt"},
				{name: "table_constraint", inline: []string{"constraint_elem", "opt_storing", "storing"}},
				{
					name:    "truncate_stmt",
					inline:  []string{"opt_table", "relation_expr_list", "opt_drop_behavior"},
					replace: map[string]string{"relation_expr": "table_name"},
					unlink:  []string{"table_name"},
					//inline:  []string{"opt_table", "relation_expr_list", "relation_expr", "opt_drop_behavior"},
					//replace: map[string]string{"'ONLY' '(' qualified_name ')'": "", "'ONLY' qualified_name": "", "qualified_name": "table_name", "'*'": "", "'CASCADE'": "", "'RESTRICT'": ""},
					//unlink:  []string{"table_name"},
				},
				{
					name:    "unique_column_level",
					stmt:    "stmt_block",
					replace: map[string]string{"stmt_list": "'CREATE' 'TABLE' table_name '(' column_name column_type 'UNIQUE' ( column_constraints | ) ( ',' ( column_def ( ',' column_def )* ) | ) ( table_constraints | ) ')' ')'"},
					unlink:  []string{"table_name", "column_name", "column_type", "table_constraints"},
				},
				{
					name:    "unique_table_level",
					stmt:    "stmt_block",
					replace: map[string]string{"stmt_list": "'CREATE' 'TABLE' table_name '(' ( column_def ( ',' column_def )* ) ( 'CONSTRAINT' name | ) 'UNIQUE' '(' ( column_name ( ',' column_name )* ) ')' ( table_constraints | ) ')'"},
					unlink:  []string{"table_name", "check_expr", "table_constraints"},
				},
				{
					name:    "update_stmt",
					inline:  []string{"relation_expr_opt_alias", "set_clause_list", "set_clause", "single_set_clause", "multiple_set_clause", "ctext_row", "ctext_expr_list", "ctext_expr", "from_clause", "from_list", "where_clause", "returning_clause"},
					replace: map[string]string{"relation_expr": "table_name", "qualified_name": "column_name", "qualified_name_list": "column_name_list"},
					relink:  map[string]string{"table_name": "relation_expr", "column_name": "qualified_name", "column_name_list": "qualified_name_list"},
					nosplit: true,
				},
				{
					name:   "upsert_stmt",
					inline: []string{"insert_target", "insert_rest", "returning_clause"},
					match:  []*regexp.Regexp{regexp.MustCompile("'UPSERT'")},
				},
				{
					name:    "validate_constraint",
					stmt:    "alter_onetable_stmt",
					replace: map[string]string{"alter_table_cmds": "'VALIDATE' 'CONSTRAINT' constraint_name", "relation_expr": "table_name"},
					unlink:  []string{"constraint_name", "table_name"},
				},
			}

			sem := make(chan struct{}, maxWorkers) // max number of concurrent workers
			for _, s := range specs {
				if !filterRE.MatchString(s.name) == !invertMatch {
					continue
				}
				wg.Add(1)
				sem <- struct{}{}
				go func(s stmtSpec) {
					defer wg.Done()
					defer func() { <-sem }()
					if s.stmt == "" {
						s.stmt = s.name
					}
					fmt.Fprintf(os.Stderr, "processing statement %q...\n", s.stmt)
					g, err := runParse(br(), s.inline, s.stmt, false, s.nosplit, s.match, s.exclude)
					if err != nil {
						log.Fatalf("%s: %s\n%s", s.name, err, g)
					}
					if printBNF {
						fmt.Printf("%s: (PRE REPLACE)\n\n%s\n", s.name, g)
					}
					for from, to := range s.replace {
						g = bytes.Replace(g, []byte(from), []byte(to), -1)
					}
					for from, to := range s.regreplace {
						re := regexp.MustCompile(from)
						g = re.ReplaceAll(g, []byte(to))
					}
					if printBNF {
						fmt.Printf("%s: (POST REPLACE)\n\n%s\n", s.name, g)
						return
					}
					rr, err := runRR(s.name, bytes.NewReader(g), railroadJar)
					if err != nil {
						log.Fatalf("%s: %s\n%s", s.name, err, g)
					}
					body, err := extract.ExtractTag(bytes.NewReader(rr), "svg")
					if err != nil {
						log.Fatal(s.name, err)
					}
					body = strings.Replace(body, `<a xlink:href="#`, `<a xlink:href="sql-grammar.html#`, -1)
					name := strings.Replace(s.name, "_stmt", "", 1)
					for _, u := range s.unlink {
						s := fmt.Sprintf(`<a xlink:href="sql-grammar.html#%s" xlink:title="%s">((?s).*?)</a>`, u, u)
						link := regexp.MustCompile(s)
						body = link.ReplaceAllString(body, "$1")
					}
					for from, to := range s.relink {
						replaceFrom := fmt.Sprintf(`<a xlink:href="sql-grammar.html#%s" xlink:title="%s">`, from, from)
						replaceTo := fmt.Sprintf(`<a xlink:href="sql-grammar.html#%s" xlink:title="%s">`, to, to)
						body = strings.Replace(body, replaceFrom, replaceTo, -1)
					}
					// Wrap the output in a <div> so that the Markdown parser
					// doesn't attempt to parse the inside of the contained
					// <svg> as Markdown.
					body = fmt.Sprintf(`<div>%s</div>`, body)
					if err := ioutil.WriteFile(filepath.Join(baseDir, fmt.Sprintf("%s.html", name)), []byte(body), 0644); err != nil {
						log.Fatal(s.name, err)
					}
				}(s)
			}
			wg.Wait()
		},
	}

	rootCmd.Flags().IntVar(&maxWorkers, "max-workers", 1, "maximum number of concurrent workers")
	rootCmd.Flags().StringVar(&addr, "addr", "https://raw.githubusercontent.com/cockroachdb/cockroach/master/pkg/sql/parser/sql.y", "Location of sql.y file. Can also specify a local file.")
	rootCmd.Flags().StringVar(&baseDir, "base", filepath.Join("..", "_includes", "sql", "v2.0", "diagrams"), "Base directory for html output.")
	rootCmd.Flags().StringVar(&filter, "filter", ".*", "Filter statement names (regular expression)")
	rootCmd.Flags().BoolVar(&invertMatch, "invert-match", false, "Generate everything that doesn't match the filter")
	rootCmd.Flags().BoolVar(&printBNF, "bnf", false, "Print BNF only; don't generate railroad diagrams")
	rootCmd.Flags().StringVar(&railroadJar, "railroad", "Railroad.jar", "Location of Railroad.jar; empty to use website")

	rootCmd.AddCommand(cmdBNF, cmdParse, cmdRR, cmdBody)
	if cmdFuncs != nil {
		rootCmd.AddCommand(cmdFuncs)
	}
	rootCmd.PersistentFlags().StringVar(&outputPath, "out", "", "Output path; stdout if empty.")
	rootCmd.PersistentFlags().StringVar(&inputPath, "in", "", "Input path; stdin if empty.")
	rootCmd.Execute()
}

type stmtSpec struct {
	name           string
	stmt           string // if unspecified, uses name
	inline         []string
	replace        map[string]string
	regreplace     map[string]string
	match, exclude []*regexp.Regexp
	unlink         []string
	relink         map[string]string
	nosplit        bool
}

func runBNF(addr string) ([]byte, error) {
	log.Printf("generate BNF: %s", addr)
	return extract.GenerateBNF(addr)
}

func runParse(r io.Reader, inline []string, topStmt string, descend, nosplit bool, match, exclude []*regexp.Regexp) ([]byte, error) {
	log.Printf("parse: %s, inline: %s, descend: %v", topStmt, inline, descend)
	g, err := extract.ParseGrammar(r)
	if err != nil {
		log.Fatal(err)
	}
	if err := g.Inline(inline...); err != nil {
		log.Fatal(err)
	}
	b, err := g.ExtractProduction(topStmt, descend, nosplit, match, exclude)
	b = bytes.Replace(b, []byte("IDENT"), []byte("identifier"), -1)
	b = bytes.Replace(b, []byte("_LA"), []byte(""), -1)
	return b, err
}

func runRR(name string, r io.Reader, railroadJar string) ([]byte, error) {
	b, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}
	var html []byte
	if railroadJar == "" {
		html, err = extract.GenerateRRNet(b)
	} else {
		html, err = extract.GenerateRRJar(railroadJar, b)
	}
	if err != nil {
		return nil, err
	}
	log.Printf("%s: generated railroad diagram", name)
	s, err := extract.XHTMLtoHTML(bytes.NewReader(html))
	return []byte(s), err
}