indexserver: add debug subcommand

This adds a subcommand "debug" to indexserver.

The new subcommand contains all debug commands that were previously
handled as flags. This means that this commit changes the API but only for
debug command. There is no change in how we start indexserver.

The debug commands now live in a separate file "debug.go" and don't polute the
defintion of the server anymore. I used the vistor pattern as described
here peterbourgon/ff#76 to reuse the root
flags for some of the debug commands.

Other changes:
- removed deprecated flag exp-git-index
- removed second argument of setupTmpDir because it was not used
- sorted imports in main.go

How to review:
The diff is actually more readable than it appears at first glance. I
recommend to start at the bottom of main.go. Most of the changes in
main.go come from deleting things relating to debug commands and
changing the parameters from pointers to values.

Test Plan:
- I ran a local instance of Sourcegraph with a verion of Zoekt based on
this PR. Start up and indexing were successful.
- I built a binary for zoekt-sourcegraph-indexserver, copied it to a
production instance and tried out all debug commands

cmd/zoekt-sourcegraph-indexserver/debug.go

@@ -0,0 +1,196 @@
+// This file contains commands which run in a non daemon mode for testing/debugging.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+
+	"github.com/peterbourgon/ff/v3/ffcli"
+
+	"github.com/google/zoekt/build"
+)
+
+func debugFind() *ffcli.Command {
+	fs := flag.NewFlagSet("debug find", flag.ExitOnError)
+	debugFindIndex := fs.String("index", getEnvWithDefaultString("DATA_DIR", build.DefaultDir), "set index directory to use")
+
+	return &ffcli.Command{
+		Name:       "find",
+		ShortUsage: "find [flags] <repository name>",
+		ShortHelp:  "find a shard by repo name",
+		FlagSet:    fs,
+		Exec: func(ctx context.Context, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("missing repository name")
+			}
+			ia := indexArgs{
+				IndexOptions: IndexOptions{
+					Name: args[0],
+				},
+				IndexDir: *debugFindIndex,
+			}
+			bo := ia.BuildOptions()
+			for _, s := range bo.FindAllShards() {
+				fmt.Println(s)
+			}
+			return nil
+		},
+	}
+}
+
+func debugIndex() *ffcli.Command {
+	fs := flag.NewFlagSet("debug index", flag.ExitOnError)
+	conf := rootConfig{}
+	conf.registerRootFlags(fs)
+
+	return &ffcli.Command{
+		Name:       "index",
+		ShortUsage: "index [flags] <repository ID>",
+		ShortHelp:  "index a repository",
+		FlagSet:    fs,
+		Exec: func(ctx context.Context, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("missing repository ID")
+			}
+			s, err := newServer(conf)
+			if err != nil {
+				return err
+			}
+			id, err := strconv.Atoi(args[0])
+			if err != nil {
+				return err
+			}
+			msg, err := s.forceIndex(uint32(id))
+			log.Println(msg)
+			if err != nil {
+				return err
+			}
+			return nil
+		},
+	}
+}
+
+func debugTrigrams() *ffcli.Command {
+	return &ffcli.Command{
+		Name:       "trigrams",
+		ShortUsage: "trigrams <path/to/shard>",
+		ShortHelp:  "list all the trigrams in a shard",
+		Exec: func(ctx context.Context, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("missing path to shard")
+			}
+			return printShardStats(args[0])
+		},
+	}
+}
+
+func debugMeta() *ffcli.Command {
+	return &ffcli.Command{
+		Name:       "meta",
+		ShortUsage: "meta <path/to/shard>",
+		ShortHelp:  "output index and repo metadata",
+		Exec: func(ctx context.Context, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("missing path to shard")
+			}
+			return printMetaData(args[0])
+		},
+	}
+}
+
+func debugMerge() *ffcli.Command {
+	fs := flag.NewFlagSet("debug merge", flag.ExitOnError)
+	simulate := fs.Bool("simulate", false, "if set, merging will be simulated")
+	targetSize := fs.Int64("merge_target_size", getEnvWithDefaultInt64("SRC_TARGET_SIZE", 2000), "the target size of compound shards in MiB")
+	index := fs.String("index", getEnvWithDefaultString("DATA_DIR", build.DefaultDir), "set index directory to use")
+	dbg := fs.Bool("debug", srcLogLevelIsDebug(), "turn on more verbose logging.")
+
+	return &ffcli.Command{
+		Name:       "merge",
+		FlagSet:    fs,
+		ShortUsage: "merge [flags] <dir>",
+		ShortHelp:  "run a full merge operation inside dir",
+		Exec: func(ctx context.Context, args []string) error {
+			if *dbg {
+				debug = log.New(os.Stderr, "", log.LstdFlags)
+			}
+			return doMerge(*index, *targetSize*1024*1024, *simulate)
+		},
+	}
+}
+
+func debugList() *ffcli.Command {
+	fs := flag.NewFlagSet("debug list", flag.ExitOnError)
+	conf := rootConfig{}
+	conf.registerRootFlags(fs)
+
+	return &ffcli.Command{
+		Name:       "list",
+		ShortUsage: "list [flags]",
+		ShortHelp:  "list the repositories that are OWNED by this indexserver",
+		FlagSet:    fs,
+		Exec: func(ctx context.Context, args []string) error {
+			s, err := newServer(conf)
+			if err != nil {
+				return err
+			}
+			repos, err := s.Sourcegraph.List(context.Background(), listIndexed(s.IndexDir))
+			if err != nil {
+				return err
+			}
+			for _, r := range repos.IDs {
+				fmt.Println(r)
+			}
+			return nil
+		},
+	}
+}
+
+func debugListIndexed() *ffcli.Command {
+	fs := flag.NewFlagSet("debug list-indexed", flag.ExitOnError)
+	conf := rootConfig{}
+	conf.registerRootFlags(fs)
+
+	return &ffcli.Command{
+		Name:       "list-indexed",
+		ShortUsage: "list-indexed [flags]",
+		ShortHelp:  "list the repositories that are INDEXED by this indexserver",
+		FlagSet:    fs,
+		Exec: func(ctx context.Context, args []string) error {
+			s, err := newServer(conf)
+			if err != nil {
+				return err
+			}
+			indexed := listIndexed(s.IndexDir)
+			for _, r := range indexed {
+				fmt.Println(r)
+			}
+			return nil
+		},
+	}
+}
+
+func debugCmd() *ffcli.Command {
+	fs := flag.NewFlagSet("debug", flag.ExitOnError)
+
+	return &ffcli.Command{
+		Name:       "debug",
+		ShortUsage: "debug <subcommand>",
+		ShortHelp:  "a set of commands for debugging and testing",
+		FlagSet:    fs,
+		Subcommands: []*ffcli.Command{
+			debugFind(),
+			debugIndex(),
+			debugList(),
+			debugListIndexed(),
+			debugMerge(),
+			debugMeta(),
+			debugTrigrams(),
+		},
+	}
+}