[MM-38708] Add import validator (mattermost#552)

commands/import.go

@@ -8,13 +8,16 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
+	"text/template"
 	"time"
 
-	"github.com/mattermost/mmctl/v6/client"
-	"github.com/mattermost/mmctl/v6/printer"
-
 	"github.com/mattermost/mattermost-server/v6/model"
 	"github.com/spf13/cobra"
+
+	"github.com/mattermost/mmctl/v6/client"
+	"github.com/mattermost/mmctl/v6/commands/importer"
+	"github.com/mattermost/mmctl/v6/printer"
 )
 
 var ImportCmd = &cobra.Command{
@@ -83,6 +86,14 @@ var ImportProcessCmd = &cobra.Command{
 	RunE:    withClient(importProcessCmdF),
 }
 
+var ImportValidateCmd = &cobra.Command{
+	Use:     "validate [filepath]",
+	Example: "  import validate import_file.zip --team myteam --team myotherteam",
+	Short:   "Validate an import file",
+	Args:    cobra.ExactArgs(1),
+	RunE:    importValidateCmdF,
+}
+
 func init() {
 	ImportUploadCmd.Flags().Bool("resume", false, "Set to true to resume an incomplete import upload.")
 	ImportUploadCmd.Flags().String("upload", "", "The ID of the import upload to resume.")
@@ -91,6 +102,10 @@ func init() {
 	ImportJobListCmd.Flags().Int("per-page", 200, "Number of import jobs to be fetched")
 	ImportJobListCmd.Flags().Bool("all", false, "Fetch all import jobs. --page flag will be ignore if provided")
 
+	ImportValidateCmd.Flags().StringArray("team", nil, "Predefined team[s] to assume as already present on the destination server. Implies --check-missing-teams. The flag can be repeated")
+	ImportValidateCmd.Flags().Bool("check-missing-teams", false, "Check for teams that are not defined but referenced in the archive")
+	ImportValidateCmd.Flags().Bool("ignore-attachments", false, "Don't check if the attached files are present in the archive")
+
 	ImportListCmd.AddCommand(
 		ImportListAvailableCmd,
 		ImportListIncompleteCmd,
@@ -104,6 +119,7 @@ func init() {
 		ImportListCmd,
 		ImportProcessCmd,
 		ImportJobCmd,
+		ImportValidateCmd,
 	)
 	RootCmd.AddCommand(ImportCmd)
 }
@@ -314,3 +330,114 @@ func jobListCmdF(c client.Client, command *cobra.Command, jobType string) error
 func importJobListCmdF(c client.Client, command *cobra.Command, args []string) error {
 	return jobListCmdF(c, command, model.JobTypeImportProcess)
 }
+
+type Statistics struct {
+	Schemes        int `json:"schemes"`
+	Teams          int `json:"teams"`
+	Channels       int `json:"channels"`
+	Users          int `json:"users"`
+	Emojis         int `json:"emojis"`
+	Posts          int `json:"posts"`
+	DirectChannels int `json:"direct_channels"`
+	DirectPosts    int `json:"direct_posts"`
+	Attachments    int `json:"attachments"`
+}
+
+func importValidateCmdF(command *cobra.Command, args []string) error {
+	configurePrinter()
+	defer printer.Print("Validation complete\n")
+
+	injectedTeams, err := command.Flags().GetStringArray("team")
+	if err != nil {
+		return err
+	}
+
+	checkMissingTeams, err := command.Flags().GetBool("check-missing-teams")
+	if err != nil {
+		return err
+	}
+
+	ignoreAttachments, err := command.Flags().GetBool("ignore-attachments")
+	if err != nil {
+		return err
+	}
+
+	createMissingTeams := !checkMissingTeams && len(injectedTeams) == 0
+	validator := importer.NewValidator(args[0], ignoreAttachments, createMissingTeams)
+
+	for _, team := range injectedTeams {
+		validator.InjectTeam(team)
+	}
+
+	templateError := template.Must(template.New("").Parse("{{ .Error }}\n"))
+	validator.OnError(func(ive *importer.ImportValidationError) error {
+		printer.PrintPreparedT(templateError, ive)
+		return nil
+	})
+
+	err = validator.Validate()
+	if err != nil {
+		return err
+	}
+
+	teams := validator.Teams()
+
+	stat := Statistics{
+		Schemes:        len(validator.Schemes()),
+		Teams:          len(teams),
+		Channels:       len(validator.Channels()),
+		Users:          len(validator.Users()),
+		Posts:          int(validator.PostCount()),
+		DirectChannels: int(validator.DirectChannelCount()),
+		DirectPosts:    int(validator.DirectPostCount()),
+		Emojis:         len(validator.Emojis()),
+		Attachments:    len(validator.Attachments()),
+	}
+
+	printStatistics(stat)
+
+	if createMissingTeams && len(teams) != 0 {
+		printer.PrintT("Automatically created teams: {{ join .CreatedTeams \", \" }}\n", struct {
+			CreatedTeams []string `json:"created_teams"`
+		}{teams})
+	}
+
+	unusedAttachments := validator.UnusedAttachments()
+	if len(unusedAttachments) > 0 {
+		printer.PrintT("Unused Attachments ({{ len .UnusedAttachments }}):\n"+
+			"{{ range .UnusedAttachments }}  {{ . }}\n{{ end }}", struct {
+			UnusedAttachments []string `json:"unused_attachments"`
+		}{unusedAttachments})
+	}
+
+	printer.PrintT("It took {{ .Elapsed }} to validate {{ .TotalLines }} lines in {{ .FileName }}\n", struct {
+		FileName   string        `json:"file_name"`
+		TotalLines uint64        `json:"total_lines"`
+		Elapsed    time.Duration `json:"elapsed_time_ns"`
+	}{args[0], validator.Lines(), validator.Duration()})
+
+	return nil
+}
+
+func configurePrinter() {
+	// we want to manage the newlines ourselves
+	printer.SetNoNewline(true)
+
+	// define a join function
+	printer.SetTemplateFunc("join", strings.Join)
+}
+
+func printStatistics(stat Statistics) {
+	tmpl := "\n" +
+		"Schemes         {{ .Schemes }}\n" +
+		"Teams           {{ .Teams }}\n" +
+		"Channels        {{ .Channels }}\n" +
+		"Users           {{ .Users }}\n" +
+		"Emojis          {{ .Emojis }}\n" +
+		"Posts           {{ .Posts }}\n" +
+		"Direct Channels {{ .DirectChannels }}\n" +
+		"Direct Posts    {{ .DirectPosts }}\n" +
+		"Attachments     {{ .Attachments }}\n"
+
+	printer.PrintT(tmpl, stat)
+}

commands/importer/types.go

@@ -0,0 +1,219 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package importer
+
+import (
+	"archive/zip"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+// Import Data Models
+
+type LineImportData struct { //nolint:govet
+	Type          string                   `json:"type"`
+	Scheme        *SchemeImportData        `json:"scheme,omitempty"`
+	Team          *TeamImportData          `json:"team,omitempty"`
+	Channel       *ChannelImportData       `json:"channel,omitempty"`
+	User          *UserImportData          `json:"user,omitempty"`
+	Post          *PostImportData          `json:"post,omitempty"`
+	DirectChannel *DirectChannelImportData `json:"direct_channel,omitempty"`
+	DirectPost    *DirectPostImportData    `json:"direct_post,omitempty"`
+	Emoji         *EmojiImportData         `json:"emoji,omitempty"`
+	Version       *int                     `json:"version,omitempty"`
+}
+
+type TeamImportData struct {
+	Name            *string `json:"name"`
+	DisplayName     *string `json:"display_name"`
+	Type            *string `json:"type"`
+	Description     *string `json:"description,omitempty"`
+	AllowOpenInvite *bool   `json:"allow_open_invite,omitempty"`
+	Scheme          *string `json:"scheme,omitempty"`
+}
+
+type ChannelImportData struct {
+	Team        *string            `json:"team"`
+	Name        *string            `json:"name"`
+	DisplayName *string            `json:"display_name"`
+	Type        *model.ChannelType `json:"type"`
+	Header      *string            `json:"header,omitempty"`
+	Purpose     *string            `json:"purpose,omitempty"`
+	Scheme      *string            `json:"scheme,omitempty"`
+}
+
+type UserImportData struct {
+	ProfileImage       *string   `json:"profile_image,omitempty"`
+	ProfileImageData   *zip.File `json:"-"`
+	Username           *string   `json:"username"`
+	Email              *string   `json:"email"`
+	AuthService        *string   `json:"auth_service"`
+	AuthData           *string   `json:"auth_data,omitempty"`
+	Password           *string   `json:"password,omitempty"`
+	Nickname           *string   `json:"nickname"`
+	FirstName          *string   `json:"first_name"`
+	LastName           *string   `json:"last_name"`
+	Position           *string   `json:"position"`
+	Roles              *string   `json:"roles"`
+	Locale             *string   `json:"locale"`
+	UseMarkdownPreview *string   `json:"feature_enabled_markdown_preview,omitempty"`
+	UseFormatting      *string   `json:"formatting,omitempty"`
+	ShowUnreadSection  *string   `json:"show_unread_section,omitempty"`
+	DeleteAt           *int64    `json:"delete_at,omitempty"`
+
+	Teams *[]UserTeamImportData `json:"teams,omitempty"`
+
+	Theme               *string `json:"theme,omitempty"`
+	UseMilitaryTime     *string `json:"military_time,omitempty"`
+	CollapsePreviews    *string `json:"link_previews,omitempty"`
+	MessageDisplay      *string `json:"message_display,omitempty"`
+	CollapseConsecutive *string `json:"collapse_consecutive_messages,omitempty"`
+	ColorizeUsernames   *string `json:"colorize_usernames,omitempty"`
+	ChannelDisplayMode  *string `json:"channel_display_mode,omitempty"`
+	TutorialStep        *string `json:"tutorial_step,omitempty"`
+	EmailInterval       *string `json:"email_interval,omitempty"`
+
+	NotifyProps *UserNotifyPropsImportData `json:"notify_props,omitempty"`
+}
+
+type UserNotifyPropsImportData struct {
+	Desktop      *string `json:"desktop"`
+	DesktopSound *string `json:"desktop_sound"`
+
+	Email *string `json:"email"`
+
+	Mobile           *string `json:"mobile"`
+	MobilePushStatus *string `json:"mobile_push_status"`
+
+	ChannelTrigger  *string `json:"channel"`
+	CommentsTrigger *string `json:"comments"`
+	MentionKeys     *string `json:"mention_keys"`
+}
+
+type UserTeamImportData struct {
+	Name     *string                  `json:"name"`
+	Roles    *string                  `json:"roles"`
+	Theme    *string                  `json:"theme,omitempty"`
+	Channels *[]UserChannelImportData `json:"channels,omitempty"`
+}
+
+type UserChannelImportData struct {
+	Name        *string                           `json:"name"`
+	Roles       *string                           `json:"roles"`
+	NotifyProps *UserChannelNotifyPropsImportData `json:"notify_props,omitempty"`
+	Favorite    *bool                             `json:"favorite,omitempty"`
+}
+
+type UserChannelNotifyPropsImportData struct {
+	Desktop    *string `json:"desktop"`
+	Mobile     *string `json:"mobile"`
+	MarkUnread *string `json:"mark_unread"`
+}
+
+type EmojiImportData struct {
+	Name  *string   `json:"name"`
+	Image *string   `json:"image"`
+	Data  *zip.File `json:"-"`
+}
+
+type ReactionImportData struct {
+	User      *string `json:"user"`
+	CreateAt  *int64  `json:"create_at"`
+	EmojiName *string `json:"emoji_name"`
+}
+
+type ReplyImportData struct {
+	User *string `json:"user"`
+
+	Type     *string `json:"type"`
+	Message  *string `json:"message"`
+	CreateAt *int64  `json:"create_at"`
+	EditAt   *int64  `json:"edit_at"`
+
+	FlaggedBy   *[]string               `json:"flagged_by,omitempty"`
+	Reactions   *[]ReactionImportData   `json:"reactions,omitempty"`
+	Attachments *[]AttachmentImportData `json:"attachments,omitempty"`
+}
+
+type PostImportData struct {
+	Team    *string `json:"team"`
+	Channel *string `json:"channel"`
+	User    *string `json:"user"`
+
+	Type     *string                `json:"type"`
+	Message  *string                `json:"message"`
+	Props    *model.StringInterface `json:"props"`
+	CreateAt *int64                 `json:"create_at"`
+	EditAt   *int64                 `json:"edit_at"`
+
+	FlaggedBy   *[]string               `json:"flagged_by,omitempty"`
+	Reactions   *[]ReactionImportData   `json:"reactions,omitempty"`
+	Replies     *[]ReplyImportData      `json:"replies,omitempty"`
+	Attachments *[]AttachmentImportData `json:"attachments,omitempty"`
+	IsPinned    *bool                   `json:"is_pinned,omitempty"`
+}
+
+type DirectChannelImportData struct {
+	Members     *[]string `json:"members"`
+	FavoritedBy *[]string `json:"favorited_by"`
+
+	Header *string `json:"header"`
+}
+
+type DirectPostImportData struct {
+	ChannelMembers *[]string `json:"channel_members"`
+	User           *string   `json:"user"`
+
+	Type     *string                `json:"type"`
+	Message  *string                `json:"message"`
+	Props    *model.StringInterface `json:"props"`
+	CreateAt *int64                 `json:"create_at"`
+	EditAt   *int64                 `json:"edit_at"`
+
+	FlaggedBy   *[]string               `json:"flagged_by"`
+	Reactions   *[]ReactionImportData   `json:"reactions"`
+	Replies     *[]ReplyImportData      `json:"replies"`
+	Attachments *[]AttachmentImportData `json:"attachments"`
+	IsPinned    *bool                   `json:"is_pinned,omitempty"`
+}
+
+type SchemeImportData struct {
+	Name                    *string         `json:"name"`
+	DisplayName             *string         `json:"display_name"`
+	Description             *string         `json:"description"`
+	Scope                   *string         `json:"scope"`
+	DefaultTeamAdminRole    *RoleImportData `json:"default_team_admin_role"`
+	DefaultTeamUserRole     *RoleImportData `json:"default_team_user_role"`
+	DefaultChannelAdminRole *RoleImportData `json:"default_channel_admin_role"`
+	DefaultChannelUserRole  *RoleImportData `json:"default_channel_user_role"`
+	DefaultTeamGuestRole    *RoleImportData `json:"default_team_guest_role"`
+	DefaultChannelGuestRole *RoleImportData `json:"default_channel_guest_role"`
+}
+
+type RoleImportData struct {
+	Name        *string   `json:"name"`
+	DisplayName *string   `json:"display_name"`
+	Description *string   `json:"description"`
+	Permissions *[]string `json:"permissions"`
+}
+
+type LineImportWorkerData struct {
+	LineImportData
+	LineNumber int
+}
+
+type LineImportWorkerError struct {
+	Error      *model.AppError
+	LineNumber int
+}
+
+type AttachmentImportData struct {
+	Path *string   `json:"path"`
+	Data *zip.File `json:"-"`
+}
+
+type ComparablePreference struct {
+	Category string
+	Name     string
+}