Merge pull request #36 from kvalev/upstream-develop

Merge upstream changes

cmd/photoprism/photoprism.go

@@ -60,6 +60,7 @@ func main() {
 		commands.ImportCommand,
 		commands.CopyCommand,
 		commands.FacesCommand,
+		commands.PlacesCommand,
 		commands.PurgeCommand,
 		commands.CleanUpCommand,
 		commands.OptimizeCommand,

internal/commands/passwd.go

@@ -39,7 +39,7 @@ func passwdAction(ctx *cli.Context) error {
 
 	user := entity.Admin
 
-	log.Infof("please enter a new password for %s (at least 6 characters)\n", txt.Quote(user.UserName))
+	log.Infof("please enter a new password for %s (at least 6 characters)\n", txt.Quote(user.Username()))
 
 	newPassword := getPassword("New Password: ")
 
@@ -57,7 +57,7 @@ func passwdAction(ctx *cli.Context) error {
 		return err
 	}
 
-	log.Infof("changed password for %s\n", txt.Quote(user.UserName))
+	log.Infof("changed password for %s\n", txt.Quote(user.Username()))
 
 	conf.Shutdown()
 

internal/commands/places.go

@@ -0,0 +1,58 @@
+package commands
+
+import (
+	"context"
+	"time"
+
+	"github.com/dustin/go-humanize/english"
+
+	"github.com/urfave/cli"
+
+	"github.com/photoprism/photoprism/internal/config"
+	"github.com/photoprism/photoprism/internal/service"
+)
+
+// PlacesCommand registers the places subcommands.
+var PlacesCommand = cli.Command{
+	Name:  "places",
+	Usage: "Location information subcommands",
+	Subcommands: []cli.Command{
+		{
+			Name:   "update",
+			Usage:  "Fetches updated location data",
+			Action: placesUpdateAction,
+		},
+	},
+}
+
+// placesUpdateAction fetches updated location data.
+func placesUpdateAction(ctx *cli.Context) error {
+	start := time.Now()
+
+	conf := config.NewConfig(ctx)
+	service.SetConfig(conf)
+
+	_, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	if err := conf.Init(); err != nil {
+		return err
+	}
+
+	conf.InitDb()
+
+	w := service.Places()
+
+	// Run places worker.
+	if updated, err := w.Start(); err != nil {
+		return err
+	} else {
+		elapsed := time.Since(start)
+
+		log.Infof("updated %s in %s", english.Plural(len(updated), "location", "locations"), elapsed)
+	}
+
+	conf.Shutdown()
+
+	return nil
+}

internal/commands/users.go

@@ -211,7 +211,7 @@ func usersListAction(ctx *cli.Context) error {
 		fmt.Printf("%-4s %-16s %-16s %-16s\n", "ID", "LOGIN", "NAME", "EMAIL")
 
 		for _, user := range users {
-			fmt.Printf("%-4d %-16s %-16s %-16s", user.ID, user.UserName, user.FullName, user.PrimaryEmail)
+			fmt.Printf("%-4d %-16s %-16s %-16s", user.ID, user.Username(), user.FullName, user.PrimaryEmail)
 			fmt.Printf("\n")
 		}
 
@@ -242,7 +242,7 @@ func usersUpdateAction(ctx *cli.Context) error {
 			if err != nil {
 				return err
 			}
-			fmt.Printf("password successfully changed: %v\n", u.UserName)
+			fmt.Printf("password successfully changed: %v\n", u.Username())
 		}
 
 		if ctx.IsSet("fullname") {
@@ -261,7 +261,7 @@ func usersUpdateAction(ctx *cli.Context) error {
 			return err
 		}
 
-		fmt.Printf("user successfully updated: %v\n", u.UserName)
+		fmt.Printf("user successfully updated: %v\n", u.Username())
 
 		return nil
 	})

internal/config/config_test.go

@@ -257,7 +257,7 @@ func TestConfig_ClientConfig(t *testing.T) {
 	assert.NotEmpty(t, cc.ManifestHash)
 	assert.Equal(t, true, cc.Debug)
 	assert.Equal(t, false, cc.Demo)
-	assert.Equal(t, false, cc.Sponsor)
+	assert.Equal(t, true, cc.Sponsor)
 	assert.Equal(t, false, cc.ReadOnly)
 }
 

internal/config/flags.go

@@ -345,7 +345,7 @@ var GlobalFlags = []cli.Flag{
 	cli.StringFlag{
 		Name:   "darktable-blacklist",
 		Usage:  "RAW file `EXTENSIONS` incompatible with Darktable",
-		Value:  "raf",
+		Value:  "cr3,dng",
 		EnvVar: "PHOTOPRISM_DARKTABLE_BLACKLIST",
 	},
 	cli.StringFlag{

internal/config/test.go

@@ -55,6 +55,7 @@ func NewTestOptions() *Options {
 		Name:           "PhotoPrism",
 		Version:        "0.0.0",
 		Copyright:      "(c) 2018-2021 Michael Mayer",
+		Test:           true,
 		Debug:          true,
 		Public:         true,
 		Experimental:   true,

internal/entity/cell.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/photoprism/photoprism/internal/event"
 	"github.com/photoprism/photoprism/internal/maps"
+
 	"github.com/photoprism/photoprism/pkg/s2"
 	"github.com/photoprism/photoprism/pkg/txt"
 )
@@ -16,14 +17,19 @@ var cellMutex = sync.Mutex{}
 // Cell represents a S2 cell with location data.
 type Cell struct {
 	ID           string    `gorm:"type:VARBINARY(42);primary_key;auto_increment:false;" json:"ID" yaml:"ID"`
-	CellName     string    `gorm:"type:VARCHAR(255);" json:"Name" yaml:"Name,omitempty"`
-	CellCategory string    `gorm:"type:VARCHAR(64);" json:"Category" yaml:"Category,omitempty"`
+	CellName     string    `gorm:"type:VARCHAR(200);" json:"Name" yaml:"Name,omitempty"`
+	CellCategory string    `gorm:"type:VARCHAR(50);" json:"Category" yaml:"Category,omitempty"`
 	PlaceID      string    `gorm:"type:VARBINARY(42);default:'zz'" json:"-" yaml:"PlaceID"`
 	Place        *Place    `gorm:"PRELOAD:true" json:"Place" yaml:"-"`
 	CreatedAt    time.Time `json:"CreatedAt" yaml:"-"`
 	UpdatedAt    time.Time `json:"UpdatedAt" yaml:"-"`
 }
 
+// TableName returns the entity database table name.
+func (Cell) TableName() string {
+	return "cells"
+}
+
 // UnknownLocation is PhotoPrism's default location.
 var UnknownLocation = Cell{
 	ID:           UnknownID,
@@ -47,6 +53,74 @@ func NewCell(lat, lng float32) *Cell {
 	return result
 }
 
+// Refresh updates the index by retrieving the latest data from an external API.
+func (m *Cell) Refresh(api string) (err error) {
+	start := time.Now()
+
+	// Unknown?
+	if m.Unknown() {
+		// Skip.
+		return nil
+	}
+
+	// Initialize.
+	l := &maps.Location{
+		ID: s2.NormalizeToken(m.ID),
+	}
+
+	// Query geodata API.
+	if err = l.QueryApi(api); err != nil {
+		return err
+	}
+
+	// Unknown location or label missing?
+	if l.Unknown() || l.Label() == "" {
+		// Ignore.
+		return nil
+	}
+
+	cellTable := Cell{}.TableName()
+	placeTable := Place{}.TableName()
+
+	place := Place{}
+
+	// Find existing place by label.
+	if err := UnscopedDb().Where("place_label = ?", l.Label()).First(&place).Error; err != nil {
+		log.Tracef("places: %s for cell %s", err, m.ID)
+		place = Place{ID: m.ID}
+	} else {
+		log.Tracef("places: found matching place %s for cell %s", place.ID, m.ID)
+	}
+
+	// Update place.
+	if place.ID == "" {
+		// Do nothing.
+	} else if res := UnscopedDb().Table(placeTable).Where("id = ?", place.ID).UpdateColumns(Values{
+		"place_label":    l.Label(),
+		"place_city":     l.City(),
+		"place_district": l.District(),
+		"place_state":    l.State(),
+		"place_country":  l.CountryCode(),
+		"place_keywords": l.KeywordString(),
+	}); res.Error != nil {
+		log.Tracef("places: %s for cell %s", err, m.ID)
+	} else if res.RowsAffected > 0 {
+		// Update cell place id, name, and category.
+		log.Tracef("places: updating place, name, and category for cell %s", m.ID)
+		err = UnscopedDb().Table(cellTable).Where("id = ?", m.ID).
+			UpdateColumns(Values{"cell_name": l.Name(), "cell_category": l.Category(), "place_id": place.ID}).Error
+	} else {
+		// Update cell name and category.
+		log.Tracef("places: updating name and category for cell %s", m.ID)
+		err = UnscopedDb().Table(cellTable).Where("id = ?", m.ID).
+			UpdateColumns(Values{"cell_name": l.Name(), "cell_category": l.Category()}).Error
+	}
+
+	log.Debugf("places: refreshed cell %s [%s]", txt.Quote(m.ID), time.Since(start))
+
+	return err
+}
+
 // Find retrieves location data from the database or an external api if not known already.
 func (m *Cell) Find(api string) error {
 	start := time.Now()

internal/entity/place.go

@@ -13,9 +13,10 @@ var placeMutex = sync.Mutex{}
 // Place used to associate photos to places
 type Place struct {
 	ID            string    `gorm:"type:VARBINARY(42);primary_key;auto_increment:false;" json:"PlaceID" yaml:"PlaceID"`
-	PlaceLabel    string    `gorm:"type:VARBINARY(755);unique_index;" json:"Label" yaml:"Label"`
-	PlaceCity     string    `gorm:"type:VARCHAR(255);" json:"City" yaml:"City,omitempty"`
-	PlaceState    string    `gorm:"type:VARCHAR(255);" json:"State" yaml:"State,omitempty"`
+	PlaceLabel    string    `gorm:"type:VARBINARY(512);unique_index;" json:"Label" yaml:"Label"`
+	PlaceCity     string    `gorm:"type:VARCHAR(128);" json:"City" yaml:"City,omitempty"`
+	PlaceState    string    `gorm:"type:VARCHAR(128);" json:"State" yaml:"State,omitempty"`
+	PlaceDistrict string    `gorm:"type:VARCHAR(128);" json:"District" yaml:"District,omitempty"`
 	PlaceCountry  string    `gorm:"type:VARBINARY(2);" json:"Country" yaml:"Country,omitempty"`
 	PlaceKeywords string    `gorm:"type:VARCHAR(255);" json:"Keywords" yaml:"Keywords,omitempty"`
 	PlaceFavorite bool      `json:"Favorite" yaml:"Favorite,omitempty"`
@@ -24,11 +25,17 @@ type Place struct {
 	UpdatedAt     time.Time `json:"UpdatedAt" yaml:"-"`
 }
 
+// TableName returns the entity database table name.
+func (Place) TableName() string {
+	return "places"
+}
+
 // UnknownPlace is PhotoPrism's default place.
 var UnknownPlace = Place{
 	ID:            UnknownID,
 	PlaceLabel:    "Unknown",
 	PlaceCity:     "Unknown",
+	PlaceDistrict: "Unknown",
 	PlaceState:    "Unknown",
 	PlaceCountry:  UnknownID,
 	PlaceKeywords: "",

internal/entity/user.go

@@ -165,6 +165,8 @@ func FirstOrCreateUser(m *User) *User {
 
 // FindUserByName returns an existing user or nil if not found.
 func FindUserByName(userName string) *User {
+	userName = txt.NormalizeUsername(userName)
+
 	if userName == "" {
 		return nil
 	}
@@ -211,8 +213,8 @@ func (m *User) Deleted() bool {
 
 // String returns an identifier that can be used in logs.
 func (m *User) String() string {
-	if m.UserName != "" {
-		return m.UserName
+	if n := m.Username(); n != "" {
+		return n
 	}
 
 	if m.FullName != "" {
@@ -222,9 +224,14 @@ func (m *User) String() string {
 	return m.UserUID
 }
 
+// Username returns the normalized username.
+func (m *User) Username() string {
+	return txt.NormalizeUsername(m.UserName)
+}
+
 // Registered tests if the user is registered e.g. has a username.
 func (m *User) Registered() bool {
-	return m.UserName != "" && rnd.IsPPID(m.UserUID, 'u')
+	return m.Username() != "" && rnd.IsPPID(m.UserUID, 'u')
 }
 
 // Admin returns true if the user is an admin with user name.
@@ -249,7 +256,7 @@ func (m *User) SetPassword(password string) error {
 	}
 
 	if len(password) < 4 {
-		return fmt.Errorf("new password for %s must be at least 4 characters", txt.Quote(m.UserName))
+		return fmt.Errorf("new password for %s must be at least 4 characters", txt.Quote(m.Username()))
 	}
 
 	pw := NewPassword(m.UserUID, password)
@@ -342,30 +349,37 @@ func (m *User) Role() acl.Role {
 
 // Validate Makes sure username and email are unique and meet requirements. Returns error if any property is invalid
 func (m *User) Validate() error {
-	if m.UserName == "" {
+	if m.Username() == "" {
 		return errors.New("username must not be empty")
 	}
-	if len(m.UserName) < 4 {
+
+	if len(m.Username()) < 4 {
 		return errors.New("username must be at least 4 characters")
 	}
+
 	var err error
 	var resultName = User{}
-	if err = Db().Where("user_name = ? AND id <> ?", m.UserName, m.ID).First(&resultName).Error; err == nil {
+
+	if err = Db().Where("user_name = ? AND id <> ?", m.Username(), m.ID).First(&resultName).Error; err == nil {
 		return errors.New("username already exists")
 	} else if err != gorm.ErrRecordNotFound {
 		return err
 	}
+
 	// stop here if no email is provided
 	if m.PrimaryEmail == "" {
 		return nil
 	}
+
 	// validate email address
 	if a, err := mail.ParseAddress(m.PrimaryEmail); err != nil {
 		return err
 	} else {
 		m.PrimaryEmail = a.Address // make sure email address will be used without name
 	}
+
 	var resultMail = User{}
+
 	if err = Db().Where("primary_email = ? AND id <> ?", m.PrimaryEmail, m.ID).First(&resultMail).Error; err == nil {
 		return errors.New("email already exists")
 	} else if err != gorm.ErrRecordNotFound {
@@ -384,7 +398,7 @@ func CreateWithPassword(uc form.UserCreate) error {
 		RoleAdmin:    true,
 	}
 	if len(uc.Password) < 4 {
-		return fmt.Errorf("new password for %s must be at least 4 characters", txt.Quote(u.UserName))
+		return fmt.Errorf("new password for %s must be at least 4 characters", txt.Quote(u.Username()))
 	}
 	err := u.Validate()
 	if err != nil {
@@ -398,7 +412,7 @@ func CreateWithPassword(uc form.UserCreate) error {
 		if err := tx.Create(&pw).Error; err != nil {
 			return err
 		}
-		log.Infof("created user %v with uid %v", txt.Quote(u.UserName), txt.Quote(u.UserUID))
+		log.Infof("created user %v with uid %v", txt.Quote(u.Username()), txt.Quote(u.UserUID))
 		return nil
 	})
 }

internal/entity/user_fixtures_test.go

@@ -10,11 +10,13 @@ func TestUserMap_Get(t *testing.T) {
 	t.Run("get existing user", func(t *testing.T) {
 		r := UserFixtures.Get("alice")
 		assert.Equal(t, "alice", r.UserName)
+		assert.Equal(t, "alice", r.Username())
 		assert.IsType(t, User{}, r)
 	})
 	t.Run("get not existing user", func(t *testing.T) {
 		r := UserFixtures.Get("monstera")
 		assert.Equal(t, "", r.UserName)
+		assert.Equal(t, "", r.Username())
 		assert.IsType(t, User{}, r)
 	})
 }