Merge branch 'master' into ttaylorr/prove

commands/command_migrate.go

@@ -173,13 +173,31 @@ func includeExcludeRefs(l *tasklog.Logger, args []string) (include, exclude []st
 		include = append(include, migrateIncludeRefs...)
 		exclude = append(exclude, migrateExcludeRefs...)
 	} else if migrateEverything {
-		localRefs, err := git.LocalRefs()
+		refs, err := git.AllRefsIn("")
 		if err != nil {
 			return nil, nil, err
 		}
 
-		for _, ref := range localRefs {
-			include = append(include, ref.Refspec())
+		for _, ref := range refs {
+			switch ref.Type {
+			case git.RefTypeLocalBranch, git.RefTypeLocalTag,
+				git.RefTypeRemoteBranch, git.RefTypeRemoteTag:
+
+				include = append(include, ref.Refspec())
+			case git.RefTypeOther:
+				parts := strings.SplitN(ref.Refspec(), "/", 3)
+				if len(parts) < 2 {
+					continue
+				}
+
+				switch parts[1] {
+				// The following are GitLab-, GitHub-, VSTS-,
+				// and BitBucket-specific reference naming
+				// conventions.
+				case "merge-requests", "pull", "pull-requests":
+					include = append(include, ref.Refspec())
+				}
+			}
 		}
 	} else {
 		bare, err := git.IsBare()

config/config.go

@@ -311,8 +311,8 @@ func (c *Configuration) LocalGitStorageDir() string {
 	return c.Filesystem().GitStorageDir
 }
 
-func (c *Configuration) LocalReferenceDir() string {
-	return c.Filesystem().ReferenceDir
+func (c *Configuration) LocalReferenceDirs() []string {
+	return c.Filesystem().ReferenceDirs
 }
 
 func (c *Configuration) LFSStorageDir() string {
@@ -346,7 +346,12 @@ func (c *Configuration) Filesystem() *fs.Filesystem {
 
 	if c.fs == nil {
 		lfsdir, _ := c.Git.Get("lfs.storage")
-		c.fs = fs.New(c.LocalGitDir(), c.LocalWorkingDir(), lfsdir)
+		c.fs = fs.New(
+			c.Os,
+			c.LocalGitDir(),
+			c.LocalWorkingDir(),
+			lfsdir,
+		)
 	}
 
 	return c.fs

config/git_fetcher.go

@@ -158,6 +158,7 @@ func keyIsUnsafe(key string) bool {
 }
 
 var safeKeys = []string{
+	"lfs.allowincompletepush",
 	"lfs.fetchexclude",
 	"lfs.fetchinclude",
 	"lfs.gitprotocol",

docs/man/git-lfs-migrate.1.ronn

@@ -190,8 +190,8 @@ The following configuration:
 
 Would, therefore, include commits: F, E, D, C, B, but exclude commit A.
 
-The presence of flag `--everything` indicates that all local references should be
-migrated.
+The presence of flag `--everything` indicates that all local and remote
+references should be migrated.
 
 ## EXAMPLES
 
@@ -253,16 +253,20 @@ Note: This will require a force push to any existing Git remotes.
 
 ### Migrate without rewriting local history
 
-You can also migrate files without modifying the existing history of your respoitory:
+You can also migrate files without modifying the existing history of your
+repository:
 
 Without a specified commit message:
+
 ```
-    git lfs migrate import --no-rewrite test.zip *.mp3 *.psd
+$ git lfs migrate import --no-rewrite test.zip *.mp3 *.psd
 ```
+
 With a specified commit message:
+
 ```
-    git lfs migrate import --no-rewrite -m "Import .zip, .mp3, .psd files" \
-      test.zip *.mpd *.psd
+$ git lfs migrate import --no-rewrite -m "Import .zip, .mp3, .psd files" \
+  test.zip *.mpd *.psd
 ```
 
 ## SEE ALSO

fs/fs.go

@@ -1,6 +1,7 @@
 package fs
 
 import (
+	"bufio"
 	"bytes"
 	"fmt"
 	"io/ioutil"
@@ -12,20 +13,29 @@ import (
 	"sync"
 
 	"github.com/git-lfs/git-lfs/tools"
+	"github.com/rubyist/tracerx"
 )
 
 var oidRE = regexp.MustCompile(`\A[[:alnum:]]{64}`)
 
+// Environment is a copy of a subset of the interface
+// github.com/git-lfs/git-lfs/config.Environment.
+//
+// For more information, see config/environment.go.
+type Environment interface {
+	Get(key string) (val string, ok bool)
+}
+
 // Object represents a locally stored LFS object.
 type Object struct {
 	Oid  string
 	Size int64
 }
 
 type Filesystem struct {
-	GitStorageDir string // parent of objects/lfs (may be same as GitDir but may not)
-	LFSStorageDir string // parent of lfs objects and tmp dirs. Default: ".git/lfs"
-	ReferenceDir  string // alternative local media dir (relative to clone reference repo)
+	GitStorageDir string   // parent of objects/lfs (may be same as GitDir but may not)
+	LFSStorageDir string   // parent of lfs objects and tmp dirs. Default: ".git/lfs"
+	ReferenceDirs []string // alternative local media dirs (relative to clone reference repo)
 	lfsobjdir     string
 	tmpdir        string
 	logdir        string
@@ -105,12 +115,16 @@ func (f *Filesystem) localObjectDir(oid string) string {
 	return filepath.Join(f.LFSObjectDir(), oid[0:2], oid[2:4])
 }
 
-func (f *Filesystem) ObjectReferencePath(oid string) string {
-	if len(f.ReferenceDir) == 0 {
-		return f.ReferenceDir
+func (f *Filesystem) ObjectReferencePaths(oid string) []string {
+	if len(f.ReferenceDirs) == 0 {
+		return nil
 	}
 
-	return filepath.Join(f.ReferenceDir, oid[0:2], oid[2:4], oid)
+	var paths []string
+	for _, ref := range f.ReferenceDirs {
+		paths = append(paths, filepath.Join(ref, oid[0:2], oid[2:4], oid))
+	}
+	return paths
 }
 
 func (f *Filesystem) LFSObjectDir() string {
@@ -159,12 +173,12 @@ func (f *Filesystem) Cleanup() error {
 // New initializes a new *Filesystem with the given directories. gitdir is the
 // path to the bare repo, workdir is the path to the repository working
 // directory, and lfsdir is the optional path to the `.git/lfs` directory.
-func New(gitdir, workdir, lfsdir string) *Filesystem {
+func New(env Environment, gitdir, workdir, lfsdir string) *Filesystem {
 	fs := &Filesystem{
 		GitStorageDir: resolveGitStorageDir(gitdir),
 	}
 
-	fs.ReferenceDir = resolveReferenceDir(fs.GitStorageDir)
+	fs.ReferenceDirs = resolveReferenceDirs(env, fs.GitStorageDir)
 
 	if len(lfsdir) == 0 {
 		lfsdir = "lfs"
@@ -179,19 +193,75 @@ func New(gitdir, workdir, lfsdir string) *Filesystem {
 	return fs
 }
 
-func resolveReferenceDir(gitStorageDir string) string {
+func resolveReferenceDirs(env Environment, gitStorageDir string) []string {
+	var references []string
+
+	envAlternates, ok := env.Get("GIT_ALTERNATE_OBJECT_DIRECTORIES")
+	if ok {
+		splits := strings.Split(envAlternates, string(os.PathListSeparator))
+		for _, split := range splits {
+			if dir, ok := existsAlternate(split); ok {
+				references = append(references, dir)
+			}
+		}
+	}
+
 	cloneReferencePath := filepath.Join(gitStorageDir, "objects", "info", "alternates")
 	if tools.FileExists(cloneReferencePath) {
-		buffer, err := ioutil.ReadFile(cloneReferencePath)
-		if err == nil {
-			path := strings.TrimSpace(string(buffer[:]))
-			referenceLfsStoragePath := filepath.Join(filepath.Dir(path), "lfs", "objects")
-			if tools.DirExists(referenceLfsStoragePath) {
-				return referenceLfsStoragePath
+		f, err := os.Open(cloneReferencePath)
+		if err != nil {
+			tracerx.Printf("could not open %s: %s",
+				cloneReferencePath, err)
+			return nil
+		}
+		defer f.Close()
+
+		scanner := bufio.NewScanner(f)
+		for scanner.Scan() {
+			text := strings.TrimSpace(scanner.Text())
+			if len(text) == 0 || strings.HasPrefix(text, "#") {
+				continue
 			}
+
+			if dir, ok := existsAlternate(text); ok {
+				references = append(references, dir)
+			}
+		}
+
+		if err := scanner.Err(); err != nil {
+			tracerx.Printf("could not scan %s: %s",
+				cloneReferencePath, err)
+		}
+	}
+	return references
+}
+
+// existsAlternate takes an object directory given in "objs" (read as a single,
+// line from .git/objects/info/alternates). If that is a satisfiable alternates
+// directory (i.e., it exists), the directory is returned along with "true". If
+// not, the empty string and false is returned instead.
+func existsAlternate(objs string) (string, bool) {
+	objs = strings.TrimSpace(objs)
+	if strings.HasPrefix(objs, "\"") {
+		var err error
+
+		unquote := strings.LastIndex(objs, "\"")
+		if unquote == 0 {
+			return "", false
 		}
+
+		objs, err = strconv.Unquote(objs[:unquote+1])
+		if err != nil {
+			return "", false
+		}
+	}
+
+	storage := filepath.Join(filepath.Dir(objs), "lfs", "objects")
+
+	if tools.DirExists(storage) {
+		return storage, true
 	}
-	return ""
+	return "", false
 }
 
 // From a git dir, get the location that objects are to be stored (we will store lfs alongside)

lfs/gitfilter_smudge.go

@@ -16,6 +16,18 @@ import (
 
 func (f *GitFilter) SmudgeToFile(filename string, ptr *Pointer, download bool, manifest *tq.Manifest, cb tools.CopyCallback) error {
 	os.MkdirAll(filepath.Dir(filename), 0755)
+
+	if stat, _ := os.Stat(filename); stat != nil && stat.Mode()&0200 == 0 {
+		if err := os.Chmod(filename, stat.Mode()|0200); err != nil {
+			return errors.Wrap(err,
+				"Could not restore write permission")
+		}
+
+		// When we're done, return the file back to its normal
+		// permission bits.
+		defer os.Chmod(filename, stat.Mode())
+	}
+
 	file, err := os.Create(filename)
 	if err != nil {
 		return fmt.Errorf("Could not create working directory file: %v", err)

lfs/lfs.go

@@ -35,12 +35,14 @@ func Environ(cfg *config.Configuration, manifest *tq.Manifest) []string {
 
 	fetchPruneConfig := NewFetchPruneConfig(cfg.Git)
 
+	references := strings.Join(cfg.LocalReferenceDirs(), ", ")
+
 	env = append(env,
 		fmt.Sprintf("LocalWorkingDir=%s", cfg.LocalWorkingDir()),
 		fmt.Sprintf("LocalGitDir=%s", cfg.LocalGitDir()),
 		fmt.Sprintf("LocalGitStorageDir=%s", cfg.LocalGitStorageDir()),
 		fmt.Sprintf("LocalMediaDir=%s", cfg.LFSObjectDir()),
-		fmt.Sprintf("LocalReferenceDir=%s", cfg.LocalReferenceDir()),
+		fmt.Sprintf("LocalReferenceDirs=%s", references),
 		fmt.Sprintf("TempDir=%s", cfg.TempDir()),
 		fmt.Sprintf("ConcurrentTransfers=%d", api.ConcurrentTransfers),
 		fmt.Sprintf("TusTransfers=%v", cfg.TusTransfersAllowed()),
@@ -98,13 +100,19 @@ func LinkOrCopyFromReference(cfg *config.Configuration, oid string, size int64)
 	if cfg.LFSObjectExists(oid, size) {
 		return nil
 	}
-	altMediafile := cfg.Filesystem().ObjectReferencePath(oid)
+	altMediafiles := cfg.Filesystem().ObjectReferencePaths(oid)
 	mediafile, err := cfg.Filesystem().ObjectPath(oid)
 	if err != nil {
 		return err
 	}
-	if altMediafile != "" && tools.FileExistsOfSize(altMediafile, size) {
-		return LinkOrCopy(cfg, altMediafile, mediafile)
+	for _, altMediafile := range altMediafiles {
+		tracerx.Printf("altMediafile: %s", altMediafile)
+		if altMediafile != "" && tools.FileExistsOfSize(altMediafile, size) {
+			err = LinkOrCopy(cfg, altMediafile, mediafile)
+			if err == nil {
+				break
+			}
+		}
 	}
-	return nil
+	return err
 }