system: filename constants; environment variables

* `cmn/fname` package and `fname/fname.go` for all system filenames * `api/env` package * add `api/env/README.md` * remove `AIS_AUTHN_CONF_FILE`, use directory/fname instead Signed-off-by: Alex Aizman <[email protected]>
NVIDIA · Jul 7, 2022 · 68abf61 · 68abf61
1 parent 5339418
commit 68abf61
Show file tree

Hide file tree

Showing 41 changed files with 224 additions and 186 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,9 +3,6 @@
 *.csv
 *.svg
 .*.swp
-tags
-FlameGraph/
-openapi-generator-cli.jar
 
 ## PYTHON ##
 
@@ -65,29 +62,15 @@ coverage.xml
 # Environments
 .env
 .venv
-env/
 venv/
 ENV/
 env.bak/
 venv.bak/
 
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Goland IntelliJ IDE
-.idea
-
-# Rope project settings
-.ropeproject
-
-# Bench
-bench/aisloader/aisloader.exe
-
 # Website related stuff
 .netlify/
 _site/
 .sass-cache/
 
 # VSCode
-.vscode
+.vscode
diff --git a/ais/bucketmeta.go b/ais/bucketmeta.go
@@ -22,6 +22,7 @@ import (
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
 	"github.com/NVIDIA/aistore/cmn/debug"
+	"github.com/NVIDIA/aistore/cmn/fname"
 	"github.com/NVIDIA/aistore/cmn/jsp"
 	"github.com/NVIDIA/aistore/fs"
 	"github.com/NVIDIA/aistore/memsys"
@@ -281,7 +282,7 @@ func (*bmdOwnerBase) persistBytes(payload msPayload, fpath string) (done bool) {
 /////////////////
 
 func newBMDOwnerPrx(config *cmn.Config) *bmdOwnerPrx {
-	return &bmdOwnerPrx{fpath: filepath.Join(config.ConfigDir, cmn.BmdFname)}
+	return &bmdOwnerPrx{fpath: filepath.Join(config.ConfigDir, fname.Bmd)}
 }
 
 func (bo *bmdOwnerPrx) init() {
@@ -355,12 +356,12 @@ func (bo *bmdOwnerTgt) init() {
 		bmd       *bucketMD
 		available = fs.GetAvail()
 	)
-	if bmd = loadBMD(available, cmn.BmdFname); bmd != nil {
+	if bmd = loadBMD(available, fname.Bmd); bmd != nil {
 		glog.Infof("loaded %s", bmd)
 		goto finalize
 	}
-	if bmd = loadBMD(available, cmn.BmdPreviousFname); bmd != nil {
-		glog.Errorf("loaded previous version of the %s (%q)", bmd, cmn.BmdPreviousFname)
+	if bmd = loadBMD(available, fname.BmdPrevious); bmd != nil {
+		glog.Errorf("loaded previous version of the %s (%q)", bmd, fname.BmdPrevious)
 		goto finalize
 	}
 	bmd = newBucketMD()
@@ -391,7 +392,7 @@ func (*bmdOwnerTgt) persist(clone *bucketMD, payload msPayload) (err error) {
 		sgl = clone._encode()
 		defer sgl.Free()
 	}
-	cnt, availCnt := fs.PersistOnMpaths(cmn.BmdFname, cmn.BmdPreviousFname, clone, bmdCopies, b, sgl)
+	cnt, availCnt := fs.PersistOnMpaths(fname.Bmd, fname.BmdPrevious, clone, bmdCopies, b, sgl)
 	if cnt > 0 {
 		return
 	}
@@ -466,7 +467,7 @@ func loadBMDFromMpath(mpath *fs.MountpathInfo, path string) (bmd *bucketMD) {
 	return nil
 }
 
-func hasEnoughBMDCopies() bool { return fs.CountPersisted(cmn.BmdFname) >= bmdCopies }
+func hasEnoughBMDCopies() bool { return fs.CountPersisted(fname.Bmd) >= bmdCopies }
 
 //////////////////////////
 // default bucket props //

diff --git a/ais/bucketmeta_test.go b/ais/bucketmeta_test.go
@@ -15,6 +15,7 @@ import (
 	"github.com/NVIDIA/aistore/cluster"
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
+	"github.com/NVIDIA/aistore/cmn/fname"
 	"github.com/NVIDIA/aistore/cmn/jsp"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -120,7 +121,7 @@ var _ = Describe("BMD marshal and unmarshal", func() {
 			})
 
 			It(fmt.Sprintf("should correctly detect bmd corruption %s", node), func() {
-				bmdFullPath := filepath.Join(mpath, cmn.BmdFname)
+				bmdFullPath := filepath.Join(mpath, fname.Bmd)
 				f, err := os.OpenFile(bmdFullPath, os.O_RDWR, 0)
 				Expect(err).NotTo(HaveOccurred())
 				_, err = f.WriteAt([]byte("xxxxxxxxxxxx"), 10)

diff --git a/ais/clustermap.go b/ais/clustermap.go
@@ -22,6 +22,7 @@ import (
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
 	"github.com/NVIDIA/aistore/cmn/debug"
+	"github.com/NVIDIA/aistore/cmn/fname"
 	"github.com/NVIDIA/aistore/cmn/jsp"
 	"github.com/NVIDIA/aistore/memsys"
 	"github.com/NVIDIA/aistore/nl"
@@ -445,7 +446,7 @@ func (m *smapX) clearNodeFlags(id string, flags cos.BitFlags) {
 func newSmapOwner(config *cmn.Config) *smapOwner {
 	return &smapOwner{
 		listeners: newSmapListeners(),
-		fpath:     filepath.Join(config.ConfigDir, cmn.SmapFname),
+		fpath:     filepath.Join(config.ConfigDir, fname.Smap),
 	}
 }
 

diff --git a/ais/emd_test.go b/ais/emd_test.go
@@ -14,6 +14,7 @@ import (
 	"github.com/NVIDIA/aistore/api/apc"
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
+	"github.com/NVIDIA/aistore/cmn/fname"
 	"github.com/NVIDIA/aistore/cmn/jsp"
 	"github.com/NVIDIA/aistore/etl"
 	. "github.com/onsi/ginkgo"
@@ -153,7 +154,7 @@ var _ = Describe("EtlMD marshal and unmarshal", func() {
 			})
 
 			It(fmt.Sprintf("should correctly detect etlMD corruption %s", node), func() {
-				etlMDFullPath := filepath.Join(mpath, cmn.EmdFname)
+				etlMDFullPath := filepath.Join(mpath, fname.Emd)
 				f, err := os.OpenFile(etlMDFullPath, os.O_RDWR, 0)
 				Expect(err).NotTo(HaveOccurred())
 				_, err = f.WriteAt([]byte("xxxxxxxxxxxx"), 10)

diff --git a/ais/etlmeta.go b/ais/etlmeta.go
@@ -17,6 +17,7 @@ import (
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
 	"github.com/NVIDIA/aistore/cmn/debug"
+	"github.com/NVIDIA/aistore/cmn/fname"
 	"github.com/NVIDIA/aistore/cmn/jsp"
 	"github.com/NVIDIA/aistore/etl"
 	"github.com/NVIDIA/aistore/fs"
@@ -151,7 +152,7 @@ func (*etlMDOwnerBase) persistBytes(payload msPayload, fpath string) (done bool)
 /////////////////
 
 func newEtlMDOwnerPrx(config *cmn.Config) *etlMDOwnerPrx {
-	return &etlMDOwnerPrx{fpath: filepath.Join(config.ConfigDir, cmn.EmdFname)}
+	return &etlMDOwnerPrx{fpath: filepath.Join(config.ConfigDir, fname.Emd)}
 }
 
 func (eo *etlMDOwnerPrx) init() {
@@ -218,7 +219,7 @@ func (eo *etlMDOwnerTgt) init() {
 		etlMD     *etlMD
 		available = fs.GetAvail()
 	)
-	if etlMD = loadEtlMD(available, cmn.EmdFname); etlMD != nil {
+	if etlMD = loadEtlMD(available, fname.Emd); etlMD != nil {
 		glog.Infof("loaded %s", etlMD)
 		goto finalize
 	}
@@ -246,7 +247,7 @@ func (*etlMDOwnerTgt) persist(clone *etlMD, payload msPayload) (err error) {
 	if b == nil {
 		b = clone.marshal()
 	}
-	cnt, availCnt := fs.PersistOnMpaths(cmn.EmdFname, "" /*backup*/, clone, etlMDCopies, b, nil /*sgl*/)
+	cnt, availCnt := fs.PersistOnMpaths(fname.Emd, "" /*backup*/, clone, etlMDCopies, b, nil /*sgl*/)
 	if cnt > 0 {
 		return
 	}
@@ -308,4 +309,4 @@ func loadEtlMDFromMpath(mpath *fs.MountpathInfo, path string) (etlMD *etlMD) {
 	return nil
 }
 
-func hasEnoughEtlMDCopies() bool { return fs.CountPersisted(cmn.EmdFname) >= etlMDCopies }
+func hasEnoughEtlMDCopies() bool { return fs.CountPersisted(fname.Emd) >= etlMDCopies }
diff --git a/ais/gconfig.go b/ais/gconfig.go
@@ -19,6 +19,7 @@ import (
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
 	"github.com/NVIDIA/aistore/cmn/debug"
+	"github.com/NVIDIA/aistore/cmn/fname"
 	"github.com/NVIDIA/aistore/cmn/jsp"
 	"github.com/NVIDIA/aistore/memsys"
 )
@@ -79,7 +80,7 @@ func (config *globalConfig) _encode(immSize int64) (sgl *memsys.SGL) {
 
 func newConfigOwner(config *cmn.Config) (co *configOwner) {
 	co = &configOwner{}
-	co.globalFpath = filepath.Join(config.ConfigDir, cmn.GlobalConfigFname)
+	co.globalFpath = filepath.Join(config.ConfigDir, fname.GlobalConfig)
 	return
 }
 
@@ -230,7 +231,7 @@ func (co *configOwner) resetDaemonConfig() (err error) {
 		return err
 	}
 	cmn.GCO.PutOverrideConfig(nil)
-	err = cos.RemoveFile(filepath.Join(oldConfig.ConfigDir, cmn.OverrideConfigFname))
+	err = cos.RemoveFile(filepath.Join(oldConfig.ConfigDir, fname.OverrideConfig))
 	if err != nil {
 		co.Unlock()
 		return

diff --git a/ais/proxy.go b/ais/proxy.go
@@ -28,6 +28,7 @@ import (
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
 	"github.com/NVIDIA/aistore/cmn/debug"
+	"github.com/NVIDIA/aistore/cmn/fname"
 	"github.com/NVIDIA/aistore/cmn/mono"
 	"github.com/NVIDIA/aistore/dsort"
 	"github.com/NVIDIA/aistore/memsys"
@@ -136,22 +137,22 @@ func initPID(config *cmn.Config) (pid string) {
 	}
 	// try to read ID
 	if pid = readProxyID(config); pid != "" {
-		glog.Infof("p[%s] from %q", pid, cmn.ProxyIDFname)
+		glog.Infof("p[%s] from %q", pid, fname.ProxyID)
 		return
 	}
 	pid = genDaemonID(apc.Proxy, config)
 	err := cos.ValidateDaemonID(pid)
 	debug.AssertNoErr(err)
 
 	// store ID on disk
-	err = os.WriteFile(filepath.Join(config.ConfigDir, cmn.ProxyIDFname), []byte(pid), cos.PermRWR)
+	err = os.WriteFile(filepath.Join(config.ConfigDir, fname.ProxyID), []byte(pid), cos.PermRWR)
 	debug.AssertNoErr(err)
 	glog.Infof("p[%s] ID randomly generated", pid)
 	return
 }
 
 func readProxyID(config *cmn.Config) (id string) {
-	if b, err := os.ReadFile(filepath.Join(config.ConfigDir, cmn.ProxyIDFname)); err == nil {
+	if b, err := os.ReadFile(filepath.Join(config.ConfigDir, fname.ProxyID)); err == nil {
 		id = string(b)
 	} else if !os.IsNotExist(err) {
 		glog.Error(err)

diff --git a/ais/rebmeta.go b/ais/rebmeta.go
@@ -16,6 +16,7 @@ import (
 	"github.com/NVIDIA/aistore/cluster"
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
+	"github.com/NVIDIA/aistore/cmn/fname"
 	"github.com/NVIDIA/aistore/cmn/jsp"
 	"github.com/NVIDIA/aistore/memsys"
 )
@@ -86,13 +87,13 @@ func newRMDOwner() *rmdOwner {
 }
 
 func (*rmdOwner) persist(rmd *rebMD) error {
-	rmdPathName := filepath.Join(cmn.GCO.Get().ConfigDir, cmn.RmdFname)
+	rmdPathName := filepath.Join(cmn.GCO.Get().ConfigDir, fname.Rmd)
 	return jsp.SaveMeta(rmdPathName, rmd, nil /*wto*/)
 }
 
 func (r *rmdOwner) load() {
 	rmd := &rebMD{}
-	_, err := jsp.LoadMeta(filepath.Join(cmn.GCO.Get().ConfigDir, cmn.RmdFname), rmd)
+	_, err := jsp.LoadMeta(filepath.Join(cmn.GCO.Get().ConfigDir, fname.Rmd), rmd)
 	if err == nil {
 		r.put(rmd)
 		return

diff --git a/ais/target.go b/ais/target.go
@@ -29,6 +29,7 @@ import (
 	"github.com/NVIDIA/aistore/cmn/cos"
 	"github.com/NVIDIA/aistore/cmn/debug"
 	"github.com/NVIDIA/aistore/cmn/feat"
+	"github.com/NVIDIA/aistore/cmn/fname"
 	"github.com/NVIDIA/aistore/cmn/mono"
 	"github.com/NVIDIA/aistore/dbdriver"
 	"github.com/NVIDIA/aistore/dsort"
@@ -408,7 +409,7 @@ func (t *target) Run() error {
 	err = t.htrun.run()
 
 	// do it after the `run()` to retain `restarted` marker on panic
-	fs.RemoveMarker(cmn.NodeRestartedMarker)
+	fs.RemoveMarker(fname.NodeRestartedMarker)
 	return err
 }
 
@@ -473,10 +474,10 @@ func (t *target) Stop(err error) {
 }
 
 func (t *target) checkRestarted() (fatalErr, writeErr error) {
-	if fs.MarkerExists(cmn.NodeRestartedMarker) {
+	if fs.MarkerExists(fname.NodeRestartedMarker) {
 		t.statsT.Add(stats.RestartCount, 1)
 	} else {
-		fatalErr, writeErr = fs.PersistMarker(cmn.NodeRestartedMarker)
+		fatalErr, writeErr = fs.PersistMarker(fname.NodeRestartedMarker)
 	}
 	return
 }
@@ -671,7 +672,7 @@ func (t *target) listObjects(w http.ResponseWriter, r *http.Request, bck *cluste
 	debug.Assert(resp.Status == http.StatusOK)
 	debug.Assert(resp.BckList.UUID != "")
 
-	if fs.MarkerExists(cmn.RebalanceMarker) || reb.IsActiveGFN() {
+	if fs.MarkerExists(fname.RebalanceMarker) || reb.IsActiveGFN() {
 		resp.BckList.Flags |= cmn.BckListFlagRebalance
 	}
 

diff --git a/ais/tests/maintain_test.go b/ais/tests/maintain_test.go
@@ -15,6 +15,7 @@ import (
 	"github.com/NVIDIA/aistore/cluster"
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
+	"github.com/NVIDIA/aistore/cmn/fname"
 	"github.com/NVIDIA/aistore/devtools/readers"
 	"github.com/NVIDIA/aistore/devtools/tassert"
 	"github.com/NVIDIA/aistore/devtools/tlog"
@@ -254,7 +255,7 @@ func TestMaintenanceDecommissionRebalance(t *testing.T) {
 func countVMDTargets(tsMpaths map[*cluster.Snode][]string) (total int) {
 	for _, mpaths := range tsMpaths {
 		for _, mpath := range mpaths {
-			if err := cos.Stat(filepath.Join(mpath, cmn.VmdFname)); err == nil {
+			if err := cos.Stat(filepath.Join(mpath, fname.Vmd)); err == nil {
 				total++
 				break
 			}

diff --git a/ais/tests/multiproxy_test.go b/ais/tests/multiproxy_test.go
@@ -22,6 +22,7 @@ import (
 	"github.com/NVIDIA/aistore/cluster"
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
+	"github.com/NVIDIA/aistore/cmn/fname"
 	"github.com/NVIDIA/aistore/cmn/jsp"
 	"github.com/NVIDIA/aistore/containers"
 	"github.com/NVIDIA/aistore/devtools/readers"
@@ -188,7 +189,7 @@ killRestore:
 	tassert.CheckFatal(t, err)
 
 	// Update local config ports.
-	localConfPath := filepath.Join(cfg.ConfigDir, "ais_local.json")
+	localConfPath := filepath.Join(cfg.ConfigDir, fname.PlaintextInitialConfig)
 	localConf := &cmn.LocalConfig{}
 	_, err = jsp.LoadMeta(localConfPath, localConf)
 	tassert.CheckFatal(t, err)

diff --git a/ais/utils.go b/ais/utils.go
@@ -18,6 +18,7 @@ import (
 	"github.com/NVIDIA/aistore/cluster"
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
+	"github.com/NVIDIA/aistore/cmn/fname"
 	"github.com/NVIDIA/aistore/cmn/k8s"
 )
 
@@ -243,14 +244,14 @@ func cleanupConfigDir(name string, keepInitialConfig bool) {
 }
 
 func writeShutdownMarker() {
-	markerDir := os.Getenv(env.AIS.ShutdownMarkerPath)
+	markerDir := os.Getenv(env.AIS.ShutdownMarkerDir)
 	if markerDir == "" {
 		if k8s.Detect() == nil {
 			glog.Warningf("marker directory not specified, skipping writing shutdown marker")
 		}
 		return
 	}
-	f, err := cos.CreateFile(filepath.Join(markerDir, cmn.ShutdownMarker))
+	f, err := cos.CreateFile(filepath.Join(markerDir, fname.ShutdownMarker))
 	if err != nil {
 		glog.Errorf("failed to create shutdown marker, %v", err)
 	}
@@ -260,8 +261,8 @@ func writeShutdownMarker() {
 }
 
 func deleteShutdownMarker() {
-	markerDir := os.Getenv(env.AIS.ShutdownMarkerPath)
+	markerDir := os.Getenv(env.AIS.ShutdownMarkerDir)
 	if markerDir != "" {
-		cos.RemoveFile(filepath.Join(markerDir, cmn.ShutdownMarker))
+		cos.RemoveFile(filepath.Join(markerDir, fname.ShutdownMarker))
 	}
 }
diff --git a/api/authn/config.go b/api/authn/config.go
@@ -17,8 +17,6 @@ import (
 type (
 	Config struct {
 		sync.RWMutex `list:"omit"` // for cmn.IterFields
-		Path         string        `json:"path"`
-		ConfDir      string        `json:"confdir"`
 		Log          LogConf       `json:"log"`
 		Net          NetConf       `json:"net"`
 		Server       ServerConf    `json:"auth"`