feat: Allow master to load legacy metadata (#144)

feat: Allow master to load legacy metadata

src/common/metadata.cc

@@ -41,6 +41,9 @@ const char kMetadataFilename[] = METADATA_FILENAME_TEMPL;
 const char kMetadataTmpFilename[] = METADATA_FILENAME_TEMPL ".tmp";
 const char kMetadataEmergencyFilename[] = METADATA_FILENAME_TEMPL ".emergency";
 #undef METADATA_FILENAME_TEMPL
+#define METADATA_LEGACY_FILENAME_TEMPL "metadata.mfs"
+const char kMetadataLegacyFilename[] = METADATA_LEGACY_FILENAME_TEMPL;
+#undef METADATA_LEGACY_FILENAME_TEMPL
 #define METADATA_ML_FILENAME_TEMPL "metadata_ml.sfs"
 const char kMetadataMlFilename[] = METADATA_ML_FILENAME_TEMPL;
 const char kMetadataMlTmpFilename[] = METADATA_ML_FILENAME_TEMPL ".tmp";
@@ -92,9 +95,15 @@ uint64_t metadataGetVersion(const std::string& file) {
 	std::string signature = std::string(chkbuff, 8);
 	std::string sfsSignature = std::string(SFSSIGNATURE "M 2.9");
 	std::string sauSignature = std::string(SAUSIGNATURE "M 2.9");
+	std::string legacySignature = std::string("LIZM 2.9");
 
 	if (signature == sfsSignature || signature == sauSignature) {
 		memcpy(eofmark,"[SFS EOF MARKER]",16);
+	} else if (signature == legacySignature) {
+		safs_pretty_syslog(LOG_WARNING,
+		                   "Legacy metadata section header %s, was detected in the metadata file %s",
+		                   legacySignature.c_str(), file.c_str());
+		memcpy(eofmark,"[MFS EOF MARKER]",16);
 	} else {
 		close(fd);
 		throw MetadataCheckException("Bad EOF MARKER in the metadata file.");

src/common/metadata.h

@@ -31,6 +31,7 @@
 
 extern const char kMetadataFilename[];
 extern const char kMetadataTmpFilename[];
+extern const char kMetadataLegacyFilename[];
 extern const char kMetadataMlFilename[];
 extern const char kMetadataMlTmpFilename[];
 extern const char kMetadataEmergencyFilename[];

src/master/filesystem.cc

@@ -216,9 +216,28 @@ int fs_loadall(void) {
 	changelogsMigrateFrom_1_6_29("changelog");
 	if (fs::exists(kMetadataTmpFilename)) {
 		throw MetadataFsConsistencyException(
-				"temporary metadata file (" + std::string(kMetadataTmpFilename) + ") exists, metadata directory is in dirty state");
+		    "temporary metadata file (" + std::string(kMetadataTmpFilename) + ") exists,"
+		    " metadata directory is in dirty state");
 	}
-	if ((metadataserver::isMaster()) && !fs::exists(kMetadataFilename)) {
+	std::string metadataFile;
+	bool metadataFileExists = fs::exists(kMetadataFilename);
+	bool legacyMetadataFileExists = fs::exists(kMetadataLegacyFilename);
+	if (metadataFileExists) {
+		metadataFile = kMetadataFilename;
+	}
+	if(metadataFileExists && legacyMetadataFileExists) {
+		metadataFile = kMetadataFilename;
+		safs_pretty_syslog(LOG_WARNING, "There are two metadata files in the data path: %s and %s."
+		                   " Please remove the legacy one (%s) to avoid damage to your storage.",
+		                   kMetadataFilename, kMetadataLegacyFilename, kMetadataLegacyFilename);
+	}
+	if (!metadataFileExists && legacyMetadataFileExists) {
+		metadataFile = kMetadataLegacyFilename;
+		safs_pretty_syslog(LOG_WARNING, "Only Legacy metadata file %s found and will be loaded instead."
+		                   " You should delete legacy metadata %s on next restart after new metadata %s is created ",
+		                   metadataFile.c_str(), kMetadataLegacyFilename, kMetadataFilename);
+	}
+	if (metadataserver::isMaster() && !metadataFileExists && !legacyMetadataFileExists) {
 		fs_unlock();
 		std::string currentPath = fs::getCurrentWorkingDirectoryNoThrow();
 		throw FilesystemException("can't open metadata file "+ currentPath + "/" + kMetadataFilename
@@ -229,7 +248,7 @@ int fs_loadall(void) {
 
 	{
 		auto scopedTimer = util::ScopedTimer("metadata load time");
-		fs_loadall(kMetadataFilename, 0);
+		fs_loadall(metadataFile, 0);
 	}
 
 	bool autoRecovery = fs_can_do_auto_recovery();

src/master/filesystem_store.cc

@@ -1011,8 +1011,11 @@ static const std::vector<MetadataSection> kMetadataSections = {
 bool isEndOfMetadata(const uint8_t *sectionPtr) {
 	static constexpr std::string_view kMetadataTrailer("[" SFSSIGNATURE
 	                                                   " EOF MARKER]");
-	return memcmp(sectionPtr, kMetadataTrailer.data(),
-	              ::kMetadataSectionHeaderSize) == kOpSuccess;
+	static constexpr std::string_view kMetadataLegacyTrailer("[MFS EOF MARKER]");
+	return ((memcmp(sectionPtr, kMetadataTrailer.data(),
+	              ::kMetadataSectionHeaderSize) == kOpSuccess) ||
+	        (memcmp(sectionPtr, kMetadataLegacyTrailer.data(),
+	              ::kMetadataSectionHeaderSize) == kOpSuccess));
 }
 
 
@@ -1329,6 +1332,7 @@ bool isNewMetadataFile([[maybe_unused]]const uint8_t *headerPtr) {
 bool checkMetadataSignature(const std::shared_ptr<MemoryMappedFile> &metadataFile) {
 	static constexpr std::string_view kMetadataHeaderNewV2_9(SFSSIGNATURE "M 2.9");
 	static constexpr std::string_view kMetadataHeaderOldV2_9(SAUSIGNATURE "M 2.9");
+	static constexpr std::string_view kMetadataHeaderLegacy("LIZM 2.9");
 	static constexpr uint8_t kMetadataHeaderSize = 8;
 	size_t kMetadataHeaderOffset{0};
 	uint8_t *headerPtr;
@@ -1343,7 +1347,8 @@ bool checkMetadataSignature(const std::shared_ptr<MemoryMappedFile> &metadataFil
 		return false;
 	}
 	if ((memcmp(headerPtr, kMetadataHeaderNewV2_9.data(), kMetadataHeaderSize) != kOpSuccess) &&
-	    (memcmp(headerPtr, kMetadataHeaderOldV2_9.data(), kMetadataHeaderSize) != kOpSuccess)) {
+	    (memcmp(headerPtr, kMetadataHeaderOldV2_9.data(), kMetadataHeaderSize) != kOpSuccess) &&
+	    (memcmp(headerPtr, kMetadataHeaderLegacy.data(), kMetadataHeaderSize) != kOpSuccess)) {
 		throw MetadataConsistencyException("wrong metadata header version");
 	}
 	return true;

src/master/matoclserv.cc

@@ -674,6 +674,7 @@ void matoclserv_store_sessions() {
 	}
 }
 
+#define MFSSIGNATURE "MFS"
 int matoclserv_load_sessions() {
 	session *asesdata;
 	uint32_t ileng;
@@ -700,19 +701,23 @@ int matoclserv_load_sessions() {
 		fclose(fd);
 		return -1;
 	}
-	if (memcmp(hdr,SFSSIGNATURE "S 1.5",8)==0) {
+	if (memcmp(hdr, SFSSIGNATURE "S 1.5", 8) == 0 ||
+	    memcmp(hdr, MFSSIGNATURE "S 1.5", 8) == 0) {
 		mapalldata = 0;
 		goaltrashdata = 0;
 		statsinfile = 16;
-	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\001",8)==0) {
+	} else if (memcmp(hdr, SFSSIGNATURE "S \001\006\001", 8) == 0 ||
+	           memcmp(hdr, MFSSIGNATURE "S \001\006\001", 8) == 0) {
 		mapalldata = 1;
 		goaltrashdata = 0;
 		statsinfile = 16;
-	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\002",8)==0) {
+	} else if (memcmp(hdr, SFSSIGNATURE "S \001\006\002", 8) == 0 ||
+	           memcmp(hdr, MFSSIGNATURE "S \001\006\002", 8) == 0) {
 		mapalldata = 1;
 		goaltrashdata = 0;
 		statsinfile = 21;
-	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\003",8)==0) {
+	} else if (memcmp(hdr, SFSSIGNATURE "S \001\006\003", 8) == 0 ||
+	           memcmp(hdr, MFSSIGNATURE "S \001\006\003", 8) == 0) {
 		mapalldata = 1;
 		goaltrashdata = 0;
 		if (fread(hdr,2,1,fd)!=1) {
@@ -722,7 +727,8 @@ int matoclserv_load_sessions() {
 		}
 		ptr = hdr;
 		statsinfile = get16bit(&ptr);
-	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\004",8)==0) {
+	} else if (memcmp(hdr, SFSSIGNATURE "S \001\006\004", 8) == 0 ||
+	           memcmp(hdr, MFSSIGNATURE "S \001\006\004", 8) == 0) {
 		mapalldata = 1;
 		goaltrashdata = 1;
 		if (fread(hdr,2,1,fd)!=1) {
@@ -815,6 +821,7 @@ int matoclserv_load_sessions() {
 	fclose(fd);
 	return 1;
 }
+#undef MFSSIGNATURE
 
 int matoclserv_insert_openfile(session* cr,uint32_t inode) {
 	filelist *ofptr,**ofpptr;

tests/test_suites/ShortSystemTests/test_metadata_load_legacy.sh

@@ -0,0 +1,77 @@
+timeout_set 30 seconds
+master_cfg+="MAGIC_DEBUG_LOG = $TEMP_DIR/syslog|LOG_FLUSH_ON=DEBUG"
+touch "$TEMP_DIR/syslog"
+
+CHUNKSERVERS=0 \
+	MASTERSERVERS=2 \
+	USE_RAMDISK="YES" \
+	MASTER_EXTRA_CONFIG="${master_cfg}" \
+	MOUNT_EXTRA_CONFIG="sfscachemode=NEVER" \
+	AUTO_SHADOW_MASTER="YES" \
+	setup_local_empty_saunafs info
+
+metadata_file="${info[master0_data_path]}/metadata.sfs"
+
+ACTUAL_HEADER_TAG="SFSM 2.9"
+ACTUAL_TRAILER_TAG="[SFS EOF MARKER]"
+LEGACY_HEADER_TAG="LIZM 2.9"
+LEGACY_TRAILER_TAG="[MFS EOF MARKER]"
+
+# Add metalogger service to the configuration
+saunafs_metalogger_daemon start
+
+# Access mountpoint to create metadata
+cd "${info[mount0]}"
+for i in {1..100}; do
+	touch file$i
+	mkdir dir$i
+done
+
+# Exit mount point and save metadata by stopping master server
+cd "${TEMP_DIR}"
+assert_success saunafs_master_n 0 stop
+assert_success saunafs_master_n 1 stop
+
+# Expected Values
+assert_equals "$(head -c8 ${metadata_file})" "${ACTUAL_HEADER_TAG}"
+assert_equals "$(tail -c16 ${metadata_file})" "${ACTUAL_TRAILER_TAG}"
+
+# Modify metadata signature for legacy and reload metadata by starting master
+echo -n "LIZ" | dd of=${metadata_file} bs=1 seek=0 count=3 conv=notrunc
+echo -n "MFS" | dd of=${metadata_file} bs=1 seek=$(( $(stat --print="%s" ${metadata_file}) - 15)) count=3 conv=notrunc
+assert_equals "$(head -c8 ${metadata_file})" "${LEGACY_HEADER_TAG}"
+assert_equals "$(tail -c16 ${metadata_file})" "${LEGACY_TRAILER_TAG}"
+
+mv -iv "${info[master0_data_path]}/metadata.sfs" "${info[master0_data_path]}/metadata.mfs"
+assertlocal_file_exists "${info[master0_data_path]}/metadata.mfs"
+
+# Start master server to reload metadata legacy format
+assertlocal_success saunafs_master_n 0 start
+# Restart master server to load metadata in new format when both are present
+assertlocal_success saunafs_master_n 0 restart
+assertlocal_file_exists "${info[master0_data_path]}/metadata.sfs"
+assertlocal_file_exists "${info[master0_data_path]}/metadata.mfs"
+
+# Shadows has not  synchronize metadata at this point
+assertlocal_file_not_exists "${info[master1_data_path]}/metadata.sfs"
+#start shadow server to synchronize metadata
+assert_success saunafs_master_n 1 start
+assert_eventually "saunafs_shadow_synchronized 1"
+assertlocal_file_exists "${info[master1_data_path]}/metadata.sfs"
+
+# Stop master server to save metadata in new format
+assert_success saunafs_master_n 0 stop
+
+# Synchronized metadata in shadows has proper Tags
+SHADOW_METADATA_FILE="${info[master1_data_path]}/metadata.sfs"
+assert_equals "$(head -c8 ${SHADOW_METADATA_FILE})" "${ACTUAL_HEADER_TAG}"
+assert_equals "$(tail -c16 ${SHADOW_METADATA_FILE})" "${ACTUAL_TRAILER_TAG}"
+
+# Stop metalogger service and verify created changelogs
+saunafs_metalogger_daemon stop
+assertlocal_file_exists "${info[master0_data_path]}/changelog_ml.sfs"
+
+# Expected Values after reload
+metadata_file="${info[master0_data_path]}/metadata.sfs"
+assert_equals "$(head -c8 ${metadata_file})" "${ACTUAL_HEADER_TAG}"
+assert_equals "$(tail -c16 ${metadata_file})" "${ACTUAL_TRAILER_TAG}"