Skip to content
Permalink

Comparing changes

This is a direct comparison between two commits made in this repository or its related repositories. View the default comparison for this range or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: leil-io/saunafs
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: ec2c53dc217651a1dfd1e1ad757573b84c2c5fb8
Choose a base ref
..
head repository: leil-io/saunafs
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 80b37dd8822fd0656e239882a3f699ad6dd99f1b
Choose a head ref
Showing with 917 additions and 123 deletions.
  1. +28 −10 CMakeLists.txt
  2. +1 −1 Jenkinsfile
  3. +1 −1 configure
  4. +5 −0 doc/saunafs-admin.8.adoc
  5. +3 −0 doc/saunafs.1.adoc
  6. +10 −1 src/admin/main.cc
  7. +20 −17 src/chunkserver/bgjobs.cc
  8. +42 −0 src/common/version.h
  9. +2 −0 src/errors/sfserr.cc
  10. +5 −3 src/main/main.cc
  11. +5 −3 src/master/filesystem_operations.cc
  12. +96 −0 src/mount/acquired_files_last_time_used.h
  13. +3 −1 src/mount/fuse/main.cc
  14. +29 −6 src/mount/mastercomm.cc
  15. +8 −0 src/mount/mastercomm.h
  16. +107 −45 src/mount/readdata.cc
  17. +10 −3 src/mount/readdata.h
  18. +49 −22 src/mount/readdata_cache.h
  19. +11 −4 src/mount/sauna_client.cc
  20. +8 −0 src/mount/sauna_client.h
  21. +10 −2 src/tools/main.cc
  22. +9 −0 src/tools/tools_commands.cc
  23. +1 −0 src/tools/tools_commands.h
  24. +1 −1 tests/setup_machine.sh
  25. +71 −0 tests/test_suites/GaneshaTests/test_nfs_ganesha_allocate_large_file.sh
  26. +70 −0 tests/test_suites/GaneshaTests/test_nfs_ganesha_append_data_until_exhausting_space.sh
  27. +67 −0 tests/test_suites/GaneshaTests/test_nfs_ganesha_copy_file_without_enough_space.sh
  28. +74 −0 tests/test_suites/GaneshaTests/test_nfs_ganesha_truncate_file_to_big_size.sh
  29. +35 −1 tests/test_suites/ShortSystemTests/test_case_insensitive_filesystem.sh
  30. +21 −0 tests/test_suites/ShortSystemTests/test_read_cache_low_level_parallel_reads.sh
  31. 0 tests/test_suites/{MachineTests → SingleMachineTests}/test_chunkserver_advise_no_cache.sh
  32. 0 tests/test_suites/{MachineTests → SingleMachineTests}/test_ec_truncate_missing_parts.sh
  33. 0 tests/test_suites/{MachineTests → SingleMachineTests}/test_ec_writing_faulty_chunkservers.sh
  34. 0 tests/test_suites/{MachineTests → SingleMachineTests}/test_ec_writing_faulty_master.sh
  35. 0 ...test_suites/{MachineTests → SingleMachineTests}/test_ec_writing_faulty_master_and_chunkservers.sh
  36. +55 −0 tests/test_suites/SingleMachineTests/test_helgrind_basic.sh
  37. 0 tests/test_suites/{MachineTests → SingleMachineTests}/test_pread_eio_in_chunkserver_all_disks.sh
  38. 0 tests/test_suites/{MachineTests → SingleMachineTests}/test_pread_eio_in_chunkserver_single_disk.sh
  39. 0 tests/test_suites/{MachineTests → SingleMachineTests}/test_pread_eio_in_chunkserver_some_disks.sh
  40. 0 tests/test_suites/{MachineTests → SingleMachineTests}/test_pread_eio_nonheader_in_chunkserver.sh
  41. +59 −1 tests/tools/valgrind-helgrind.supp
  42. +1 −1 tests/tools/valgrind.sh
38 changes: 28 additions & 10 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -50,6 +50,8 @@ option(ENABLE_NFS_ACL_SUPPORT "Enable nfs-ganesha ACL support"
option(ENABLE_COMPILE_COMMANDS "Enable generation of compile_commands.json" ON)
option(ENABLE_PROMETHEUS "Enable prometheus metrics" ON)

option(GENERATE_GIT_INFO "Generate git info from current repository (for version commands)" ON)

# If enabled (default), the file compile_commands.json is generated in the build
# directory. Tools like clangd and vscode use this file for proper analysis.
set(CMAKE_EXPORT_COMPILE_COMMANDS ${ENABLE_COMPILE_COMMANDS})
@@ -188,16 +190,16 @@ endif()

project(saunafs)

if(NOT PACKAGE_VERSION)
set(DEFAULT_MIN_VERSION "4.6.0")
execute_process(COMMAND "git" "rev-parse" "--short" "HEAD"
OUTPUT_VARIABLE GIT_SHA1_SHORT RESULT_VARIABLE GIT_SHA1_SHORT_RETURN_VALUE
OUTPUT_STRIP_TRAILING_WHITESPACE)
execute_process(COMMAND "git" "rev-parse" "--abbrev-ref" "HEAD"
OUTPUT_VARIABLE GIT_BRANCH_NAME RESULT_VARIABLE GIT_BRANCH_NAME_RETURN_VALUE
OUTPUT_STRIP_TRAILING_WHITESPACE)
string(TIMESTAMP BUILD_TIMESTAMP "%y%m%d-%H%M%S")
set(PACKAGE_VERSION "${DEFAULT_MIN_VERSION}-${BUILD_TIMESTAMP}-unstable-${GIT_BRANCH_NAME}-${GIT_SHA1_SHORT}")
set(DEFAULT_MIN_VERSION "4.6.0")

string(TIMESTAMP BUILD_TIMESTAMP)
add_compile_definitions(BUILD_TIME="${BUILD_TIMESTAMP}")


if(VERSION_SUFFIX)
set(PACKAGE_VERSION "${DEFAULT_MIN_VERSION}${VERSION_SUFFIX}")
else()
set(PACKAGE_VERSION "${DEFAULT_MIN_VERSION}-dev")
endif()

string(REGEX MATCH "^([0-9]+)\\.([0-9]+)\\.([0-9]+)" PACKAGE_VERSION_MATCH ${PACKAGE_VERSION})
@@ -218,6 +220,22 @@ if(CMAKE_SYSTEM_NAME STREQUAL "SunOS")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -m64")
endif()

if(GIT_COMMIT)
add_compile_definitions(GIT_COMMIT="${GIT_COMMIT}")
elseif(GENERATE_GIT_INFO)
execute_process(COMMAND "git" "rev-parse" "HEAD"
OUTPUT_VARIABLE GIT_COMMIT OUTPUT_STRIP_TRAILING_WHITESPACE)
add_compile_definitions(GIT_COMMIT="${GIT_COMMIT}")
endif()

if(GIT_BRANCH)
add_compile_definitions(GIT_BRANCH="${GIT_BRANCH}")
elseif(GENERATE_GIT_INFO)
execute_process(COMMAND "git" "rev-parse" "--abbrev-ref" "HEAD"
OUTPUT_VARIABLE GIT_BRANCH OUTPUT_STRIP_TRAILING_WHITESPACE)
add_compile_definitions(GIT_BRANCH="${GIT_BRANCH}")
endif()

if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
set(LINUX ON)
set(UNIX ON)
2 changes: 1 addition & 1 deletion Jenkinsfile
Original file line number Diff line number Diff line change
@@ -50,7 +50,7 @@ pipeline {

stage('Run machine tests') {
steps {
sh ''' ./sfstests/sfstests --auth /etc/apt/auth.conf.d/ --suite MachineTests --workers 1 --multiplier 2'''
sh ''' ./sfstests/sfstests --auth /etc/apt/auth.conf.d/ --suite SingleMachineTests --workers 1 --multiplier 2'''
}
}

2 changes: 1 addition & 1 deletion configure
Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@ cmake .. \
-DENABLE_CLIENT_LIB=YES \
-DGSH_CAN_HOST_LOCAL_FS=YES \
-DENABLE_NFS_GANESHA=YES \
-DPACKAGE_VERSION="${version}" \
-DVERSION_SUFFIX="${VERSION_SUFFIX:-"-unofficial"}" \
-DENABLE_URAFT=YES \
-DENABLE_POLONAISE=OFF \
-DCMAKE_CXX_FLAGS="-Wno-error=maybe-uninitialized"
5 changes: 5 additions & 0 deletions doc/saunafs-admin.8.adoc
Original file line number Diff line number Diff line change
@@ -148,6 +148,11 @@ chunks with number of copies specified in the label to replicate/delete. +
Dumps the configuration files of the master server +
Authentication with the admin password is required +

== OPTIONS

*-v*, *--version*::
Print version and exit

== REPORTING BUGS

Report bugs to the Github repository <https://github.com/leil/saunafs> as an
3 changes: 3 additions & 0 deletions doc/saunafs.1.adoc
Original file line number Diff line number Diff line change
@@ -97,6 +97,9 @@ Other options:
*-r*::
This option enables recursive mode.

*-v*, *--version*::
Print version and exit

== INHERITANCE

When new object is created in SaunaFS, attributes such as goal, trashtime and
11 changes: 10 additions & 1 deletion src/admin/main.cc
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@

#include <algorithm>
#include <iostream>
#include <ostream>

#include "admin/chunk_health_command.h"
#include "admin/info_command.h"
@@ -50,6 +51,7 @@
#include "protocol/SFSCommunication.h"
#include "errors/sfserr.h"
#include "common/sockets.h"
#include <common/version.h>

int main(int argc, const char** argv) {
std::vector<const SaunaFsAdminCommand*> allCommands = {
@@ -86,7 +88,11 @@ int main(int argc, const char** argv) {
#ifdef _WIN32
socketinit();
#endif
command_name = argv[1];
std::string command_name = argv[1];
if (command_name == "-v" || command_name == "--version") {
std::cout << common::version() << '\n';
return 0;
}
std::vector<std::string> arguments(argv + 2, argv + argc);
for (auto command : allCommands) {
if (command->name() == command_name) {
@@ -115,6 +121,9 @@ int main(int argc, const char** argv) {
std::cerr << ex.message() << std::endl;
std::cerr << "Usage:\n";
std::cerr << " " << argv[0] << " COMMAND [OPTIONS...] [ARGUMENTS...]\n\n";

std::cerr << "Options:\n";
std::cerr << "-v --version: Print version\n\n";
if (command_name.empty()) {
std::cerr << "Available COMMANDs:\n\n";
for (auto command : allCommands) {
37 changes: 20 additions & 17 deletions src/chunkserver/bgjobs.cc
Original file line number Diff line number Diff line change
@@ -31,6 +31,7 @@
#include <unistd.h>
#include <cassert>
#include <cstdint>
#include <mutex>

#include "chunkserver/chunk_replicator.h"
#include "chunkserver/hddspacemgr.h"
@@ -144,8 +145,8 @@ struct jobpool {
int rpipe,wpipe;
uint8_t workers;
pthread_t *workerthreads;
pthread_mutex_t pipelock;
pthread_mutex_t jobslock;
std::mutex pipeMutex;
std::mutex jobsMutex;
std::unique_ptr<ProducerConsumerQueue> jobsQueue;
std::unique_ptr<ProducerConsumerQueue> statusQueue;
job* jobhash[JHASHSIZE];
@@ -155,8 +156,6 @@ struct jobpool {
: rpipe(rpipe), wpipe(wpipe), workers(workers) {
workerthreads = (pthread_t *)malloc(sizeof(pthread_t) * workers);
passert(workerthreads);
zassert(pthread_mutex_init(&(pipelock), nullptr));
zassert(pthread_mutex_init(&(jobslock), nullptr));

jobsQueue = std::make_unique<ProducerConsumerQueue>(maxJobs);
statusQueue = std::make_unique<ProducerConsumerQueue>();
@@ -167,28 +166,29 @@ struct jobpool {

static inline void job_send_status(jobpool *jp, uint32_t jobid, uint8_t status) {
TRACETHIS2(jobid, (int)status);
zassert(pthread_mutex_lock(&(jp->pipelock)));

std::lock_guard pipeLockGuard(jp->pipeMutex);

if (jp->statusQueue->isEmpty()) { // first status
eassert(write(jp->wpipe,&status,1)==1); // write anything to wake up select
}
jp->statusQueue->put(jobid, status, nullptr, 1);
zassert(pthread_mutex_unlock(&(jp->pipelock)));
}

static inline int job_receive_status(jobpool *jp,uint32_t *jobid,uint8_t *status) {
TRACETHIS();
uint32_t qstatus;
zassert(pthread_mutex_lock(&(jp->pipelock)));

std::lock_guard pipeLockGuard(jp->pipeMutex);

jp->statusQueue->get(jobid, &qstatus, nullptr, nullptr);
*status = qstatus;
PRINTTHIS(*jobid);
PRINTTHIS((int)*status);
if (jp->statusQueue->isEmpty()) {
eassert(read(jp->rpipe,&qstatus,1)==1); // make pipe empty
zassert(pthread_mutex_unlock(&(jp->pipelock)));
return 0; // last element
}
zassert(pthread_mutex_unlock(&(jp->pipelock)));
return 1; // not last
}

@@ -206,11 +206,13 @@ void* job_worker(void *th_arg) {
uint32_t jobid;
uint32_t op;

std::unique_lock jobsUniqueLock(jp->jobsMutex, std::defer_lock);

for (;;) {
jp->jobsQueue->get(&jobid, &op, &jptrarg, nullptr);
jptr = (job*)jptrarg;
PRINTTHIS(op);
zassert(pthread_mutex_lock(&(jp->jobslock)));
jobsUniqueLock.lock();
if (jptr!=NULL) {
jstate=jptr->jstate;
if (jptr->jstate==JSTATE_ENABLED) {
@@ -219,7 +221,7 @@ void* job_worker(void *th_arg) {
} else {
jstate=JSTATE_DISABLED;
}
zassert(pthread_mutex_unlock(&(jp->jobslock)));
jobsUniqueLock.unlock();
switch (op) {
case OP_INVAL:
status = SAUNAFS_ERROR_EINVAL;
@@ -406,7 +408,8 @@ void job_pool_disable_and_change_callback_all(void *jpool,void (*callback)(uint8
uint32_t jhpos;
job *jptr;

zassert(pthread_mutex_lock(&(jp->jobslock)));
std::lock_guard jobsLockGuard(jp->jobsMutex);

for (jhpos = 0 ; jhpos<JHASHSIZE ; jhpos++) {
for (jptr = jp->jobhash[jhpos] ; jptr ; jptr=jptr->next) {
if (jptr->jstate==JSTATE_ENABLED) {
@@ -415,21 +418,23 @@ void job_pool_disable_and_change_callback_all(void *jpool,void (*callback)(uint8
jptr->callback=callback;
}
}
zassert(pthread_mutex_unlock(&(jp->jobslock)));
}

void job_pool_disable_job(void *jpool,uint32_t jobid) {
TRACETHIS();
jobpool* jp = (jobpool*)jpool;
uint32_t jhpos = JHASHPOS(jobid);
job *jptr;

std::unique_lock jobsUniqueLock(jp->jobsMutex, std::defer_lock);

for (jptr = jp->jobhash[jhpos] ; jptr ; jptr=jptr->next) {
if (jptr->jobid==jobid) {
zassert(pthread_mutex_lock(&(jp->jobslock)));
jobsUniqueLock.lock();
if (jptr->jstate==JSTATE_ENABLED) {
jptr->jstate=JSTATE_DISABLED;
}
zassert(pthread_mutex_unlock(&(jp->jobslock)));
jobsUniqueLock.unlock();
}
}
}
@@ -497,8 +502,6 @@ void job_pool_delete(void *jpool) {

jp->jobsQueue.reset();
jp->statusQueue.reset();
zassert(pthread_mutex_destroy(&(jp->pipelock)));
zassert(pthread_mutex_destroy(&(jp->jobslock)));
free(jp->workerthreads);
close(jp->rpipe);
close(jp->wpipe);
42 changes: 42 additions & 0 deletions src/common/version.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
Copyright 2024 Leil Storage OÜ
This file is part of SaunaFS.
SaunaFS is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, version 3.
SaunaFS is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with SaunaFS. If not, see <http://www.gnu.org/licenses/>.
*/

#include "common/platform.h"

#include <string>

namespace common {

inline std::string version() {
return "Version: " \
SAUNAFS_PACKAGE_VERSION "\n"
"Build time: " BUILD_TIME "\n"
#ifdef GIT_COMMIT
"Git commit: " GIT_COMMIT "\n"
#else
"Git commit: N/A\n"
#endif
#ifdef GIT_BRANCH
"Git branch: " GIT_BRANCH
#else
"Git branch: N/A"
#endif
;
}

} // namespace common
2 changes: 2 additions & 0 deletions src/errors/sfserr.cc
Original file line number Diff line number Diff line change
@@ -60,6 +60,8 @@ int saunafs_error_conv(uint8_t status) {
return ENOTEMPTY;
case SAUNAFS_ERROR_IO:
return EIO;
case SAUNAFS_ERROR_NOSPACE:
return ENOSPC;
case SAUNAFS_ERROR_EROFS:
return EROFS;
case SAUNAFS_ERROR_QUOTA:
8 changes: 5 additions & 3 deletions src/main/main.cc
Original file line number Diff line number Diff line change
@@ -42,21 +42,23 @@
#include <algorithm>
#include <atomic>
#include <fstream>
#include <iostream>
#include <list>
#include <memory>

#include "config/cfg.h"
#include "common/crc.h"
#include "common/cwrap.h"
#include "common/event_loop.h"
#include "common/exceptions.h"
#include "common/exit_status.h"
#include "common/event_loop.h"
#include "common/main.h"
#include "common/massert.h"
#include "common/setup.h"
#include "slogger/slogger.h"
#include "common/time_utils.h"
#include "common/version.h"
#include "protocol/SFSCommunication.h"
#include "slogger/slogger.h"

#if defined(SAUNAFS_HAVE_MLOCKALL)
# if defined(SAUNAFS_HAVE_SYS_MMAN_H)
@@ -812,7 +814,7 @@ int main(int argc,char **argv) {
switch(ch) {
case 'v':
case 'V':
printf("version: %s\n",SAUNAFS_PACKAGE_VERSION);
std::cout << common::version() << '\n';
return 0;
case 'd':
gRunAsDaemon = false;
8 changes: 5 additions & 3 deletions src/master/filesystem_operations.cc
Original file line number Diff line number Diff line change
@@ -447,9 +447,7 @@ uint8_t fs_lookup(const FsContext &context, uint32_t parent, const HString &name
return SAUNAFS_ERROR_EINVAL;
}

auto parentNode = static_cast<FSNodeDirectory *>(wd);
parentNode->case_insensitive = context.sesflags() & SESFLAG_CASEINSENSITIVE;
FSNode *child = fsnodes_lookup(parentNode, name);
FSNode *child = fsnodes_lookup(static_cast<FSNodeDirectory*>(wd), name);
if (!child) {
return SAUNAFS_ERROR_ENOENT;
}
@@ -961,6 +959,8 @@ uint8_t fs_mknod(const FsContext &context, uint32_t parent, const HString &name,
fsnodes_quota_exceeded_dir(wd, {{QuotaResource::kInodes, 1}})) {
return SAUNAFS_ERROR_QUOTA;
}
static_cast<FSNodeDirectory *>(wd)->case_insensitive =
context.sesflags() & SESFLAG_CASEINSENSITIVE;
p = fsnodes_create_node(ts, static_cast<FSNodeDirectory*>(wd), name, type, mode, umask, context.uid(), context.gid(), 0,
AclInheritance::kInheritAcl);
if (type == FSNode::kBlockDev || type == FSNode::kCharDev) {
@@ -1006,6 +1006,8 @@ uint8_t fs_mkdir(const FsContext &context, uint32_t parent, const HString &name,
fsnodes_quota_exceeded_dir(wd, {{QuotaResource::kInodes, 1}})) {
return SAUNAFS_ERROR_QUOTA;
}
static_cast<FSNodeDirectory *>(wd)->case_insensitive =
context.sesflags() & SESFLAG_CASEINSENSITIVE;
p = fsnodes_create_node(ts, static_cast<FSNodeDirectory *>(wd), name, FSNode::kDirectory, mode,
umask, context.uid(), context.gid(), copysgid, AclInheritance::kInheritAcl);
*inode = p->id;
Loading