Skip to content

Commit

Permalink
Merge pull request s3fs-fuse#710 from orozery/disk_space_reservation
Browse files Browse the repository at this point in the history 
add disk space reservation
  • Loading branch information
@ggtakec
ggtakec authored Mar 4, 2018
2 parents b9c9de7 + 8b657ee commit 45c7ea9
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 56 deletions.
2 changes: 1 addition & 1 deletion doc/man/s3fs.1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ number of one part size in multipart uploading request.
The default size is 10MB(10485760byte), minimum value is 5MB(5242880byte).
Specify number of MB and over 5(MB).
.TP
\fB\-o\fR ensure_diskfree(default the same as multipart_size value)
\fB\-o\fR ensure_diskfree(default 0)
sets MB to ensure disk free space. This option means the threshold of free space size on disk which is used for the cache file by s3fs.
s3fs makes file for downloading, and uploading and caching files.
If the disk free space is smaller than this value, s3fs do not use diskspace as possible in exchange for the performance.
Expand Down
143 changes: 94 additions & 49 deletions src/fdcache.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1429,7 +1429,7 @@ int FdEntity::NoCacheCompleteMultipartPost(void)

int FdEntity::RowFlush(const char* tpath, bool force_sync)
{
int result;
int result = 0;

S3FS_PRN_INFO3("[tpath=%s][path=%s][fd=%d]", SAFESTRPTR(tpath), path.c_str(), fd);

Expand All @@ -1448,10 +1448,12 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
if(0 < restsize){
if(0 == upload_id.length()){
// check disk space
if(FdManager::IsSafeDiskSpace(NULL, restsize)){
if(ReserveDiskSpace(restsize)){
// enough disk space
// Load all uninitialized area
if(0 != (result = Load())){
result = Load();
FdManager::get()->FreeReservedDiskSpace(restsize);
if(0 != result){
S3FS_PRN_ERR("failed to upload all area(errno=%d)", result);
return static_cast<ssize_t>(result);
}
Expand Down Expand Up @@ -1554,17 +1556,39 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
return result;
}

// [NOTICE]
// Need to lock before calling this method.
bool FdEntity::ReserveDiskSpace(size_t size)
{
if(FdManager::get()->ReserveDiskSpace(size)){
return true;
}

if(!is_modify){
// try to clear all cache for this fd.
pagelist.Init(pagelist.Size(), false);
if(-1 == ftruncate(fd, 0) || -1 == ftruncate(fd, pagelist.Size())){
S3FS_PRN_ERR("failed to truncate temporary file(%d).", fd);
return false;
}

if(FdManager::get()->ReserveDiskSpace(size)){
return true;
}
}

FdManager::get()->CleanupCacheDir();

return FdManager::get()->ReserveDiskSpace(size);
}

ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)
{
S3FS_PRN_DBG("[path=%s][fd=%d][offset=%jd][size=%zu]", path.c_str(), fd, (intmax_t)start, size);

if(-1 == fd){
return -EBADF;
}
// check if not enough disk space left BEFORE locking fd
if(FdManager::IsCacheDir() && !FdManager::IsSafeDiskSpace(NULL, size)){
FdManager::get()->CleanupCacheDir();
}
AutoLock auto_lock(&fdent_lock);

if(force_load){
Expand All @@ -1575,23 +1599,6 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)

// check disk space
if(0 < pagelist.GetTotalUnloadedPageSize(start, size)){
if(!FdManager::IsSafeDiskSpace(NULL, size)){
// [NOTE]
// If the area of this entity fd used can be released, try to do it.
// But If file data is updated, we can not even release of fd.
// Fundamentally, this method will fail as long as the disk capacity
// is not ensured.
//
if(!is_modify){
// try to clear all cache for this fd.
pagelist.Init(pagelist.Size(), false);
if(-1 == ftruncate(fd, 0) || -1 == ftruncate(fd, pagelist.Size())){
S3FS_PRN_ERR("failed to truncate temporary file(%d).", fd);
return -ENOSPC;
}
}
}

// load size(for prefetch)
size_t load_size = size;
if(static_cast<size_t>(start + size) < pagelist.Size()){
Expand All @@ -1603,9 +1610,25 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)
load_size = static_cast<size_t>(pagelist.Size() - start);
}
}

if(!ReserveDiskSpace(load_size)){
S3FS_PRN_WARN("could not reserve disk space for pre-fetch download");
load_size = size;
if(!ReserveDiskSpace(load_size)){
S3FS_PRN_ERR("could not reserve disk space for pre-fetch download");
return -ENOSPC;
}
}

// Loading
int result;
if(0 < size && 0 != (result = Load(start, load_size))){
int result = 0;
if(0 < size){
result = Load(start, load_size);
}

FdManager::get()->FreeReservedDiskSpace(load_size);

if(0 != result){
S3FS_PRN_ERR("could not download. start(%jd), size(%zu), errno(%d)", (intmax_t)start, size, result);
return -EIO;
}
Expand Down Expand Up @@ -1642,17 +1665,21 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
pagelist.SetPageLoadedStatus(static_cast<off_t>(pagelist.Size()), static_cast<size_t>(start) - pagelist.Size(), false);
}

int result;
int result = 0;
ssize_t wsize;

if(0 == upload_id.length()){
// check disk space
size_t restsize = pagelist.GetTotalUnloadedPageSize(0, start) + size;
if(FdManager::IsSafeDiskSpace(NULL, restsize)){
if(ReserveDiskSpace(restsize)){
// enough disk space

// Load uninitialized area which starts from 0 to (start + size) before writing.
if(0 < start && 0 != (result = Load(0, static_cast<size_t>(start)))){
if(0 < start){
result = Load(0, static_cast<size_t>(start));
}
FdManager::get()->FreeReservedDiskSpace(restsize);
if(0 != result){
S3FS_PRN_ERR("failed to load uninitialized area before writing(errno=%d)", result);
return static_cast<ssize_t>(result);
}
Expand Down Expand Up @@ -1750,6 +1777,7 @@ void FdEntity::CleanupCache()
FdManager FdManager::singleton;
pthread_mutex_t FdManager::fd_manager_lock;
pthread_mutex_t FdManager::cache_cleanup_lock;
pthread_mutex_t FdManager::reserved_diskspace_lock;
bool FdManager::is_lock_init(false);
string FdManager::cache_dir("");
bool FdManager::check_cache_dir_exist(false);
Expand Down Expand Up @@ -1901,19 +1929,7 @@ bool FdManager::CheckCacheDirExist(void)
size_t FdManager::SetEnsureFreeDiskSpace(size_t size)
{
size_t old = FdManager::free_disk_space;
if(0 == size){
if(0 == FdManager::free_disk_space){
FdManager::free_disk_space = static_cast<size_t>(S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount());
}
}else{
if(0 == FdManager::free_disk_space){
FdManager::free_disk_space = max(size, static_cast<size_t>(S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount()));
}else{
if(static_cast<size_t>(S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount()) <= size){
FdManager::free_disk_space = size;
}
}
}
FdManager::free_disk_space = size;
return old;
}

Expand Down Expand Up @@ -1957,6 +1973,7 @@ FdManager::FdManager()
try{
pthread_mutex_init(&FdManager::fd_manager_lock, NULL);
pthread_mutex_init(&FdManager::cache_cleanup_lock, NULL);
pthread_mutex_init(&FdManager::reserved_diskspace_lock, NULL);
FdManager::is_lock_init = true;
}catch(exception& e){
FdManager::is_lock_init = false;
Expand All @@ -1980,6 +1997,7 @@ FdManager::~FdManager()
try{
pthread_mutex_destroy(&FdManager::fd_manager_lock);
pthread_mutex_destroy(&FdManager::cache_cleanup_lock);
pthread_mutex_destroy(&FdManager::reserved_diskspace_lock);
}catch(exception& e){
S3FS_PRN_CRIT("failed to init mutex");
}
Expand Down Expand Up @@ -2181,17 +2199,22 @@ bool FdManager::ChangeEntityToTempPath(FdEntity* ent, const char* path)

void FdManager::CleanupCacheDir()
{
if (!FdManager::IsCacheDir()) {
S3FS_PRN_INFO("cache cleanup requested");

if(!FdManager::IsCacheDir()){
return;
}

AutoLock auto_lock(&FdManager::cache_cleanup_lock, true);
AutoLock auto_lock_no_wait(&FdManager::cache_cleanup_lock, true);

if (!auto_lock.isLockAcquired()) {
return;
if(auto_lock_no_wait.isLockAcquired()){
S3FS_PRN_INFO("cache cleanup started");
CleanupCacheDirInternal("");
S3FS_PRN_INFO("cache cleanup ended");
}else{
// wait for other thread to finish cache cleanup
AutoLock auto_lock(&FdManager::cache_cleanup_lock);
}

CleanupCacheDirInternal("");
}

void FdManager::CleanupCacheDirInternal(const std::string &path)
Expand Down Expand Up @@ -2224,16 +2247,38 @@ void FdManager::CleanupCacheDirInternal(const std::string &path)
}else{
FdEntity* ent;
if(NULL == (ent = FdManager::get()->Open(next_path.c_str(), NULL, -1, -1, false, true, true))){
S3FS_PRN_DBG("skipping locked file: %s", next_path.c_str());
continue;
}

ent->CleanupCache();
if(ent->IsMultiOpened()){
S3FS_PRN_DBG("skipping opened file: %s", next_path.c_str());
}else{
ent->CleanupCache();
S3FS_PRN_DBG("cleaned up: %s", next_path.c_str());
}
Close(ent);
}
}
closedir(dp);
}

bool FdManager::ReserveDiskSpace(size_t size)
{
AutoLock auto_lock(&FdManager::reserved_diskspace_lock);
if(IsSafeDiskSpace(NULL, size)){
free_disk_space += size;
return true;
}
return false;
}

void FdManager::FreeReservedDiskSpace(size_t size)
{
AutoLock auto_lock(&FdManager::reserved_diskspace_lock);
free_disk_space -= size;
}

/*
* Local variables:
* tab-width: 4
Expand Down
6 changes: 5 additions & 1 deletion src/fdcache.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ class FdEntity

void Close(void);
bool IsOpen(void) const { return (-1 != fd); }
bool IsMultiOpened(void) const { return refcnt > 1; }
int Open(headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool no_fd_lock_wait = false);
bool OpenAndLoadAll(headers_t* pmeta = NULL, size_t* size = NULL, bool force_load = false);
int Dup(bool no_fd_lock_wait = false);
Expand Down Expand Up @@ -173,6 +174,7 @@ class FdEntity
ssize_t Read(char* bytes, off_t start, size_t size, bool force_load = false);
ssize_t Write(const char* bytes, off_t start, size_t size);

bool ReserveDiskSpace(size_t size);
void CleanupCache();
};
typedef std::map<std::string, class FdEntity*> fdent_map_t; // key=path, value=FdEntity*
Expand All @@ -186,6 +188,7 @@ class FdManager
static FdManager singleton;
static pthread_mutex_t fd_manager_lock;
static pthread_mutex_t cache_cleanup_lock;
static pthread_mutex_t reserved_diskspace_lock;
static bool is_lock_init;
static std::string cache_dir;
static bool check_cache_dir_exist;
Expand Down Expand Up @@ -217,8 +220,9 @@ class FdManager

static size_t GetEnsureFreeDiskSpace(void) { return FdManager::free_disk_space; }
static size_t SetEnsureFreeDiskSpace(size_t size);
static size_t InitEnsureFreeDiskSpace(void) { return SetEnsureFreeDiskSpace(0); }
static bool IsSafeDiskSpace(const char* path, size_t size);
static void FreeReservedDiskSpace(size_t size);
bool ReserveDiskSpace(size_t size);

FdEntity* GetFdEntity(const char* path, int existfd = -1);
FdEntity* Open(const char* path, headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true, bool no_fd_lock_wait = false);
Expand Down
5 changes: 1 addition & 4 deletions src/s3fs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4671,8 +4671,6 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
S3FS_PRN_EXIT("multipart_size option must be at least 5 MB.");
return -1;
}
// update ensure free disk space if it is not set.
FdManager::InitEnsureFreeDiskSpace();
return 0;
}
if(0 == STR2NCMP(arg, "ensure_diskfree=")){
Expand Down Expand Up @@ -5034,8 +5032,7 @@ int main(int argc, char* argv[])
}

// check free disk space
FdManager::InitEnsureFreeDiskSpace();
if(!FdManager::IsSafeDiskSpace(NULL, S3fsCurl::GetMultipartSize())){
if(!FdManager::IsSafeDiskSpace(NULL, S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount())){
S3FS_PRN_EXIT("There is no enough disk space for used as cache(or temporary) directory by s3fs.");
exit(EXIT_FAILURE);
}
Expand Down
2 changes: 1 addition & 1 deletion src/s3fs_util.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1125,7 +1125,7 @@ void show_help (void)
" multipart_size (default=\"10\")\n"
" - part size, in MB, for each multipart request.\n"
"\n"
" ensure_diskfree (default same multipart_size value)\n"
" ensure_diskfree (default 0)\n"
" - sets MB to ensure disk free space. s3fs makes file for\n"
" downloading, uploading and caching files. If the disk free\n"
" space is smaller than this value, s3fs do not use diskspace\n"
Expand Down

0 comments on commit 45c7ea9

Please sign in to comment.