Skip to content

Commit

Permalink
Reduce the number of threads for redirect to resource data transfer
Browse files Browse the repository at this point in the history
iychoi committed Jun 6, 2024
1 parent 8298bb8 commit 258fcd2
Showing 5 changed files with 14 additions and 14 deletions.
4 changes: 2 additions & 2 deletions cmd/subcmd/get.go
Original file line number Diff line number Diff line change
@@ -265,14 +265,14 @@ func getOne(parallelJobManager *commons.ParallelJobManager, inputPathMap map[str
if parallelTransferFlagValues.SingleTread || parallelTransferFlagValues.ThreadNumber == 1 {
downloadErr = fs.DownloadFileResumable(sourcePath, "", targetFilePath, callbackGet)
} else if parallelTransferFlagValues.RedirectToResource {
downloadErr = fs.DownloadFileRedirectToResource(sourcePath, "", targetFilePath, callbackGet)
downloadErr = fs.DownloadFileRedirectToResource(sourcePath, "", targetFilePath, 0, callbackGet)
} else if parallelTransferFlagValues.Icat {
downloadErr = fs.DownloadFileParallelResumable(sourcePath, "", targetFilePath, 0, callbackGet)
} else {
// auto
if sourceEntry.Size >= commons.RedirectToResourceMinSize {
// redirect-to-resource
downloadErr = fs.DownloadFileRedirectToResource(sourcePath, "", targetFilePath, callbackGet)
downloadErr = fs.DownloadFileRedirectToResource(sourcePath, "", targetFilePath, 0, callbackGet)
} else {
downloadErr = fs.DownloadFileParallelResumable(sourcePath, "", targetFilePath, 0, callbackGet)
}
6 changes: 3 additions & 3 deletions cmd/subcmd/put.go
Original file line number Diff line number Diff line change
@@ -286,22 +286,22 @@ func putOne(parallelJobManager *commons.ParallelJobManager, inputPathMap map[str
if parallelTransferFlagValues.SingleTread || parallelTransferFlagValues.ThreadNumber == 1 {
uploadErr = fs.UploadFile(sourcePath, targetFilePath, "", false, callbackPut)
} else if parallelTransferFlagValues.RedirectToResource {
uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", false, callbackPut)
uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", 0, false, callbackPut)
} else if parallelTransferFlagValues.Icat {
uploadErr = fs.UploadFileParallel(sourcePath, targetFilePath, "", 0, false, callbackPut)
} else {
// auto
if sourceStat.Size() >= commons.RedirectToResourceMinSize {
// redirect-to-resource
uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", false, callbackPut)
uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", 0, false, callbackPut)
} else {
if filesystem.SupportParallelUpload() {
uploadErr = fs.UploadFileParallel(sourcePath, targetFilePath, "", 0, false, callbackPut)
} else {
if sourceStat.Size() >= commons.ParallelUploadMinSize {
// does not support parall upload via iCAT
// redirect-to-resource
uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", false, callbackPut)
uploadErr = fs.UploadFileParallelRedirectToResource(sourcePath, targetFilePath, "", 0, false, callbackPut)
} else {
uploadErr = fs.UploadFileParallel(sourcePath, targetFilePath, "", 0, false, callbackPut)
}
12 changes: 6 additions & 6 deletions commons/bundle_transfer.go
Original file line number Diff line number Diff line change
@@ -1042,22 +1042,22 @@ func (manager *BundleTransferManager) processBundleUpload(bundle *Bundle) error
if manager.singleThreaded || manager.uploadThreadNum == 1 {
err = manager.filesystem.UploadFile(bundle.localBundlePath, bundle.irodsBundlePath, "", false, callback)
} else if manager.redirectToResource {
err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", false, callback)
err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, callback)
} else if manager.useIcat {
err = manager.filesystem.UploadFileParallel(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, callback)
} else {
// auto
if bundle.size >= RedirectToResourceMinSize {
// redirect-to-resource
err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", false, callback)
err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, callback)
} else {
if manager.filesystem.SupportParallelUpload() {
err = manager.filesystem.UploadFileParallel(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, callback)
} else {
if bundle.size >= ParallelUploadMinSize {
// does not support parall upload via iCAT
// redirect-to-resource
err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", false, callback)
err = manager.filesystem.UploadFileParallelRedirectToResource(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, callback)
} else {
err = manager.filesystem.UploadFileParallel(bundle.localBundlePath, bundle.irodsBundlePath, "", 0, false, callback)
}
@@ -1137,22 +1137,22 @@ func (manager *BundleTransferManager) processBundleUpload(bundle *Bundle) error
if manager.singleThreaded || manager.uploadThreadNum == 1 {
err = manager.filesystem.UploadFile(file.LocalPath, file.IRODSPath, "", false, callbackFileUpload)
} else if manager.redirectToResource {
err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", false, callbackFileUpload)
err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, callbackFileUpload)
} else if manager.useIcat {
err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, callbackFileUpload)
} else {
// auto
if bundle.size >= RedirectToResourceMinSize {
// redirect-to-resource
err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", false, callbackFileUpload)
err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, callbackFileUpload)
} else {
if manager.filesystem.SupportParallelUpload() {
err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, callbackFileUpload)
} else {
if bundle.size >= ParallelUploadMinSize {
// does not support parall upload via iCAT
// redirect-to-resource
err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", false, callbackFileUpload)
err = manager.filesystem.UploadFileParallelRedirectToResource(file.LocalPath, file.IRODSPath, "", 0, false, callbackFileUpload)
} else {
err = manager.filesystem.UploadFileParallel(file.LocalPath, file.IRODSPath, "", 0, false, callbackFileUpload)
}
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@ go 1.18

require (
github.com/creativeprojects/go-selfupdate v1.0.1
github.com/cyverse/go-irodsclient v0.14.6
github.com/cyverse/go-irodsclient v0.14.7
github.com/dustin/go-humanize v1.0.1
github.com/gliderlabs/ssh v0.3.5
github.com/jedib0t/go-pretty/v6 v6.3.1
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
@@ -9,8 +9,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creativeprojects/go-selfupdate v1.0.1 h1:5Un4MTv4puCR5GBgkDLC14J72fljGuMC60E63cFGq1o=
github.com/creativeprojects/go-selfupdate v1.0.1/go.mod h1:nm7AWUJfrfYt/SB97NAcMhR0KEpPqlrVHXkWFti+ezw=
github.com/cyverse/go-irodsclient v0.14.6 h1:otPfQIZ7JVIU8liWkwKfUpds/tu9a35weliIrNzB/Tc=
github.com/cyverse/go-irodsclient v0.14.6/go.mod h1:eBXha3cwfrM0p1ijYVqsrLJQHpRwTfpA4c5dKCQsQFc=
github.com/cyverse/go-irodsclient v0.14.7 h1:gZOZhpw9HTyUdciAvy3b7axWnRA/ak+3PwLKKOWrIbw=
github.com/cyverse/go-irodsclient v0.14.7/go.mod h1:eBXha3cwfrM0p1ijYVqsrLJQHpRwTfpA4c5dKCQsQFc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

0 comments on commit 258fcd2

Please sign in to comment.