Skip to content

Commit

Permalink
fix: [filecopy]When copying large files, the swap partition is heavil…
Browse files Browse the repository at this point in the history 
…y used and the UI interface lags severely

Modify the copying method to remove mmap and use copy_file_range

Log: When copying large files, the swap partition is heavily used and the UI interface lags severely
Bug: https://pms.uniontech.com/bug-view-273191.html
  • Loading branch information
@liyigang1
liyigang1 committed Oct 14, 2024
1 parent edcf903 commit 341fe27
Show file tree
Hide file tree
Showing 5 changed files with 163 additions and 274 deletions.
195 changes: 148 additions & 47 deletions ...mmon/core/dfmplugin-fileoperations/fileoperations/fileoperationutils/docopyfileworker.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,53 +78,6 @@ void DoCopyFileWorker::doFileCopy(const DFileInfoPointer fromInfo, const DFileIn
workData->completeFileCount++;
}

void DoCopyFileWorker::doMemcpyLocalBigFile(const DFileInfoPointer fromInfo, const DFileInfoPointer toInfo, char *dest, char *source, size_t size)
{
size_t copySize = size;
char *destStart = dest;
char *sourceStart = source;
size_t everyCopySize = kMaxBufferLength;
while (copySize > 0 && !isStopped()) {
if (Q_UNLIKELY(!stateCheck())) {
break;
}
everyCopySize = copySize >= everyCopySize ? everyCopySize : copySize;

AbstractJobHandler::SupportAction action { AbstractJobHandler::SupportAction::kNoAction };

do {
action = AbstractJobHandler::SupportAction::kNoAction;
if (!memcpy(destStart, sourceStart, everyCopySize)) {
auto lastError = strerror(errno);
fmWarning() << "file memcpy error, url from: " << fromInfo->uri()
<< " url to: " << toInfo->uri()
<< " error code: " << errno << " error msg: " << lastError;

action = doHandleErrorAndWait(fromInfo->uri(), toInfo->uri(),
AbstractJobHandler::JobErrorType::kWriteError,
true, lastError);
}
} while (action == AbstractJobHandler::SupportAction::kRetryAction && !isStopped());

checkRetry();

if (!actionOperating(action, static_cast<qint64>(copySize), nullptr)) {
if (action == AbstractJobHandler::SupportAction::kSkipAction)
emit skipCopyLocalBigFile(fromInfo->uri());
return;
}

copySize -= everyCopySize;
destStart += everyCopySize;
sourceStart += everyCopySize;

if (memcpySkipUrl.isValid() && memcpySkipUrl == fromInfo->uri())
return;

workData->currentWriteSize += static_cast<int64_t>(everyCopySize);
}
}

bool DoCopyFileWorker::doDfmioFileCopy(const DFileInfoPointer fromInfo,
const DFileInfoPointer toInfo, bool *skip)
{
Expand Down Expand Up @@ -206,6 +159,54 @@ void DoCopyFileWorker::syncBlockFile(const DFileInfoPointer toInfo)
close(tofd);
}
}
/*!
* \brief DoCopyFileWorker::openFile
* \param fromInfo
* \param toInfo
* \param flags
* \param skip
* \param isSource
* \return
*/
int DoCopyFileWorker::openFileBySys(const DFileInfoPointer &fromInfo, const DFileInfoPointer &toInfo,
const int flags, bool *skip, const bool isSource)
{
int fd = -1;
AbstractJobHandler::SupportAction action = AbstractJobHandler::SupportAction::kNoAction;
auto openUrl = isSource ? fromInfo->uri() : toInfo->uri();
do {
action = AbstractJobHandler::SupportAction::kNoAction;
if (flags & O_CREAT) {
fd = open(openUrl.path().toStdString().c_str(), flags, 0666);
} else {
fd = open(openUrl.path().toStdString().c_str(), flags);
}

if (fd < 0) {
auto lastError = strerror(errno);
fmWarning() << "file open error, url from: " << fromInfo->uri()
<< " url to: " << toInfo->uri() << " open flag: " << flags
<< " open url : " << openUrl << " error msg: " << lastError;

action = doHandleErrorAndWait(fromInfo->uri(), toInfo->uri(),
AbstractJobHandler::JobErrorType::kOpenError,
!isSource, lastError);
}
} while (action == AbstractJobHandler::SupportAction::kRetryAction && !isStopped());

checkRetry();

auto fileSize = fromInfo->attribute(DFileInfo::AttributeID::kStandardSize).toLongLong();
if (!actionOperating(action, fileSize <= 0 ? FileUtils::getMemoryPageSize() : fileSize, skip)) {
close(fd);
return -1;
}

if (isSource && fileSize > 100 * 1024 * 1024)
readahead(fd, 0, static_cast<size_t>(fileSize));

return fd;
}

// copy thread using
DoCopyFileWorker::NextDo DoCopyFileWorker::doCopyFilePractically(const DFileInfoPointer fromInfo, const DFileInfoPointer toInfo, bool *skip)
Expand Down Expand Up @@ -300,6 +301,106 @@ DoCopyFileWorker::NextDo DoCopyFileWorker::doCopyFilePractically(const DFileInfo

return NextDo::kDoCopyNext;
}
/*!
* \brief DoCopyFileWorker::doCopyFileByRange
* \param fromInfo
* \param toInfo
* \param skip
* \return
*/
DoCopyFileWorker::NextDo DoCopyFileWorker::doCopyFileByRange(const DFileInfoPointer fromInfo, const DFileInfoPointer toInfo, bool *skip)
{
if (isStopped())
return NextDo::kDoCopyErrorAddCancel;

// emit current task url
emit currentTask(fromInfo->uri(), toInfo->uri());

// open source file
int sourcFd = openFileBySys(fromInfo, toInfo, O_RDONLY, skip);
if (sourcFd < 0)
return NextDo::kDoCopyErrorAddCancel;

int targetFd = openFileBySys(fromInfo, toInfo, O_CREAT | O_WRONLY | O_TRUNC, skip, false);
if (targetFd < 0) {
close(sourcFd);
return NextDo::kDoCopyErrorAddCancel;
}

// 源文件大小如果为0
auto fromSize = fromInfo->attribute(DFileInfo::AttributeID::kStandardSize).toLongLong();
if (fromSize <= 0) {
// 对文件加权
setTargetPermissions(fromInfo->uri(), toInfo->uri());
workData->zeroOrlinkOrDirWriteSize += FileUtils::getMemoryPageSize();
FileUtils::notifyFileChangeManual(DFMBASE_NAMESPACE::Global::FileNotifyType::kFileAdded, toInfo->uri());
if (workData->exBlockSyncEveryWrite || DeviceUtils::isSamba(toInfo->uri()))
syncfs(targetFd);
close(sourcFd);
close(targetFd);
return NextDo::kDoCopyNext;
}

// 循环读取和写入文件，拷贝
auto toIsSmb = DeviceUtils::isSamba(toInfo->uri());
size_t blockSize = static_cast<size_t>(fromSize > kMaxBufferLength ? kMaxBufferLength : fromSize);

off_t offset_in = 0;
off_t offset_out = 0;
ssize_t result = -1;
AbstractJobHandler::SupportAction action { AbstractJobHandler::SupportAction::kNoAction };
do {
if (Q_UNLIKELY(!stateCheck()))
return NextDo::kDoCopyErrorAddCancel;

do {
if (Q_UNLIKELY(!stateCheck()))
return NextDo::kDoCopyErrorAddCancel;
result = copy_file_range(sourcFd, &offset_in, targetFd, &offset_out, blockSize, 0);

if (result < 0) {
auto lastError = strerror(errno);
fmWarning() << "copy file range error, url from: " << fromInfo->uri()
<< " url to: " << toInfo->uri() << " error msg: " << lastError;

action = doHandleErrorAndWait(fromInfo->uri(), toInfo->uri(),
AbstractJobHandler::JobErrorType::kOpenError,
false, lastError);
offset_in = qMin(offset_in, offset_out);
offset_out = offset_in;
} else {
workData->currentWriteSize += result;
}
} while (action == AbstractJobHandler::SupportAction::kRetryAction && !isStopped());

checkRetry();

if (!actionOperating(action, fromSize - offset_out, skip)) {
close(sourcFd);
close(targetFd);
return NextDo::kDoCopyErrorAddCancel;
}

// 执行同步策略
if (workData->exBlockSyncEveryWrite || toIsSmb)
syncfs(targetFd);

} while (offset_out != fromSize);

// 执行同步策略
if (workData->exBlockSyncEveryWrite || toIsSmb)
syncfs(targetFd);

// 对文件加权
setTargetPermissions(fromInfo->uri(), toInfo->uri());
if (!stateCheck())
return NextDo::kDoCopyErrorAddCancel;

if (skip && *skip)
FileUtils::notifyFileChangeManual(DFMBASE_NAMESPACE::Global::FileNotifyType::kFileAdded, toInfo->uri());

return NextDo::kDoCopyNext;
}

bool DoCopyFileWorker::stateCheck()
{
Expand Down
8 changes: 6 additions & 2 deletions ...common/core/dfmplugin-fileoperations/fileoperations/fileoperationutils/docopyfileworker.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,12 @@ class DoCopyFileWorker : public QObject
// normal copy
NextDo doCopyFilePractically(const DFileInfoPointer fromInfo, const DFileInfoPointer toInfo,
bool *skip);
// normal copy
NextDo doCopyFileByRange(const DFileInfoPointer fromInfo, const DFileInfoPointer toInfo,
bool *skip);
// small file copy
void doFileCopy(const DFileInfoPointer fromInfo, const DFileInfoPointer toInfo);
// big file copy in system device
void doMemcpyLocalBigFile(const DFileInfoPointer fromInfo, const DFileInfoPointer toInfo, char *dest, char *source, size_t size);

// copy file by dfmio
bool doDfmioFileCopy(const DFileInfoPointer fromInfo, const DFileInfoPointer toInfo, bool *skip);
signals:
Expand Down Expand Up @@ -117,6 +119,8 @@ class DoCopyFileWorker : public QObject
void checkRetry();
bool isStopped();
void syncBlockFile(const DFileInfoPointer toInfo);
int openFileBySys(const DFileInfoPointer &fromInfo, const DFileInfoPointer &toInfo,
const int flags, bool *skip, const bool isSource = true);
public:
static void progressCallback(int64_t current, int64_t total, void *progressData);

Expand Down
Loading

0 comments on commit 341fe27

Please sign in to comment.