mirror/qBittorrent
2
0
Fork 0
mirror of https://github.com/qbittorrent/qBittorrent.git synced 2024-12-21 08:09:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

Avoid holding entire file in memory

Browse Source 
Previously we need a file buffer that is as large as the file size and
this could be a problem when user has less free memory available or
having very large data. Now with the help of `FileOutputIterator`,
we can have a much smaller, fixed size immediate file buffer and also
the code looks nice with `lt::bencode()`.
This commit is contained in:
Chocobo1 2020-04-01 14:25:00 +08:00
parent 96c5af7ae9
commit 9f281c2d25
No known key found for this signature in database
GPG Key ID: 210D9C873253A68C
6 changed files with 158 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/base/CMakeLists.txt
View File

@ -58,6 +58,7 @@ utils/bytearray.h
utils/foreignapps.h
utils/fs.h
utils/gzip.h
utils/io.h
utils/misc.h
utils/net.h
utils/password.h
@ -133,6 +134,7 @@ utils/bytearray.cpp
utils/foreignapps.cpp
utils/fs.cpp
utils/gzip.cpp
utils/io.cpp
utils/misc.cpp
utils/net.cpp
utils/password.cpp

2
src/base/base.pri
View File

@ -73,6 +73,7 @@ HEADERS += \
$$PWD/utils/foreignapps.h \
$$PWD/utils/fs.h \
$$PWD/utils/gzip.h \
$$PWD/utils/io.h \
$$PWD/utils/misc.h \
$$PWD/utils/net.h \
$$PWD/utils/password.h \
@ -143,6 +144,7 @@ SOURCES += \
$$PWD/utils/foreignapps.cpp \
$$PWD/utils/fs.cpp \
$$PWD/utils/gzip.cpp \
$$PWD/utils/io.cpp \
$$PWD/utils/misc.cpp \
$$PWD/utils/net.cpp \
$$PWD/utils/password.cpp \

22
src/base/bittorrent/torrentcreatorthread.cpp
View File

@ -41,8 +41,10 @@
#include <QFileInfo>
#include <QHash>
#include "base/exceptions.h"
#include "base/global.h"
#include "base/utils/fs.h"
#include "base/utils/io.h"
#include "base/utils/string.h"
#include "private/ltunderlyingtype.h"
@ -182,19 +184,19 @@ void TorrentCreatorThread::run()
if (isInterruptionRequested()) return;
// create the torrent
std::ofstream outfile(
#ifdef _MSC_VER
Utils::Fs::toNativePath(m_params.savePath).toStdWString().c_str()
#else
Utils::Fs::toNativePath(m_params.savePath).toUtf8().constData()
#endif
, (std::ios_base::out | std::ios_base::binary | std::ios_base::trunc));
if (outfile.fail())
throw std::runtime_error(tr("create new torrent file failed").toStdString());
QFile outfile {m_params.savePath};
if (!outfile.open(QIODevice::WriteOnly)) {
throw RuntimeError {tr("Create new torrent file failed. Reason: %1")
.arg(outfile.errorString())};
}
if (isInterruptionRequested()) return;
lt::bencode(std::ostream_iterator<char>(outfile), entry);
lt::bencode(Utils::IO::FileDeviceOutputIterator {outfile}, entry);
if (outfile.error() != QFileDevice::NoError) {
throw RuntimeError {tr("Create new torrent file failed. Reason: %1")
.arg(outfile.errorString())};
}
outfile.close();
emit updateProgress(100);

11
src/base/bittorrent/torrentinfo.cpp
View File

@ -47,6 +47,7 @@
#include "base/exceptions.h"
#include "base/global.h"
#include "base/utils/fs.h"
#include "base/utils/io.h"
#include "base/utils/misc.h"
#include "infohash.h"
#include "trackerentry.h"
@ -166,12 +167,12 @@ void TorrentInfo::saveToFile(const QString &path) const
#endif
const lt::entry torrentEntry = torrentCreator.generate();
QByteArray out;
out.reserve(1024 * 1024); // most torrent file sizes are under 1 MB
lt::bencode(std::back_inserter(out), torrentEntry);
QFile torrentFile {path};
if (!torrentFile.open(QIODevice::WriteOnly) || (torrentFile.write(out) != out.size()))
if (!torrentFile.open(QIODevice::WriteOnly))
throw RuntimeError {torrentFile.errorString()};
lt::bencode(Utils::IO::FileDeviceOutputIterator {torrentFile}, torrentEntry);
if (torrentFile.error() != QFileDevice::NoError)
throw RuntimeError {torrentFile.errorString()};
}

73
src/base/utils/io.cpp Normal file
View File

@ -0,0 +1,73 @@
/*
* Bittorrent Client using Qt and libtorrent.
* Copyright (C) 2020 Mike Tzou (Chocobo1)
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
* In addition, as a special exception, the copyright holders give permission to
* link this program with the OpenSSL project's "OpenSSL" library (or with
* modified versions of it that use the same license as the "OpenSSL" library),
* and distribute the linked executables. You must obey the GNU General Public
* License in all respects for all of the code used other than "OpenSSL". If you
* modify file(s), you may extend this exception to your version of the file(s),
* but you are not obligated to do so. If you do not wish to do so, delete this
* exception statement from your version.
*/
#include "io.h"
#include <QByteArray>
#include <QFileDevice>
Utils::IO::FileDeviceOutputIterator::FileDeviceOutputIterator(QFileDevice &device, const int bufferSize)
: m_device {&device}
, m_buffer {std::make_shared<QByteArray>()}
, m_bufferSize {bufferSize}
{
m_buffer->reserve(bufferSize);
}
Utils::IO::FileDeviceOutputIterator::~FileDeviceOutputIterator()
{
if (m_device->error() == QFileDevice::NoError)
m_device->write(*m_buffer);
m_buffer->clear();
}
Utils::IO::FileDeviceOutputIterator &Utils::IO::FileDeviceOutputIterator::operator=(const char c)
{
m_buffer->append(c);
if (m_buffer->size() >= m_bufferSize) {
if (m_device->error() == QFileDevice::NoError)
m_device->write(*m_buffer);
m_buffer->clear();
}
return *this;
}
Utils::IO::FileDeviceOutputIterator &Utils::IO::FileDeviceOutputIterator::operator*()
{
return *this;
}
Utils::IO::FileDeviceOutputIterator &Utils::IO::FileDeviceOutputIterator::operator++()
{
return *this;
}
Utils::IO::FileDeviceOutputIterator &Utils::IO::FileDeviceOutputIterator::operator++(int)
{
return *this;
}

63
src/base/utils/io.h Normal file
View File

@ -0,0 +1,63 @@
/*
* Bittorrent Client using Qt and libtorrent.
* Copyright (C) 2020 Mike Tzou (Chocobo1)
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
* In addition, as a special exception, the copyright holders give permission to
* link this program with the OpenSSL project's "OpenSSL" library (or with
* modified versions of it that use the same license as the "OpenSSL" library),
* and distribute the linked executables. You must obey the GNU General Public
* License in all respects for all of the code used other than "OpenSSL". If you
* modify file(s), you may extend this exception to your version of the file(s),
* but you are not obligated to do so. If you do not wish to do so, delete this
* exception statement from your version.
*/
#pragma once
#include <iterator>
#include <memory>
class QByteArray;
class QFileDevice;
namespace Utils
{
namespace IO
{
// A wrapper class that satisfy LegacyOutputIterator requirement
class FileDeviceOutputIterator
: public std::iterator<std::output_iterator_tag, void, void, void, void>
{
public:
explicit FileDeviceOutputIterator(QFileDevice &device, const int bufferSize = (4 * 1024));
FileDeviceOutputIterator(const FileDeviceOutputIterator &other) = default;
~FileDeviceOutputIterator();
// mimic std::ostream_iterator behavior
FileDeviceOutputIterator &operator=(char c);
// TODO: make these `constexpr` in C++17
FileDeviceOutputIterator &operator*();
FileDeviceOutputIterator &operator++();
FileDeviceOutputIterator &operator++(int);
private:
QFileDevice *m_device;
std::shared_ptr<QByteArray> m_buffer;
int m_bufferSize;
};
}
}