Skip to content

Commit

Permalink
Files streaming
Browse files Browse the repository at this point in the history
GitOrigin-RevId: 78df1cd44c95380cd4af46f4db809ce28876db1f
  • Loading branch information
arseny30 committed Nov 11, 2018
1 parent 7ee12fd commit 3b238f6
Show file tree
Hide file tree
Showing 28 changed files with 526 additions and 105 deletions.
2 changes: 2 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,7 @@ set(TDLIB_SOURCE
td/telegram/DialogParticipant.cpp
td/telegram/DocumentsManager.cpp
td/telegram/DraftMessage.cpp
td/telegram/files/FileBitmask.cpp
td/telegram/files/FileDb.cpp
td/telegram/files/FileDownloader.cpp
td/telegram/files/FileFromBytes.cpp
Expand Down Expand Up @@ -478,6 +479,7 @@ set(TDLIB_SOURCE
td/telegram/DialogParticipant.h
td/telegram/DocumentsManager.h
td/telegram/DraftMessage.h
td/telegram/files/FileBitmask.h
td/telegram/files/FileDb.h
td/telegram/files/FileDownloader.h
td/telegram/files/FileFromBytes.h
Expand Down
16 changes: 14 additions & 2 deletions td/generate/scheme/td_api.tl
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,10 @@ temporaryPasswordState has_password:Bool valid_for:int32 = TemporaryPasswordStat
//@can_be_deleted True, if the file can be deleted
//@is_downloading_active True, if the file is currently being downloaded (or a local copy is being generated by some other means)
//@is_downloading_completed True, if the local copy is fully available
//@download_offset Download will be started from this offset. downloaded_prefix_size is calculated from this offset.
//@downloaded_prefix_size If is_downloading_completed is false, then only some prefix of the file is ready to be read. downloaded_prefix_size is the size of that prefix
//@downloaded_size Total downloaded file bytes. Should be used only for calculating download progress. The actual file size may be bigger, and some parts of it may contain garbage
localFile path:string can_be_downloaded:Bool can_be_deleted:Bool is_downloading_active:Bool is_downloading_completed:Bool downloaded_prefix_size:int32 downloaded_size:int32 = LocalFile;
localFile path:string can_be_downloaded:Bool can_be_deleted:Bool is_downloading_active:Bool is_downloading_completed:Bool download_offset:int32 downloaded_prefix_size:int32 downloaded_size:int32 = LocalFile;

//@description Represents a remote file
//@id Remote file identifier; may be empty. Can be used across application restarts or even from other devices for the current user. If the ID starts with "http://" or "https://", it represents the HTTP URL of the file. TDLib is currently unable to download files if only their URL is known.
Expand Down Expand Up @@ -2948,7 +2949,18 @@ setPinnedChats chat_ids:vector<int53> = Ok;
//@description Asynchronously downloads a file from the cloud. updateFile will be used to notify about the download progress and successful completion of the download. Returns file state just after the download has been started
//@file_id Identifier of the file to download
//@priority Priority of the download (1-32). The higher the priority, the earlier the file will be downloaded. If the priorities of two files are equal, then the last one for which downloadFile was called will be downloaded first
downloadFile file_id:int32 priority:int32 = File;
//@offset File will be downloaded starting from offset first. Supposed to be used for streaming.
downloadFile file_id:int32 priority:int32 offset:int32 = File;

//@description Set offset for file downloading
//@file_id Identifier of file
//@offset File download offset
setFileDownloadOffset file_id:int32 offset:int32 = File;

//@description Get downloaded prefix from a given offset
//@file_id Identifier of file
//@offset Offset from which downloaded prefix is calculated
getFileDownloadedPrefix file_id:int32 offset:int32 = Count;

//@description Stops the downloading of a file. If a file has already been downloaded, does nothing @file_id Identifier of a file to stop downloading @only_if_pending Pass true to stop downloading only if it hasn't been started, i.e. request hasn't been sent to server
cancelDownloadFile file_id:int32 only_if_pending:Bool = Ok;
Expand Down
Binary file modified td/generate/scheme/td_api.tlo
Binary file not shown.
6 changes: 3 additions & 3 deletions td/telegram/InlineQueriesManager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -822,9 +822,9 @@ static tl_object_ptr<T> copy(const tl_object_ptr<T> &obj) {

template <>
td_api::object_ptr<td_api::localFile> copy(const td_api::localFile &obj) {
return td_api::make_object<td_api::localFile>(obj.path_, obj.can_be_downloaded_, obj.can_be_deleted_,
obj.is_downloading_active_, obj.is_downloading_completed_,
obj.downloaded_prefix_size_, obj.downloaded_size_);
return td_api::make_object<td_api::localFile>(
obj.path_, obj.can_be_downloaded_, obj.can_be_deleted_, obj.is_downloading_active_, obj.is_downloading_completed_,
obj.download_offset_, obj.downloaded_prefix_size_, obj.downloaded_size_);
}
template <>
td_api::object_ptr<td_api::remoteFile> copy(const td_api::remoteFile &obj) {
Expand Down
2 changes: 1 addition & 1 deletion td/telegram/MessagesManager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6224,7 +6224,7 @@ void MessagesManager::load_secret_thumbnail(FileId thumbnail_file_id) {
});

send_closure(G()->file_manager(), &FileManager::download, thumbnail_file_id,
std::make_shared<Callback>(std::move(download_promise)), 1);
std::make_shared<Callback>(std::move(download_promise)), 1, -1);
}

void MessagesManager::on_upload_media(FileId file_id, tl_object_ptr<telegram_api::InputFile> input_file,
Expand Down
23 changes: 21 additions & 2 deletions td/telegram/Td.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4858,6 +4858,15 @@ void Td::on_request(uint64 id, const td_api::getFile &request) {
send_closure(actor_id(this), &Td::send_result, id, file_manager_->get_file_object(FileId(request.file_id_, 0)));
}

void Td::on_request(uint64 id, const td_api::getFileDownloadedPrefix &request) {
auto file_view = file_manager_->get_file_view(FileId(request.file_id_, 0));
if (file_view.empty()) {
return send_closure(actor_id(this), &Td::send_error, id, Status::Error(10, "Unknown file id"));
}
send_closure(actor_id(this), &Td::send_result, id,
td_api::make_object<td_api::count>(static_cast<int32>(file_view.downloaded_prefix(request.offset_))));
}

void Td::on_request(uint64 id, td_api::getRemoteFile &request) {
CLEAN_INPUT_STRING(request.remote_file_id_);
auto r_file_id = file_manager_->from_persistent_id(
Expand Down Expand Up @@ -5668,8 +5677,18 @@ void Td::on_request(uint64 id, const td_api::downloadFile &request) {
if (!(1 <= priority && priority <= 32)) {
return send_error_raw(id, 5, "Download priority must be in [1;32] range");
}
file_manager_->download(FileId(request.file_id_, 0), download_file_callback_, priority);
file_manager_->download(FileId(request.file_id_, 0), download_file_callback_, priority, request.offset_);

auto file = file_manager_->get_file_object(FileId(request.file_id_, 0), false);
if (file->id_ == 0) {
return send_error_raw(id, 400, "Invalid file id");
}

send_closure(actor_id(this), &Td::send_result, id, std::move(file));
}

void Td::on_request(uint64 id, const td_api::setFileDownloadOffset &request) {
file_manager_->download_set_offset(FileId(request.file_id_, 0), request.offset_);
auto file = file_manager_->get_file_object(FileId(request.file_id_, 0), false);
if (file->id_ == 0) {
return send_error_raw(id, 400, "Invalid file id");
Expand All @@ -5679,7 +5698,7 @@ void Td::on_request(uint64 id, const td_api::downloadFile &request) {
}

void Td::on_request(uint64 id, const td_api::cancelDownloadFile &request) {
file_manager_->download(FileId(request.file_id_, 0), nullptr, request.only_if_pending_ ? -1 : 0);
file_manager_->download(FileId(request.file_id_, 0), nullptr, 0, request.only_if_pending_ ? -1 : 0);

send_closure(actor_id(this), &Td::send_result, id, make_tl_object<td_api::ok>());
}
Expand Down
4 changes: 4 additions & 0 deletions td/telegram/Td.h
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,8 @@ class Td final : public NetQueryCallback {

void on_request(uint64 id, const td_api::getFile &request);

void on_request(uint64 id, const td_api::getFileDownloadedPrefix &request);

void on_request(uint64 id, td_api::getRemoteFile &request);

void on_request(uint64 id, td_api::getStorageStatistics &request);
Expand Down Expand Up @@ -644,6 +646,8 @@ class Td final : public NetQueryCallback {

void on_request(uint64 id, const td_api::downloadFile &request);

void on_request(uint64 id, const td_api::setFileDownloadOffset &request);

void on_request(uint64 id, const td_api::cancelDownloadFile &request);

void on_request(uint64 id, td_api::uploadFile &request);
Expand Down
25 changes: 21 additions & 4 deletions td/telegram/cli.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2251,6 +2251,11 @@ class CliClient final : public Actor {
send_request(make_tl_object<td_api::getChatMessageByDate>(as_chat_id(chat_id), to_integer<int32>(date)));
} else if (op == "gf" || op == "GetFile") {
send_request(make_tl_object<td_api::getFile>(as_file_id(args)));
} else if (op == "gfp" || op == "GetFileDownloadedPrefix") {
string file_id;
string offset;
std::tie(file_id, offset) = split(args);
send_request(make_tl_object<td_api::getFileDownloadedPrefix>(as_file_id(file_id), to_integer<int32>(offset)));
} else if (op == "grf") {
send_request(make_tl_object<td_api::getRemoteFile>(args, nullptr));
} else if (op == "gmtf") {
Expand All @@ -2271,26 +2276,38 @@ class CliClient final : public Actor {
send_request(make_tl_object<td_api::getMapThumbnailFile>(
as_location(latitude, longitude), to_integer<int32>(zoom), to_integer<int32>(width),
to_integer<int32>(height), to_integer<int32>(scale), as_chat_id(chat_id)));
} else if (op == "sfdo" || op == "SetDownloadFileOffset") {
string file_id_str;
string offset;
std::tie(file_id_str, offset) = split(args);

auto file_id = as_file_id(file_id_str);
send_request(make_tl_object<td_api::setFileDownloadOffset>(file_id, to_integer<int32>(offset)));
} else if (op == "df" || op == "DownloadFile") {
string file_id_str;
string priority;
std::tie(file_id_str, priority) = split(args);
string offset;
std::tie(file_id_str, args) = split(args);
std::tie(priority, offset) = split(args);
if (priority.empty()) {
priority = "1";
}

auto file_id = as_file_id(file_id_str);
send_request(make_tl_object<td_api::downloadFile>(file_id, to_integer<int32>(priority)));
send_request(
make_tl_object<td_api::downloadFile>(file_id, to_integer<int32>(priority), to_integer<int32>(offset)));
} else if (op == "dff") {
string file_id;
string priority;
std::tie(file_id, priority) = split(args);
string offset;
std::tie(file_id, args) = split(args);
std::tie(priority, offset) = split(args);
if (priority.empty()) {
priority = "1";
}

for (int i = 1; i <= as_file_id(file_id); i++) {
send_request(make_tl_object<td_api::downloadFile>(i, to_integer<int32>(priority)));
send_request(make_tl_object<td_api::downloadFile>(i, to_integer<int32>(priority), to_integer<int32>(offset)));
}
} else if (op == "cdf") {
send_request(make_tl_object<td_api::cancelDownloadFile>(as_file_id(args), false));
Expand Down
84 changes: 84 additions & 0 deletions td/telegram/files/FileBitmask.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
//
// Copyright Aliaksei Levin ([email protected]), Arseny Smirnov ([email protected]) 2014-2018
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/telegram/files/FileBitmask.h"
#include "td/utils/misc.h"
namespace td {
Bitmask::Bitmask(Decode, Slice data) : data_(zero_one_decode(data)) {
}
Bitmask::Bitmask(Ones, int64 count) : data_((count + 7) / 8, '\0') {
for (int64 i = 0; i < count; i++) {
set(i);
}
}
std::string Bitmask::encode() const {
// remove zeroes in the end to make encoding deteministic
td::Slice data(data_);
while (!data.empty() && data.back() == 0) {
data.remove_suffix(1);
}
return zero_one_encode(data_);
}
Bitmask::ReadySize Bitmask::get_ready_size(int64 offset, int64 part_size) const {
ReadySize res;
res.offset = offset;
auto offset_part = offset / part_size;
auto ones = get_ready_parts(offset_part);
if (ones == 0) {
res.ready_size = 0;
} else {
res.ready_size = (offset_part + ones) * part_size - offset;
}
CHECK(res.ready_size >= 0);
return res;
}
int64 Bitmask::get_total_size(int64 part_size) const {
int64 res = 0;
for (int64 i = 0; i < size(); i++) {
res += get(i);
}
return res * part_size;
}
bool Bitmask::get(int64 offset) const {
if (offset < 0) {
return 0;
}
if (offset / 8 >= narrow_cast<int64>(data_.size())) {
return 0;
}
return (data_[offset / 8] & (1 << (offset % 8))) != 0;
}

int64 Bitmask::get_ready_parts(int64 offset) const {
int64 res = 0;
while (get(offset + res)) {
res++;
}
return res;
};

std::vector<int32> Bitmask::as_vector() const {
std::vector<int32> res;
for (int32 i = 0; i < narrow_cast<int32>(data_.size() * 8); i++) {
if (get(i)) {
res.push_back(i);
}
}
return res;
}
void Bitmask::set(int64 offset) {
auto need_size = narrow_cast<size_t>(offset / 8 + 1);
if (need_size > data_.size()) {
data_.resize(need_size, 0);
}
data_[need_size - 1] |= (1 << (offset % 8));
}

int64 Bitmask::size() const {
return data_.size() * 8;
}

} // namespace td
50 changes: 50 additions & 0 deletions td/telegram/files/FileBitmask.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
//
// Copyright Aliaksei Levin ([email protected]), Arseny Smirnov ([email protected]) 2014-2018
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/StringBuilder.h"

namespace td {
class Bitmask {
public:
struct ReadySize {
int64 offset{-1};
int64 ready_size{-1};
bool empty() const {
return offset == -1;
}
};
struct Decode {};
struct Ones {};
Bitmask() = default;
Bitmask(Decode, Slice data);
Bitmask(Ones, int64 count);
std::string encode() const;
ReadySize get_ready_size(int64 offset, int64 part_size) const;
int64 get_total_size(int64 part_size) const;
bool get(int64 offset) const;

int64 get_ready_parts(int64 offset) const;

std::vector<int32> as_vector() const;
void set(int64 offset);
int64 size() const;

private:
std::string data_;
};

inline StringBuilder &operator<<(StringBuilder &sb, const Bitmask &mask) {
std::string res;
for (int64 i = 0; i < mask.size(); i++) {
res += mask.get(i) ? "1" : "0";
}
return sb << res;
}

} // namespace td
Loading

0 comments on commit 3b238f6

Please sign in to comment.