leonetienne/Tubio
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added failure-detection for downloads, and made the clear_download_cache method wait asyncronously for all active downloads to finish

Browse Source
This commit is contained in:
Leon Etienne (ubuntu wsl) 2020-09-27 13:46:15 +02:00
parent cdd8eded30
commit a19d27203f
3 changed files with 146 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

177
Tubio/DownloadManager.cpp
View File

@ -26,56 +26,63 @@ std::string DownloadManager::QueueDownload(std::string url, DOWNLOAD_MODE mode)
DownloadEntry newDownload; DownloadEntry newDownload;
newDownload.tubio_id = tubioId; newDownload.tubio_id = tubioId;
newDownload.status = DOWNLOAD_STATUS::QUEUED;
newDownload.mode = mode; newDownload.mode = mode;
newDownload.download_progress = 0; newDownload.download_progress = 0;
Json j; if (!IsJsonValid(jsString))
j.Parse(jsString);
if (j.GetDataType() != JDType::JSON)
{ {
newDownload.status = DOWNLOAD_STATUS::FAILED; newDownload.status = DOWNLOAD_STATUS::FAILED;
} }
else else
{ {
if ((j.AsJson.DoesExist("title")) && (j.AsJson["title"].GetDataType() == JDType::STRING)) newDownload.status = DOWNLOAD_STATUS::QUEUED;
Json j;
j.Parse(jsString);
if (j.GetDataType() != JDType::JSON)
{ {
newDownload.title = j["title"]; newDownload.status = DOWNLOAD_STATUS::FAILED;
} }
else
if ((j.AsJson.DoesExist("description")) && (j.AsJson["description"].GetDataType() == JDType::STRING))
{ {
newDownload.description = j["description"]; if ((j.AsJson.DoesExist("title")) && (j.AsJson["title"].GetDataType() == JDType::STRING))
}
if ((j.AsJson.DoesExist("uploader")) && (j.AsJson["uploader"].GetDataType() == JDType::STRING))
{
newDownload.uploader = j["uploader"];
}
if ((j.AsJson.DoesExist("duration")) && (j.AsJson["duration"].GetDataType() == JDType::INT))
{
newDownload.duration = j["duration"];
}
if ((j.AsJson.DoesExist("webpage_url")) && (j.AsJson["webpage_url"].GetDataType() == JDType::STRING))
{
newDownload.webpage_url = j["webpage_url"];
}
if ((j.AsJson.DoesExist("thumbnails")) && (j.AsJson["thumbnails"].GetDataType() == JDType::ARRAY))
{
JsonArray& thumbnails = j.AsJson["thumbnails"].AsArray;
if (thumbnails.Size() > 0)
{ {
if (thumbnails.Size() > 1) newDownload.title = j["title"];
}
if ((j.AsJson.DoesExist("description")) && (j.AsJson["description"].GetDataType() == JDType::STRING))
{
newDownload.description = j["description"];
}
if ((j.AsJson.DoesExist("uploader")) && (j.AsJson["uploader"].GetDataType() == JDType::STRING))
{
newDownload.uploader = j["uploader"];
}
if ((j.AsJson.DoesExist("duration")) && (j.AsJson["duration"].GetDataType() == JDType::INT))
{
newDownload.duration = j["duration"];
}
if ((j.AsJson.DoesExist("webpage_url")) && (j.AsJson["webpage_url"].GetDataType() == JDType::STRING))
{
newDownload.webpage_url = j["webpage_url"];
}
if ((j.AsJson.DoesExist("thumbnails")) && (j.AsJson["thumbnails"].GetDataType() == JDType::ARRAY))
{
JsonArray& thumbnails = j.AsJson["thumbnails"].AsArray;
if (thumbnails.Size() > 0)
{ {
// If we have more than one thumbnail to choose from, choose the second-highes quality. if (thumbnails.Size() > 1)
newDownload.thumbnail_url = thumbnails[thumbnails.Size() - 2]["url"]; {
} // If we have more than one thumbnail to choose from, choose the second-highes quality.
else newDownload.thumbnail_url = thumbnails[thumbnails.Size() - 2]["url"];
{ }
newDownload.thumbnail_url = thumbnails[thumbnails.Size() - 1]["url"]; else
{
newDownload.thumbnail_url = thumbnails[thumbnails.Size() - 1]["url"];
}
} }
} }
} }
@ -90,24 +97,33 @@ void DownloadManager::Update()
{ {
if (shouldSave) Save(); if (shouldSave) Save();
std::size_t numActiveDownloads = GetNumActiveDownloads(); std::size_t cachedNumActiveDownloads = GetNumActiveDownloads();
if (numActiveDownloads < XGConfig::downloader.num_threads) // Queue next download, if available
if (cachedNumActiveDownloads < XGConfig::downloader.num_threads)
{ {
DownloadNext(); DownloadNext();
} }
// Check every second, non-blocking // Check every second, non-blocking
if ((numActiveDownloads > 0) && (time(0) - lastProgressCheck > 2)) if ((time(0) - lastProgressCheck > 2) && (cachedNumActiveDownloads > 0))
{ {
UpdateDownloadProgressPercentages(); UpdateDownloadProgressPercentages();
} }
// Clear cache, if requested
if ((shouldClearCacheASAP) && (cachedNumActiveDownloads == 0))
{
shouldClearCacheASAP = false;
ClearDownloadCache();
}
return; return;
} }
void DownloadManager::DownloadNext() void DownloadManager::DownloadNext()
{ {
// Abort, if queue is empty
if (GetQueueLength() == 0) return; if (GetQueueLength() == 0) return;
DownloadEntry* next = nullptr; DownloadEntry* next = nullptr;
@ -128,29 +144,52 @@ void DownloadManager::DownloadNext()
std::stringstream ss; std::stringstream ss;
if (entry->mode == DOWNLOAD_MODE::VIDEO) if (entry->mode == DOWNLOAD_MODE::VIDEO)
{ {
ss << "youtube-dl --newline --no-call-home --no-playlist --limit-rate " << XGConfig::downloader.max_dlrate_per_thread std::string ytdl_call_video =
<< " --no-mtime --no-cache-dir --format \"bestvideo[ext=mp4]+bestaudio/best[ext=mp4]/best\" --merge-output-format mp4" "youtube-dl --newline --no-call-home --no-playlist --limit-rate $$DL_RATE"
<< " -o \"" << XGConfig::downloader.cachedir << "/download/" << entry->tubio_id " --no-mtime --no-cache-dir --format \"bestvideo[ext=mp4]+bestaudio/best[ext=mp4]/best\""
<< ".mp4\" " << entry->webpage_url << " > \"" << XGConfig::downloader.cachedir " --merge-output-format mp4 -o \"$$DL_FILE\" $$DL_URL > \"$$DL_PROG_BUF_FILE\"";
<< "/dlprogbuf/" << entry->tubio_id << ".buf" << "\"";
ytdl_call_video = Internal::StringHelpers::Replace(ytdl_call_video, "$$DL_RATE", XGConfig::downloader.max_dlrate_per_thread);
ytdl_call_video = Internal::StringHelpers::Replace(ytdl_call_video, "$$DL_FILE", XGConfig::downloader.cachedir + "/download/" + entry->tubio_id + ".%(ext)s");
ytdl_call_video = Internal::StringHelpers::Replace(ytdl_call_video, "$$DL_URL", entry->webpage_url);
ytdl_call_video = Internal::StringHelpers::Replace(ytdl_call_video, "$$DL_PROG_BUF_FILE", XGConfig::downloader.cachedir + "/dlprogbuf/" + entry->tubio_id + ".buf");
ss << ytdl_call_video;
} }
else // DOWNLOAD_MODE::AUDIO else // DOWNLOAD_MODE::AUDIO
{ {
ss << "youtube-dl --newline --no-call-home --no-playlist --limit-rate " << XGConfig::downloader.max_dlrate_per_thread std::string ytdl_call_audio =
<< " --no-mtime --no-cache-dir --audio-format mp3 --audio-quality 0 --extract-audio -o \"" "youtube-dl --newline --no-call-home --no-playlist --limit-rate $$DL_RATE"
<< XGConfig::downloader.cachedir << "/download/" << entry->tubio_id << ".%(ext)s\" " " --no-mtime --no-cache-dir --audio-format mp3 --audio-quality 0 --extract-audio -o \"$$DL_FILE\""
<< entry->webpage_url << " > \"" << XGConfig::downloader.cachedir " $$DL_URL > \"$$DL_PROG_BUF_FILE\"";
<< "/dlprogbuf/" << entry->tubio_id << ".buf" << "\"";
ytdl_call_audio = Internal::StringHelpers::Replace(ytdl_call_audio, "$$DL_RATE", XGConfig::downloader.max_dlrate_per_thread);
ytdl_call_audio = Internal::StringHelpers::Replace(ytdl_call_audio, "$$DL_FILE", XGConfig::downloader.cachedir + "/download/" + entry->tubio_id + ".%(ext)s");
ytdl_call_audio = Internal::StringHelpers::Replace(ytdl_call_audio, "$$DL_URL", entry->webpage_url);
ytdl_call_audio = Internal::StringHelpers::Replace(ytdl_call_audio, "$$DL_PROG_BUF_FILE", XGConfig::downloader.cachedir + "/dlprogbuf/" + entry->tubio_id + ".buf");
ss << ytdl_call_audio;
} }
int returnCode = system(ss.str().c_str()); int returnCode = system(ss.str().c_str());
std::cout << returnCode << std::endl; std::cout << returnCode << std::endl;
entry->status = DOWNLOAD_STATUS::FINISHED; if (returnCode == 0)
entry->download_progress = 100; {
shouldSave = true; // Download succeeded
entry->status = DOWNLOAD_STATUS::FINISHED;
entry->download_progress = 100;
shouldSave = true;
}
else
{
// Download failed
entry->status = DOWNLOAD_STATUS::FAILED;
entry->download_progress = -1;
}
return; return;
}); });
downloadThreads.push_back(downloadThread); downloadThreads.push_back(downloadThread);
return; return;
@ -223,18 +262,29 @@ JsonArray DownloadManager::GetQueueAsJson()
return arr; return arr;
} }
void Downloader::DownloadManager::ClearDownloadCache() bool DownloadManager::ClearDownloadCache()
{ {
if (FileSystem::ExistsDirectory(XGConfig::downloader.cachedir)) if (GetNumActiveDownloads() == 0)
{ {
FileSystem::DeleteDirectory(XGConfig::downloader.cachedir); log->cout << "Clearing download cache...";
FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/metadata"); log->Flush();
FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/download");
FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/dlprogbuf"); if (FileSystem::ExistsDirectory(XGConfig::downloader.cachedir))
queue.clear(); {
FileSystem::DeleteDirectory(XGConfig::downloader.cachedir);
FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/metadata");
FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/download");
FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/dlprogbuf");
queue.clear();
}
return true;
} }
return; log->cout << "Download cache will be cleared as soon as possible...";
log->Flush();
shouldClearCacheASAP = true;
return false;
} }
void DownloadManager::Save() void DownloadManager::Save()
@ -403,7 +453,7 @@ std::string DownloadManager::CreateNewTubioID()
return newId; return newId;
} }
std::size_t Downloader::DownloadManager::GetNumActiveDownloads() std::size_t DownloadManager::GetNumActiveDownloads()
{ {
std::size_t counter = 0; std::size_t counter = 0;
for (std::size_t i = 0; i < queue.size(); i++) for (std::size_t i = 0; i < queue.size(); i++)
@ -439,8 +489,9 @@ void DownloadManager::PostExit()
std::vector<DownloadEntry> DownloadManager::queue; std::vector<DownloadEntry> DownloadManager::queue;
std::vector<std::thread*> DownloadManager::downloadThreads; std::vector<std::thread*> DownloadManager::downloadThreads;
::Logging::Logger* DownloadManager::log; ::Logging::Logger* DownloadManager::log;
bool DownloadManager::shouldSave = false;
time_t DownloadManager::lastProgressCheck = 0; time_t DownloadManager::lastProgressCheck = 0;
bool DownloadManager::shouldSave = false;
bool DownloadManager::shouldClearCacheASAP = false;

7
Tubio/DownloadManager.h
View File

@ -67,8 +67,10 @@ namespace Downloader
/// <summary> /// <summary>
/// Will delete all cached downloads! /// Will delete all cached downloads!
/// If downloads are currently active, tubio will wait for them to finish and return false!
/// If no downloads are active it will clear immediately and return true
/// </summary> /// </summary>
static void ClearDownloadCache(); static bool ClearDownloadCache();
private: private:
static void Save(); static void Save();
@ -89,7 +91,8 @@ namespace Downloader
static std::vector<std::thread*> downloadThreads; static std::vector<std::thread*> downloadThreads;
static Logging::Logger* log; static Logging::Logger* log;
// This gets set by other threads // This gets set by other threads
static bool shouldSave;
static time_t lastProgressCheck; static time_t lastProgressCheck;
static bool shouldSave;
static bool shouldClearCacheASAP;
}; };
} }

31
Tubio/RestQueryHandler.cpp
View File

@ -96,6 +96,9 @@ bool RestQueryHandler::QueueDownload(const JsonBlock& request, JsonBlock& respon
bool RestQueryHandler::FetchQueue(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode) bool RestQueryHandler::FetchQueue(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode)
{ {
log->cout << "Asking for queue...";
log->Flush();
responseCode = OK; responseCode = OK;
responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK)); responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK));
responseBody.Set("queue") = DownloadManager::GetQueueAsJson(); responseBody.Set("queue") = DownloadManager::GetQueueAsJson();
@ -104,13 +107,24 @@ bool RestQueryHandler::FetchQueue(const JsonBlock& request, JsonBlock& responseB
bool RestQueryHandler::ClearDownloadCache(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode) bool RestQueryHandler::ClearDownloadCache(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode)
{ {
responseCode = OK;
responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK));
DownloadManager::ClearDownloadCache();
log->cout << "Clearing download cache..."; log->cout << "Clearing download cache...";
log->Flush(); log->Flush();
bool wait = !DownloadManager::ClearDownloadCache();
responseCode = OK;
responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK));
if (wait)
{
responseBody.Set("status") = "OK_WAIT";
responseBody.Set("message") = "Download cache cannot be cleared right now because there are active downloads, but will be cleared as soon as those have finished!";
}
else
{
responseBody.Set("message") = "Download cache has been cleared!";
}
return true; return true;
} }
@ -131,6 +145,9 @@ bool RestQueryHandler::HideConsole(const JsonBlock& request, JsonBlock& response
{ {
if (ConsoleManager::IsSupported()) if (ConsoleManager::IsSupported())
{ {
log->cout << "Hiding console...";
log->Flush();
bool didAnythingChange = ConsoleManager::HideConsole(); bool didAnythingChange = ConsoleManager::HideConsole();
responseCode = OK; responseCode = OK;
responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK)); responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK));
@ -150,6 +167,9 @@ bool RestQueryHandler::ShowConsole(const JsonBlock& request, JsonBlock& response
{ {
if (ConsoleManager::IsSupported()) if (ConsoleManager::IsSupported())
{ {
log->cout << "Showing console...";
log->Flush();
bool didAnythingChange = ConsoleManager::ShowConsole(); bool didAnythingChange = ConsoleManager::ShowConsole();
responseCode = OK; responseCode = OK;
responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK)); responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK));
@ -167,6 +187,9 @@ bool RestQueryHandler::ShowConsole(const JsonBlock& request, JsonBlock& response
bool RestQueryHandler::GetOSName(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode) bool RestQueryHandler::GetOSName(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode)
{ {
log->cout << "Asking for server OS name...";
log->Flush();
std::string osName = "other"; std::string osName = "other";
#ifdef _WIN #ifdef _WIN
osName = "Windows"; osName = "Windows";