leonetienne/Tubio
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added failure-detection for downloads, and made the clear_download_cache method wait asyncronously for all active downloads to finish

Browse Source
This commit is contained in:
Leon Etienne (ubuntu wsl) 2020-09-27 13:46:15 +02:00
parent cdd8eded30
commit a19d27203f
3 changed files with 146 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
Tubio/DownloadManager.cpp
View File

@ -26,10 +26,16 @@ std::string DownloadManager::QueueDownload(std::string url, DOWNLOAD_MODE mode)
DownloadEntry newDownload;
newDownload.tubio_id = tubioId;
newDownload.status = DOWNLOAD_STATUS::QUEUED;
newDownload.mode = mode;
newDownload.download_progress = 0;
if (!IsJsonValid(jsString))
{
newDownload.status = DOWNLOAD_STATUS::FAILED;
}
else
{
newDownload.status = DOWNLOAD_STATUS::QUEUED;
Json j;
j.Parse(jsString);
if (j.GetDataType() != JDType::JSON)
@ -80,6 +86,7 @@ std::string DownloadManager::QueueDownload(std::string url, DOWNLOAD_MODE mode)
}
}
}
}
queue.push_back(newDownload);
@ -90,24 +97,33 @@ void DownloadManager::Update()
{
if (shouldSave) Save();
std::size_t numActiveDownloads = GetNumActiveDownloads();
std::size_t cachedNumActiveDownloads = GetNumActiveDownloads();
if (numActiveDownloads < XGConfig::downloader.num_threads)
// Queue next download, if available
if (cachedNumActiveDownloads < XGConfig::downloader.num_threads)
{
DownloadNext();
}
// Check every second, non-blocking
if ((numActiveDownloads > 0) && (time(0) - lastProgressCheck > 2))
if ((time(0) - lastProgressCheck > 2) && (cachedNumActiveDownloads > 0))
{
UpdateDownloadProgressPercentages();
}
// Clear cache, if requested
if ((shouldClearCacheASAP) && (cachedNumActiveDownloads == 0))
{
shouldClearCacheASAP = false;
ClearDownloadCache();
}
return;
}
void DownloadManager::DownloadNext()
{
// Abort, if queue is empty
if (GetQueueLength() == 0) return;
DownloadEntry* next = nullptr;
@ -128,29 +144,52 @@ void DownloadManager::DownloadNext()
std::stringstream ss;
if (entry->mode == DOWNLOAD_MODE::VIDEO)
{
ss << "youtube-dl --newline --no-call-home --no-playlist --limit-rate " << XGConfig::downloader.max_dlrate_per_thread
<< " --no-mtime --no-cache-dir --format \"bestvideo[ext=mp4]+bestaudio/best[ext=mp4]/best\" --merge-output-format mp4"
<< " -o \"" << XGConfig::downloader.cachedir << "/download/" << entry->tubio_id
<< ".mp4\" " << entry->webpage_url << " > \"" << XGConfig::downloader.cachedir
<< "/dlprogbuf/" << entry->tubio_id << ".buf" << "\"";
std::string ytdl_call_video =
"youtube-dl --newline --no-call-home --no-playlist --limit-rate $$DL_RATE"
" --no-mtime --no-cache-dir --format \"bestvideo[ext=mp4]+bestaudio/best[ext=mp4]/best\""
" --merge-output-format mp4 -o \"$$DL_FILE\" $$DL_URL > \"$$DL_PROG_BUF_FILE\"";
ytdl_call_video = Internal::StringHelpers::Replace(ytdl_call_video, "$$DL_RATE", XGConfig::downloader.max_dlrate_per_thread);
ytdl_call_video = Internal::StringHelpers::Replace(ytdl_call_video, "$$DL_FILE", XGConfig::downloader.cachedir + "/download/" + entry->tubio_id + ".%(ext)s");
ytdl_call_video = Internal::StringHelpers::Replace(ytdl_call_video, "$$DL_URL", entry->webpage_url);
ytdl_call_video = Internal::StringHelpers::Replace(ytdl_call_video, "$$DL_PROG_BUF_FILE", XGConfig::downloader.cachedir + "/dlprogbuf/" + entry->tubio_id + ".buf");
ss << ytdl_call_video;
}
else // DOWNLOAD_MODE::AUDIO
{
ss << "youtube-dl --newline --no-call-home --no-playlist --limit-rate " << XGConfig::downloader.max_dlrate_per_thread
<< " --no-mtime --no-cache-dir --audio-format mp3 --audio-quality 0 --extract-audio -o \""
<< XGConfig::downloader.cachedir << "/download/" << entry->tubio_id << ".%(ext)s\" "
<< entry->webpage_url << " > \"" << XGConfig::downloader.cachedir
<< "/dlprogbuf/" << entry->tubio_id << ".buf" << "\"";
std::string ytdl_call_audio =
"youtube-dl --newline --no-call-home --no-playlist --limit-rate $$DL_RATE"
" --no-mtime --no-cache-dir --audio-format mp3 --audio-quality 0 --extract-audio -o \"$$DL_FILE\""
" $$DL_URL > \"$$DL_PROG_BUF_FILE\"";
ytdl_call_audio = Internal::StringHelpers::Replace(ytdl_call_audio, "$$DL_RATE", XGConfig::downloader.max_dlrate_per_thread);
ytdl_call_audio = Internal::StringHelpers::Replace(ytdl_call_audio, "$$DL_FILE", XGConfig::downloader.cachedir + "/download/" + entry->tubio_id + ".%(ext)s");
ytdl_call_audio = Internal::StringHelpers::Replace(ytdl_call_audio, "$$DL_URL", entry->webpage_url);
ytdl_call_audio = Internal::StringHelpers::Replace(ytdl_call_audio, "$$DL_PROG_BUF_FILE", XGConfig::downloader.cachedir + "/dlprogbuf/" + entry->tubio_id + ".buf");
ss << ytdl_call_audio;
}
int returnCode = system(ss.str().c_str());
std::cout << returnCode << std::endl;
if (returnCode == 0)
{
// Download succeeded
entry->status = DOWNLOAD_STATUS::FINISHED;
entry->download_progress = 100;
shouldSave = true;
}
else
{
// Download failed
entry->status = DOWNLOAD_STATUS::FAILED;
entry->download_progress = -1;
}
return;
});
downloadThreads.push_back(downloadThread);
return;
@ -223,8 +262,13 @@ JsonArray DownloadManager::GetQueueAsJson()
return arr;
}
void Downloader::DownloadManager::ClearDownloadCache()
bool DownloadManager::ClearDownloadCache()
{
if (GetNumActiveDownloads() == 0)
{
log->cout << "Clearing download cache...";
log->Flush();
if (FileSystem::ExistsDirectory(XGConfig::downloader.cachedir))
{
FileSystem::DeleteDirectory(XGConfig::downloader.cachedir);
@ -234,7 +278,13 @@ void Downloader::DownloadManager::ClearDownloadCache()
queue.clear();
}
return;
return true;
}
log->cout << "Download cache will be cleared as soon as possible...";
log->Flush();
shouldClearCacheASAP = true;
return false;
}
void DownloadManager::Save()
@ -403,7 +453,7 @@ std::string DownloadManager::CreateNewTubioID()
return newId;
}
std::size_t Downloader::DownloadManager::GetNumActiveDownloads()
std::size_t DownloadManager::GetNumActiveDownloads()
{
std::size_t counter = 0;
for (std::size_t i = 0; i < queue.size(); i++)
@ -439,8 +489,9 @@ void DownloadManager::PostExit()
std::vector<DownloadEntry> DownloadManager::queue;
std::vector<std::thread*> DownloadManager::downloadThreads;
::Logging::Logger* DownloadManager::log;
bool DownloadManager::shouldSave = false;
time_t DownloadManager::lastProgressCheck = 0;
bool DownloadManager::shouldSave = false;
bool DownloadManager::shouldClearCacheASAP = false;

7
Tubio/DownloadManager.h
View File

@ -67,8 +67,10 @@ namespace Downloader
/// <summary>
/// Will delete all cached downloads!
/// If downloads are currently active, tubio will wait for them to finish and return false!
/// If no downloads are active it will clear immediately and return true
/// </summary>
static void ClearDownloadCache();
static bool ClearDownloadCache();
private:
static void Save();
@ -89,7 +91,8 @@ namespace Downloader
static std::vector<std::thread*> downloadThreads;
static Logging::Logger* log;
// This gets set by other threads
static bool shouldSave;
static time_t lastProgressCheck;
static bool shouldSave;
static bool shouldClearCacheASAP;
};
}

31
Tubio/RestQueryHandler.cpp
View File

@ -96,6 +96,9 @@ bool RestQueryHandler::QueueDownload(const JsonBlock& request, JsonBlock& respon
bool RestQueryHandler::FetchQueue(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode)
{
log->cout << "Asking for queue...";
log->Flush();
responseCode = OK;
responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK));
responseBody.Set("queue") = DownloadManager::GetQueueAsJson();
@ -104,13 +107,24 @@ bool RestQueryHandler::FetchQueue(const JsonBlock& request, JsonBlock& responseB
bool RestQueryHandler::ClearDownloadCache(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode)
{
responseCode = OK;
responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK));
DownloadManager::ClearDownloadCache();
log->cout << "Clearing download cache...";
log->Flush();
bool wait = !DownloadManager::ClearDownloadCache();
responseCode = OK;
responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK));
if (wait)
{
responseBody.Set("status") = "OK_WAIT";
responseBody.Set("message") = "Download cache cannot be cleared right now because there are active downloads, but will be cleared as soon as those have finished!";
}
else
{
responseBody.Set("message") = "Download cache has been cleared!";
}
return true;
}
@ -131,6 +145,9 @@ bool RestQueryHandler::HideConsole(const JsonBlock& request, JsonBlock& response
{
if (ConsoleManager::IsSupported())
{
log->cout << "Hiding console...";
log->Flush();
bool didAnythingChange = ConsoleManager::HideConsole();
responseCode = OK;
responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK));
@ -150,6 +167,9 @@ bool RestQueryHandler::ShowConsole(const JsonBlock& request, JsonBlock& response
{
if (ConsoleManager::IsSupported())
{
log->cout << "Showing console...";
log->Flush();
bool didAnythingChange = ConsoleManager::ShowConsole();
responseCode = OK;
responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK));
@ -167,6 +187,9 @@ bool RestQueryHandler::ShowConsole(const JsonBlock& request, JsonBlock& response
bool RestQueryHandler::GetOSName(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode)
{
log->cout << "Asking for server OS name...";
log->Flush();
std::string osName = "other";
#ifdef _WIN
osName = "Windows";