diff --git a/Tubio/DownloadManager.cpp b/Tubio/DownloadManager.cpp index a1590ea..b5942a7 100644 --- a/Tubio/DownloadManager.cpp +++ b/Tubio/DownloadManager.cpp @@ -26,56 +26,63 @@ std::string DownloadManager::QueueDownload(std::string url, DOWNLOAD_MODE mode) DownloadEntry newDownload; newDownload.tubio_id = tubioId; - newDownload.status = DOWNLOAD_STATUS::QUEUED; newDownload.mode = mode; newDownload.download_progress = 0; - Json j; - j.Parse(jsString); - if (j.GetDataType() != JDType::JSON) + if (!IsJsonValid(jsString)) { newDownload.status = DOWNLOAD_STATUS::FAILED; } else { - if ((j.AsJson.DoesExist("title")) && (j.AsJson["title"].GetDataType() == JDType::STRING)) + newDownload.status = DOWNLOAD_STATUS::QUEUED; + Json j; + j.Parse(jsString); + if (j.GetDataType() != JDType::JSON) { - newDownload.title = j["title"]; + newDownload.status = DOWNLOAD_STATUS::FAILED; } - - if ((j.AsJson.DoesExist("description")) && (j.AsJson["description"].GetDataType() == JDType::STRING)) + else { - newDownload.description = j["description"]; - } - - if ((j.AsJson.DoesExist("uploader")) && (j.AsJson["uploader"].GetDataType() == JDType::STRING)) - { - newDownload.uploader = j["uploader"]; - } - - if ((j.AsJson.DoesExist("duration")) && (j.AsJson["duration"].GetDataType() == JDType::INT)) - { - newDownload.duration = j["duration"]; - } - - if ((j.AsJson.DoesExist("webpage_url")) && (j.AsJson["webpage_url"].GetDataType() == JDType::STRING)) - { - newDownload.webpage_url = j["webpage_url"]; - } - - if ((j.AsJson.DoesExist("thumbnails")) && (j.AsJson["thumbnails"].GetDataType() == JDType::ARRAY)) - { - JsonArray& thumbnails = j.AsJson["thumbnails"].AsArray; - if (thumbnails.Size() > 0) + if ((j.AsJson.DoesExist("title")) && (j.AsJson["title"].GetDataType() == JDType::STRING)) { - if (thumbnails.Size() > 1) + newDownload.title = j["title"]; + } + + if ((j.AsJson.DoesExist("description")) && (j.AsJson["description"].GetDataType() == JDType::STRING)) + { + newDownload.description = j["description"]; + } + + if ((j.AsJson.DoesExist("uploader")) && (j.AsJson["uploader"].GetDataType() == JDType::STRING)) + { + newDownload.uploader = j["uploader"]; + } + + if ((j.AsJson.DoesExist("duration")) && (j.AsJson["duration"].GetDataType() == JDType::INT)) + { + newDownload.duration = j["duration"]; + } + + if ((j.AsJson.DoesExist("webpage_url")) && (j.AsJson["webpage_url"].GetDataType() == JDType::STRING)) + { + newDownload.webpage_url = j["webpage_url"]; + } + + if ((j.AsJson.DoesExist("thumbnails")) && (j.AsJson["thumbnails"].GetDataType() == JDType::ARRAY)) + { + JsonArray& thumbnails = j.AsJson["thumbnails"].AsArray; + if (thumbnails.Size() > 0) { - // If we have more than one thumbnail to choose from, choose the second-highes quality. - newDownload.thumbnail_url = thumbnails[thumbnails.Size() - 2]["url"]; - } - else - { - newDownload.thumbnail_url = thumbnails[thumbnails.Size() - 1]["url"]; + if (thumbnails.Size() > 1) + { + // If we have more than one thumbnail to choose from, choose the second-highes quality. + newDownload.thumbnail_url = thumbnails[thumbnails.Size() - 2]["url"]; + } + else + { + newDownload.thumbnail_url = thumbnails[thumbnails.Size() - 1]["url"]; + } } } } @@ -90,24 +97,33 @@ void DownloadManager::Update() { if (shouldSave) Save(); - std::size_t numActiveDownloads = GetNumActiveDownloads(); + std::size_t cachedNumActiveDownloads = GetNumActiveDownloads(); - if (numActiveDownloads < XGConfig::downloader.num_threads) + // Queue next download, if available + if (cachedNumActiveDownloads < XGConfig::downloader.num_threads) { DownloadNext(); } // Check every second, non-blocking - if ((numActiveDownloads > 0) && (time(0) - lastProgressCheck > 2)) + if ((time(0) - lastProgressCheck > 2) && (cachedNumActiveDownloads > 0)) { UpdateDownloadProgressPercentages(); } + // Clear cache, if requested + if ((shouldClearCacheASAP) && (cachedNumActiveDownloads == 0)) + { + shouldClearCacheASAP = false; + ClearDownloadCache(); + } + return; } void DownloadManager::DownloadNext() { + // Abort, if queue is empty if (GetQueueLength() == 0) return; DownloadEntry* next = nullptr; @@ -128,29 +144,52 @@ void DownloadManager::DownloadNext() std::stringstream ss; if (entry->mode == DOWNLOAD_MODE::VIDEO) { - ss << "youtube-dl --newline --no-call-home --no-playlist --limit-rate " << XGConfig::downloader.max_dlrate_per_thread - << " --no-mtime --no-cache-dir --format \"bestvideo[ext=mp4]+bestaudio/best[ext=mp4]/best\" --merge-output-format mp4" - << " -o \"" << XGConfig::downloader.cachedir << "/download/" << entry->tubio_id - << ".mp4\" " << entry->webpage_url << " > \"" << XGConfig::downloader.cachedir - << "/dlprogbuf/" << entry->tubio_id << ".buf" << "\""; + std::string ytdl_call_video = + "youtube-dl --newline --no-call-home --no-playlist --limit-rate $$DL_RATE" + " --no-mtime --no-cache-dir --format \"bestvideo[ext=mp4]+bestaudio/best[ext=mp4]/best\"" + " --merge-output-format mp4 -o \"$$DL_FILE\" $$DL_URL > \"$$DL_PROG_BUF_FILE\""; + + ytdl_call_video = Internal::StringHelpers::Replace(ytdl_call_video, "$$DL_RATE", XGConfig::downloader.max_dlrate_per_thread); + ytdl_call_video = Internal::StringHelpers::Replace(ytdl_call_video, "$$DL_FILE", XGConfig::downloader.cachedir + "/download/" + entry->tubio_id + ".%(ext)s"); + ytdl_call_video = Internal::StringHelpers::Replace(ytdl_call_video, "$$DL_URL", entry->webpage_url); + ytdl_call_video = Internal::StringHelpers::Replace(ytdl_call_video, "$$DL_PROG_BUF_FILE", XGConfig::downloader.cachedir + "/dlprogbuf/" + entry->tubio_id + ".buf"); + + ss << ytdl_call_video; } else // DOWNLOAD_MODE::AUDIO { - ss << "youtube-dl --newline --no-call-home --no-playlist --limit-rate " << XGConfig::downloader.max_dlrate_per_thread - << " --no-mtime --no-cache-dir --audio-format mp3 --audio-quality 0 --extract-audio -o \"" - << XGConfig::downloader.cachedir << "/download/" << entry->tubio_id << ".%(ext)s\" " - << entry->webpage_url << " > \"" << XGConfig::downloader.cachedir - << "/dlprogbuf/" << entry->tubio_id << ".buf" << "\""; + std::string ytdl_call_audio = + "youtube-dl --newline --no-call-home --no-playlist --limit-rate $$DL_RATE" + " --no-mtime --no-cache-dir --audio-format mp3 --audio-quality 0 --extract-audio -o \"$$DL_FILE\"" + " $$DL_URL > \"$$DL_PROG_BUF_FILE\""; + + ytdl_call_audio = Internal::StringHelpers::Replace(ytdl_call_audio, "$$DL_RATE", XGConfig::downloader.max_dlrate_per_thread); + ytdl_call_audio = Internal::StringHelpers::Replace(ytdl_call_audio, "$$DL_FILE", XGConfig::downloader.cachedir + "/download/" + entry->tubio_id + ".%(ext)s"); + ytdl_call_audio = Internal::StringHelpers::Replace(ytdl_call_audio, "$$DL_URL", entry->webpage_url); + ytdl_call_audio = Internal::StringHelpers::Replace(ytdl_call_audio, "$$DL_PROG_BUF_FILE", XGConfig::downloader.cachedir + "/dlprogbuf/" + entry->tubio_id + ".buf"); + + ss << ytdl_call_audio; } int returnCode = system(ss.str().c_str()); std::cout << returnCode << std::endl; - entry->status = DOWNLOAD_STATUS::FINISHED; - entry->download_progress = 100; - shouldSave = true; + if (returnCode == 0) + { + // Download succeeded + entry->status = DOWNLOAD_STATUS::FINISHED; + entry->download_progress = 100; + shouldSave = true; + } + else + { + // Download failed + entry->status = DOWNLOAD_STATUS::FAILED; + entry->download_progress = -1; + } return; }); + downloadThreads.push_back(downloadThread); return; @@ -223,18 +262,29 @@ JsonArray DownloadManager::GetQueueAsJson() return arr; } -void Downloader::DownloadManager::ClearDownloadCache() +bool DownloadManager::ClearDownloadCache() { - if (FileSystem::ExistsDirectory(XGConfig::downloader.cachedir)) + if (GetNumActiveDownloads() == 0) { - FileSystem::DeleteDirectory(XGConfig::downloader.cachedir); - FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/metadata"); - FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/download"); - FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/dlprogbuf"); - queue.clear(); + log->cout << "Clearing download cache..."; + log->Flush(); + + if (FileSystem::ExistsDirectory(XGConfig::downloader.cachedir)) + { + FileSystem::DeleteDirectory(XGConfig::downloader.cachedir); + FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/metadata"); + FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/download"); + FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/dlprogbuf"); + queue.clear(); + } + + return true; } - return; + log->cout << "Download cache will be cleared as soon as possible..."; + log->Flush(); + shouldClearCacheASAP = true; + return false; } void DownloadManager::Save() @@ -403,7 +453,7 @@ std::string DownloadManager::CreateNewTubioID() return newId; } -std::size_t Downloader::DownloadManager::GetNumActiveDownloads() +std::size_t DownloadManager::GetNumActiveDownloads() { std::size_t counter = 0; for (std::size_t i = 0; i < queue.size(); i++) @@ -439,8 +489,9 @@ void DownloadManager::PostExit() std::vector DownloadManager::queue; std::vector DownloadManager::downloadThreads; ::Logging::Logger* DownloadManager::log; -bool DownloadManager::shouldSave = false; time_t DownloadManager::lastProgressCheck = 0; +bool DownloadManager::shouldSave = false; +bool DownloadManager::shouldClearCacheASAP = false; diff --git a/Tubio/DownloadManager.h b/Tubio/DownloadManager.h index e093938..8660ca9 100644 --- a/Tubio/DownloadManager.h +++ b/Tubio/DownloadManager.h @@ -67,8 +67,10 @@ namespace Downloader /// /// Will delete all cached downloads! + /// If downloads are currently active, tubio will wait for them to finish and return false! + /// If no downloads are active it will clear immediately and return true /// - static void ClearDownloadCache(); + static bool ClearDownloadCache(); private: static void Save(); @@ -89,7 +91,8 @@ namespace Downloader static std::vector downloadThreads; static Logging::Logger* log; // This gets set by other threads - static bool shouldSave; static time_t lastProgressCheck; + static bool shouldSave; + static bool shouldClearCacheASAP; }; } diff --git a/Tubio/RestQueryHandler.cpp b/Tubio/RestQueryHandler.cpp index 71d7d8c..41d654c 100644 --- a/Tubio/RestQueryHandler.cpp +++ b/Tubio/RestQueryHandler.cpp @@ -96,6 +96,9 @@ bool RestQueryHandler::QueueDownload(const JsonBlock& request, JsonBlock& respon bool RestQueryHandler::FetchQueue(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode) { + log->cout << "Asking for queue..."; + log->Flush(); + responseCode = OK; responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK)); responseBody.Set("queue") = DownloadManager::GetQueueAsJson(); @@ -104,13 +107,24 @@ bool RestQueryHandler::FetchQueue(const JsonBlock& request, JsonBlock& responseB bool RestQueryHandler::ClearDownloadCache(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode) { - responseCode = OK; - responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK)); - DownloadManager::ClearDownloadCache(); - log->cout << "Clearing download cache..."; log->Flush(); + bool wait = !DownloadManager::ClearDownloadCache(); + + responseCode = OK; + responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK)); + + if (wait) + { + responseBody.Set("status") = "OK_WAIT"; + responseBody.Set("message") = "Download cache cannot be cleared right now because there are active downloads, but will be cleared as soon as those have finished!"; + } + else + { + responseBody.Set("message") = "Download cache has been cleared!"; + } + return true; } @@ -131,6 +145,9 @@ bool RestQueryHandler::HideConsole(const JsonBlock& request, JsonBlock& response { if (ConsoleManager::IsSupported()) { + log->cout << "Hiding console..."; + log->Flush(); + bool didAnythingChange = ConsoleManager::HideConsole(); responseCode = OK; responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK)); @@ -150,6 +167,9 @@ bool RestQueryHandler::ShowConsole(const JsonBlock& request, JsonBlock& response { if (ConsoleManager::IsSupported()) { + log->cout << "Showing console..."; + log->Flush(); + bool didAnythingChange = ConsoleManager::ShowConsole(); responseCode = OK; responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK)); @@ -167,6 +187,9 @@ bool RestQueryHandler::ShowConsole(const JsonBlock& request, JsonBlock& response bool RestQueryHandler::GetOSName(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode) { + log->cout << "Asking for server OS name..."; + log->Flush(); + std::string osName = "other"; #ifdef _WIN osName = "Windows";