From c44d56f27b29c76a20566b60e9dd616f46661f7d Mon Sep 17 00:00:00 2001 From: "Leon Etienne (ubuntu wsl)" Date: Mon, 28 Sep 2020 13:02:10 +0200 Subject: [PATCH] Fixed a bit of bullshittery and added max_age and max_num controls to fetch_session_cache --- Art/Blender/logo.svg | 1 - Tubio/DownloadManager.cpp | 250 +++++++++++------- Tubio/DownloadManager.h | 12 +- Tubio/HttpServer.cpp | 2 +- Tubio/RestQueryHandler.cpp | 41 ++- Tubio/RestQueryHandler.h | 3 +- Tubio/XGControl.cpp | 1 + Tubio/XGControl.h | 2 + .../components/Layout/Header.vue | 2 - 9 files changed, 207 insertions(+), 107 deletions(-) delete mode 100644 Art/Blender/logo.svg diff --git a/Art/Blender/logo.svg b/Art/Blender/logo.svg deleted file mode 100644 index b32ac84..0000000 --- a/Art/Blender/logo.svg +++ /dev/null @@ -1 +0,0 @@ -Element 2 \ No newline at end of file diff --git a/Tubio/DownloadManager.cpp b/Tubio/DownloadManager.cpp index 48705b7..b6f63b1 100644 --- a/Tubio/DownloadManager.cpp +++ b/Tubio/DownloadManager.cpp @@ -28,6 +28,7 @@ std::string DownloadManager::QueueDownload(std::string url, DOWNLOAD_MODE mode) newDownload.tubio_id = tubioId; newDownload.mode = mode; newDownload.download_progress = 0; + newDownload.queued_timestamp = time(0); newDownload.download_url = "/download/" + newDownload.tubio_id; if (!IsJsonValid(jsString)) @@ -89,7 +90,7 @@ std::string DownloadManager::QueueDownload(std::string url, DOWNLOAD_MODE mode) } } - queue.push_back(newDownload); + unfinishedCache.push_back(newDownload); return tubioId; } @@ -130,11 +131,11 @@ void DownloadManager::DownloadNext() DownloadEntry* next = nullptr; - for (std::size_t i = 0; i < queue.size(); i++) + for (std::size_t i = 0; i < unfinishedCache.size(); i++) { - if (queue[i].status == DOWNLOAD_STATUS::QUEUED) + if (unfinishedCache[i].status == DOWNLOAD_STATUS::QUEUED) { - next = &queue[i]; + next = &unfinishedCache[i]; break; } } @@ -200,11 +201,11 @@ void DownloadManager::DownloadNext() void DownloadManager::UpdateDownloadProgressPercentages() { - for (std::size_t i = 0; i < queue.size(); i++) + for (std::size_t i = 0; i < unfinishedCache.size(); i++) { - if (queue[i].status == DOWNLOAD_STATUS::DOWNLOADING) + if (unfinishedCache[i].status == DOWNLOAD_STATUS::DOWNLOADING) { - std::string filePath = XGConfig::downloader.cachedir + "/dlprogbuf/" + queue[i].tubio_id + ".buf"; + std::string filePath = XGConfig::downloader.cachedir + "/dlprogbuf/" + unfinishedCache[i].tubio_id + ".buf"; if (FileSystem::Exists(filePath)) { std::ifstream ifs; @@ -227,9 +228,9 @@ void DownloadManager::UpdateDownloadProgressPercentages() if (ss.str().length() > 0) { int newPercentage = std::stoi(ss.str()); - queue[i].download_progress = newPercentage; + unfinishedCache[i].download_progress = newPercentage; - //if (newPercentage == 100) queue[i].status = DOWNLOAD_STATUS::FINISHED; + if (newPercentage == 100) unfinishedCache[i].status = DOWNLOAD_STATUS::FINISHED; } } } @@ -247,39 +248,79 @@ void DownloadManager::UpdateDownloadProgressPercentages() std::size_t DownloadManager::GetQueueLength() { std::size_t counter = 0; - for (std::size_t i = 0; i < queue.size(); i++) + for (std::size_t i = 0; i < unfinishedCache.size(); i++) { - if (queue[i].status == DOWNLOAD_STATUS::QUEUED) counter++; + if (unfinishedCache[i].status == DOWNLOAD_STATUS::QUEUED) counter++; } return counter; } -JsonArray DownloadManager::GetQueueAsJson() +JsonArray DownloadManager::GetAlltimeCacheAsJson(time_t max_age, std::size_t max_num) { JsonArray arr; - for (std::size_t i = 0; i < queue.size(); i++) + for (std::size_t i = 0; i < unfinishedCache.size(); i++) { - arr += queue[i].GetAsJson(); + arr += unfinishedCache[i].GetAsJson(); } - return arr; + arr.Merge(saveFileCache); + arr.Sort("queued_timestamp", JSON_ARRAY_SORT_MODE::NUM_DESC); + + // Both limits are inifnite. Just return arr as is + if ((max_age == -1) && (max_num == (std::size_t)-1)) return arr; + + JsonArray cutArr; + // If max_num is -1 (would mean inifnite) it would underflow to size_t::max + for (std::size_t i = 0; ((i < arr.Size()) && (i < max_num)); i++) + { + // If max_age is > 0, we have to check against the max age + if (max_age > 0) + { + if (arr[i].AsJson.DoesExist("queued_timestamp")) + { + if (arr[i]["queued_timestamp"].GetDataType() == JDType::INT) + { + if ((time(0) - arr[i]["queued_timestamp"].AsInt) < max_age) + { + cutArr += arr[i]; + } + } + } + } + else // If not, just insert it + { + cutArr += arr[i]; + } + } + + return cutArr; } bool DownloadManager::DoesTubioIDExist(std::string tubioId) { - for (std::size_t i = 0; i < queue.size(); i++) + for (std::size_t i = 0; i < unfinishedCache.size(); i++) { - if (queue[i].tubio_id == tubioId) return true; + if (unfinishedCache[i].tubio_id == tubioId) return true; } + for (std::size_t i = 0; i < saveFileCache_Atomic.size(); i++) + { + if (saveFileCache_Atomic[i].tubio_id == tubioId) return true; + } + return false; } DownloadEntry& DownloadManager::GetDownloadEntryByTubioID(std::string tubioId) { - for (std::size_t i = 0; i < queue.size(); i++) + for (std::size_t i = 0; i < unfinishedCache.size(); i++) { - if (queue[i].tubio_id == tubioId) return queue[i]; + if (unfinishedCache[i].tubio_id == tubioId) return unfinishedCache[i]; } + for (std::size_t i = 0; i < saveFileCache_Atomic.size(); i++) + { + if (saveFileCache_Atomic[i].tubio_id == tubioId) return saveFileCache_Atomic[i]; + } + throw std::exception("TubioID not found!"); std::terminate(); } @@ -298,7 +339,9 @@ bool DownloadManager::ClearDownloadCache() FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/metadata"); FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/download"); FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/dlprogbuf"); - queue.clear(); + unfinishedCache.clear(); + saveFileCache.Clear(); + saveFileCache_Atomic.clear(); } return true; @@ -331,21 +374,23 @@ void DownloadManager::Save() log->Flush(); } - JsonArray arr; - for (std::size_t i = 0; i < queue.size(); i++) + // Filecache has contain all and only nonsaved entries. + for (long long int i = unfinishedCache.size() - 1; i >= 0; i--) { - if (queue[i].status == DOWNLOAD_STATUS::FINISHED) + if (unfinishedCache[i].status == DOWNLOAD_STATUS::FINISHED) { - arr += queue[i].GetAsJson(); + saveFileCache += unfinishedCache[i].GetAsJson(); + unfinishedCache.erase(unfinishedCache.begin() + i); } } - Json j(arr); + Json j(saveFileCache); if (!FileSystem::WriteFile(XGConfig::downloader.cachedir + "/index.json", j.Render())) { log->cout << log->Err() << "Unable to save download cache index file!"; log->Flush(); } + saveFileCache_Atomic = ParseJsonArrayToEntries(saveFileCache); shouldSave = false; @@ -375,70 +420,8 @@ void DownloadManager::Load() if (j.GetDataType() == JDType::ARRAY) { - const JsonArray& cachedArr = j.AsArray; - - for (std::size_t i = 0; i < cachedArr.Size(); i++) - { - JsonBlock iter = cachedArr[i].AsJson; - DownloadEntry newEntry; - newEntry.download_progress = 100; - newEntry.status = DOWNLOAD_STATUS::FINISHED; // All saved entries are finished... - - if ((iter.DoesExist("title")) && (iter["title"].GetDataType() == JDType::STRING)) - { - newEntry.title = iter["title"]; - } - - if ((iter.DoesExist("description")) && (iter["description"].GetDataType() == JDType::STRING)) - { - newEntry.description = iter["description"]; - } - - if ((iter.DoesExist("uploader")) && (iter["uploader"].GetDataType() == JDType::STRING)) - { - newEntry.uploader = iter["uploader"]; - } - - if ((iter.DoesExist("duration")) && (iter["duration"].GetDataType() == JDType::INT)) - { - newEntry.duration = iter["duration"]; - } - - if ((iter.DoesExist("tubio_id")) && (iter["tubio_id"].GetDataType() == JDType::STRING)) - { - newEntry.tubio_id = iter["tubio_id"]; - } - - if ((iter.DoesExist("webpage_url")) && (iter["webpage_url"].GetDataType() == JDType::STRING)) - { - newEntry.webpage_url = iter["webpage_url"]; - } - - if ((iter.DoesExist("thumbnail_url")) && (iter["thumbnail_url"].GetDataType() == JDType::STRING)) - { - newEntry.thumbnail_url = iter["thumbnail_url"]; - } - - if ((iter.DoesExist("download_url")) && (iter["download_url"].GetDataType() == JDType::STRING)) - { - newEntry.download_url = iter["download_url"]; - } - - if ((iter.DoesExist("downloaded_filename")) && (iter["downloaded_filename"].GetDataType() == JDType::STRING)) - { - newEntry.downloaded_filename = iter["downloaded_filename"]; - } - - if ((iter.DoesExist("mode")) && (iter["mode"].GetDataType() == JDType::STRING)) - { - std::string cachedStrMode = iter["mode"]; - if (cachedStrMode == "video") newEntry.mode = DOWNLOAD_MODE::VIDEO; - else if (cachedStrMode == "audio") newEntry.mode = DOWNLOAD_MODE::AUDIO; - else newEntry.mode = DOWNLOAD_MODE::VIDEO; - } - - queue.push_back(newEntry); - } + saveFileCache.CloneFrom(j.AsArray); + saveFileCache_Atomic = ParseJsonArrayToEntries(saveFileCache); } else { @@ -455,6 +438,81 @@ void DownloadManager::Load() return; } +std::vector DownloadManager::ParseJsonArrayToEntries(const JasonPP::JsonArray& arr) +{ + std::vector entries; + + for (std::size_t i = 0; i < arr.Size(); i++) + { + JsonBlock iter = arr[i].AsJson; + DownloadEntry newEntry; + newEntry.download_progress = 100; + newEntry.status = DOWNLOAD_STATUS::FINISHED; // All saved entries are finished... + + if ((iter.DoesExist("title")) && (iter["title"].GetDataType() == JDType::STRING)) + { + newEntry.title = iter["title"]; + } + + if ((iter.DoesExist("description")) && (iter["description"].GetDataType() == JDType::STRING)) + { + newEntry.description = iter["description"]; + } + + if ((iter.DoesExist("uploader")) && (iter["uploader"].GetDataType() == JDType::STRING)) + { + newEntry.uploader = iter["uploader"]; + } + + if ((iter.DoesExist("duration")) && (iter["duration"].GetDataType() == JDType::INT)) + { + newEntry.duration = iter["duration"]; + } + + if ((iter.DoesExist("tubio_id")) && (iter["tubio_id"].GetDataType() == JDType::STRING)) + { + newEntry.tubio_id = iter["tubio_id"]; + } + + if ((iter.DoesExist("webpage_url")) && (iter["webpage_url"].GetDataType() == JDType::STRING)) + { + newEntry.webpage_url = iter["webpage_url"]; + } + + if ((iter.DoesExist("thumbnail_url")) && (iter["thumbnail_url"].GetDataType() == JDType::STRING)) + { + newEntry.thumbnail_url = iter["thumbnail_url"]; + } + + if ((iter.DoesExist("download_url")) && (iter["download_url"].GetDataType() == JDType::STRING)) + { + newEntry.download_url = iter["download_url"]; + } + + if ((iter.DoesExist("downloaded_filename")) && (iter["downloaded_filename"].GetDataType() == JDType::STRING)) + { + newEntry.downloaded_filename = iter["downloaded_filename"]; + } + + if ((iter.DoesExist("queued_timestamp")) && (iter["queued_timestamp"].GetDataType() == JDType::INT)) + { + newEntry.queued_timestamp = iter["queued_timestamp"]; + } + + if ((iter.DoesExist("mode")) && (iter["mode"].GetDataType() == JDType::STRING)) + { + std::string cachedStrMode = iter["mode"]; + if (cachedStrMode == "video") newEntry.mode = DOWNLOAD_MODE::VIDEO; + else if (cachedStrMode == "audio") newEntry.mode = DOWNLOAD_MODE::AUDIO; + else newEntry.mode = DOWNLOAD_MODE::VIDEO; + } + + entries.push_back(newEntry); + } + + return entries; +} + void DownloadManager::FetchInformation(std::string url, std::string tubId) { std::stringstream ss; @@ -476,10 +534,11 @@ std::string DownloadManager::CreateNewTubioID() newId = Internal::Helpers::Base10_2_X(time(0), "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"); isIdUnique = true; - for (std::size_t i = 0; i < queue.size(); i++) + if (unfinishedCache.size() > 0) { - if (queue[i].tubio_id == newId) isIdUnique = false; + isIdUnique = unfinishedCache[unfinishedCache.size() - 1].tubio_id != newId; } + counter++; } @@ -489,9 +548,9 @@ std::string DownloadManager::CreateNewTubioID() std::size_t DownloadManager::GetNumActiveDownloads() { std::size_t counter = 0; - for (std::size_t i = 0; i < queue.size(); i++) + for (std::size_t i = 0; i < unfinishedCache.size(); i++) { - if (queue[i].status == DOWNLOAD_STATUS::DOWNLOADING) counter++; + if (unfinishedCache[i].status == DOWNLOAD_STATUS::DOWNLOADING) counter++; } return counter; } @@ -519,8 +578,10 @@ void DownloadManager::PostExit() } -std::vector DownloadManager::queue; +std::vector DownloadManager::unfinishedCache; std::vector DownloadManager::downloadThreads; +JasonPP::JsonArray DownloadManager::saveFileCache; +std::vector DownloadManager::saveFileCache_Atomic; ::Logging::Logger* DownloadManager::log; time_t DownloadManager::lastProgressCheck = 0; bool DownloadManager::shouldSave = false; @@ -541,6 +602,7 @@ JsonBlock DownloadEntry::GetAsJson() jb.Set(Ele("download_progress", download_progress)); jb.Set(Ele("downloaded_filename", downloaded_filename)); jb.Set(Ele("download_url", download_url)); + jb.Set(Ele("queued_timestamp", (long long int)queued_timestamp)); switch (mode) { diff --git a/Tubio/DownloadManager.h b/Tubio/DownloadManager.h index 326caa6..ab31655 100644 --- a/Tubio/DownloadManager.h +++ b/Tubio/DownloadManager.h @@ -39,6 +39,7 @@ namespace Downloader DOWNLOAD_STATUS status; DOWNLOAD_MODE mode; int download_progress; + time_t queued_timestamp; JasonPP::JsonBlock GetAsJson(); @@ -67,10 +68,12 @@ namespace Downloader static std::size_t GetQueueLength(); /// - /// Will return the whole queue in json format + /// Will return the whole cache in json format /// + /// /// Maximum age of the entry in seconds. -1 = infinite + /// /// Maximum of entries to fetch. -1 = infinite /// - static JasonPP::JsonArray GetQueueAsJson(); + static JasonPP::JsonArray GetAlltimeCacheAsJson(time_t max_age, std::size_t max_num); /// /// Returns whether or not a tubio id exists @@ -94,6 +97,7 @@ namespace Downloader private: static void Save(); static void Load(); + static std::vector ParseJsonArrayToEntries(const JasonPP::JsonArray& arr); static void FetchInformation(std::string url, std::string tubId); static std::string CreateNewTubioID(); @@ -106,8 +110,10 @@ namespace Downloader static void DownloadNext(); static void UpdateDownloadProgressPercentages(); - static std::vector queue; + static std::vector unfinishedCache; static std::vector downloadThreads; + static JasonPP::JsonArray saveFileCache; // Content of the save-file + static std::vector saveFileCache_Atomic; // Content of the save-file static Logging::Logger* log; // This gets set by other threads static time_t lastProgressCheck; diff --git a/Tubio/HttpServer.cpp b/Tubio/HttpServer.cpp index 346e7d3..4bbe415 100644 --- a/Tubio/HttpServer.cpp +++ b/Tubio/HttpServer.cpp @@ -177,7 +177,7 @@ void HttpServer::ServeDownloadeableResource(mg_connection* pNc, int ev, void* p, else { Json j; - j.CloneFrom(RestResponseTemplates::GetByCode(BAD_REQUEST, "Invalid file id!")); + j.CloneFrom(RestResponseTemplates::GetByCode(BAD_REQUEST, "Invalid tubio id!")); ServeStringToConnection(pNc, j.Render(), BAD_REQUEST); } diff --git a/Tubio/RestQueryHandler.cpp b/Tubio/RestQueryHandler.cpp index 77ebea3..ab3b487 100644 --- a/Tubio/RestQueryHandler.cpp +++ b/Tubio/RestQueryHandler.cpp @@ -28,7 +28,8 @@ bool RestQueryHandler::ProcessQuery(const std::string clientAdress, const Json& if (requestName == "kill_yourself") return KillYourself(requestBody, responseBody, responseCode); else if (requestName == "queue_download") return QueueDownload(requestBody, responseBody, responseCode); - else if (requestName == "fetch_queue") return FetchQueue(requestBody, responseBody, responseCode); + else if (requestName == "fetch_session_cache") return FetchSessionCache(requestBody, responseBody, responseCode); + else if (requestName == "fetch_alltime_cache") return FetchAlltimeCache(requestBody, responseBody, responseCode); else if (requestName == "clear_download_cache") return ClearDownloadCache(requestBody, responseBody, responseCode); else if (requestName == "foo") return Example_Foo(requestBody, responseBody, responseCode); else if (requestName == "show_console") return ShowConsole(requestBody, responseBody, responseCode); @@ -91,18 +92,48 @@ bool RestQueryHandler::QueueDownload(const JsonBlock& request, JsonBlock& respon responseBody.Set("message") = "Download queued!"; responseBody.Set("queue_position") = (long long int)DownloadManager::GetQueueLength(); responseBody.Set("tubio_id") = tubId; - responseBody.Set("queue") = DownloadManager::GetQueueAsJson(); + JsonArray cache = DownloadManager::GetAlltimeCacheAsJson(time(0) - XGControl::boot_time, -1); + responseBody.Set("cache_size") = (long long int)cache.Size(); + responseBody.Set("cache") = cache; return true; } -bool RestQueryHandler::FetchQueue(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode) +bool RestQueryHandler::FetchSessionCache(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode) { - log->cout << "Asking for queue..."; + log->cout << "Asking for session cache..."; + log->Flush(); + + time_t max_age = time(0) - XGControl::boot_time; // Default max_age is session length + std::size_t max_num = (std::size_t )-1; // Default max_num is infinite + + JsonBlock dummy; + if (ValidateField("max_age", JDType::INT, request, dummy)) + { + max_age = request["max_age"].AsInt; + } + if (ValidateField("max_num", JDType::INT, request, dummy)) + { + max_num = request["max_num"].AsInt; + } + + responseCode = OK; + responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK)); + JsonArray cache = DownloadManager::GetAlltimeCacheAsJson(max_age, max_num); + responseBody.Set("cache_size") = (long long int)cache.Size(); + responseBody.Set("cache") = cache; + return true; +} + +bool RestQueryHandler::FetchAlltimeCache(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode) +{ + log->cout << "Asking for whole cache..."; log->Flush(); responseCode = OK; responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK)); - responseBody.Set("queue") = DownloadManager::GetQueueAsJson(); + JsonArray cache = DownloadManager::GetAlltimeCacheAsJson(-1, -1); // Get ALL the data + responseBody.Set("cache_size") = (long long int)cache.Size(); + responseBody.Set("cache") = cache; return true; } diff --git a/Tubio/RestQueryHandler.h b/Tubio/RestQueryHandler.h index a3af8d4..9153ad7 100644 --- a/Tubio/RestQueryHandler.h +++ b/Tubio/RestQueryHandler.h @@ -19,7 +19,8 @@ namespace Rest private: static bool Example_Foo(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode); - static bool FetchQueue(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode); + static bool FetchSessionCache(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode); + static bool FetchAlltimeCache(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode); static bool QueueDownload(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode); static bool ClearDownloadCache(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode); static bool KillYourself(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode); diff --git a/Tubio/XGControl.cpp b/Tubio/XGControl.cpp index 9ddc0d2..97fcd77 100644 --- a/Tubio/XGControl.cpp +++ b/Tubio/XGControl.cpp @@ -1,3 +1,4 @@ #include "XGControl.h" bool XGControl::keepServerRunning = false; +time_t XGControl::boot_time = time(0); diff --git a/Tubio/XGControl.h b/Tubio/XGControl.h index 2a29a42..3e4ba58 100644 --- a/Tubio/XGControl.h +++ b/Tubio/XGControl.h @@ -1,5 +1,6 @@ #pragma once #include +#include /// /// Class to house control variables @@ -8,5 +9,6 @@ class XGControl { public: static bool keepServerRunning; + static time_t boot_time; }; diff --git a/tubio-frontend-nuxt-app/components/Layout/Header.vue b/tubio-frontend-nuxt-app/components/Layout/Header.vue index 9f058d2..fc02480 100644 --- a/tubio-frontend-nuxt-app/components/Layout/Header.vue +++ b/tubio-frontend-nuxt-app/components/Layout/Header.vue @@ -35,7 +35,6 @@ export default { width: 50px; stroke: theme("colors.purple-3"); fill: theme("colors.purple-3"); - -webkit-tap-highlight-color: #0000; transition: stroke 0.2s, fill 0.2s, @@ -55,7 +54,6 @@ export default { width: 60px; fill: theme("colors.purple-3"); stroke: theme("colors.purple-3"); - -webkit-tap-highlight-color: #0000; transition: stroke 0.2s, fill 0.2s,