Fixed a bit of bullshittery and added max_age and max_num controls to fetch_session_cache

Tubio/DownloadManager.cpp

@@ -28,6 +28,7 @@ std::string DownloadManager::QueueDownload(std::string url, DOWNLOAD_MODE mode)
 	newDownload.tubio_id = tubioId;
 	newDownload.mode = mode;
 	newDownload.download_progress = 0;
+	newDownload.queued_timestamp = time(0);
 	newDownload.download_url = "/download/" + newDownload.tubio_id;
 
 	if (!IsJsonValid(jsString))
@@ -89,7 +90,7 @@ std::string DownloadManager::QueueDownload(std::string url, DOWNLOAD_MODE mode)
 		}
 	}
 
-	queue.push_back(newDownload);
+	unfinishedCache.push_back(newDownload);
 
 	return tubioId;
 }
@@ -130,11 +131,11 @@ void DownloadManager::DownloadNext()
 
 	DownloadEntry* next = nullptr;
 
-	for (std::size_t i = 0; i < queue.size(); i++)
+	for (std::size_t i = 0; i < unfinishedCache.size(); i++)
 	{
-		if (queue[i].status == DOWNLOAD_STATUS::QUEUED)
+		if (unfinishedCache[i].status == DOWNLOAD_STATUS::QUEUED)
 		{
-			next = &queue[i];
+			next = &unfinishedCache[i];
 			break;
 		}
 	}
@@ -200,11 +201,11 @@ void DownloadManager::DownloadNext()
 
 void DownloadManager::UpdateDownloadProgressPercentages()
 {
-	for (std::size_t i = 0; i < queue.size(); i++)
+	for (std::size_t i = 0; i < unfinishedCache.size(); i++)
 	{
-		if (queue[i].status == DOWNLOAD_STATUS::DOWNLOADING)
+		if (unfinishedCache[i].status == DOWNLOAD_STATUS::DOWNLOADING)
 		{
-			std::string filePath = XGConfig::downloader.cachedir + "/dlprogbuf/" + queue[i].tubio_id + ".buf";
+			std::string filePath = XGConfig::downloader.cachedir + "/dlprogbuf/" + unfinishedCache[i].tubio_id + ".buf";
 			if (FileSystem::Exists(filePath))
 			{
 				std::ifstream ifs;
@@ -227,9 +228,9 @@ void DownloadManager::UpdateDownloadProgressPercentages()
 								if (ss.str().length() > 0)
 								{
 									int newPercentage = std::stoi(ss.str());
-									queue[i].download_progress = newPercentage;
+									unfinishedCache[i].download_progress = newPercentage;
 
-									//if (newPercentage == 100) queue[i].status = DOWNLOAD_STATUS::FINISHED;
+									if (newPercentage == 100) unfinishedCache[i].status = DOWNLOAD_STATUS::FINISHED;
 								}
 							}
 						}
@@ -247,39 +248,79 @@ void DownloadManager::UpdateDownloadProgressPercentages()
 std::size_t DownloadManager::GetQueueLength()
 {
 	std::size_t counter = 0;
-	for (std::size_t i = 0; i < queue.size(); i++)
+	for (std::size_t i = 0; i < unfinishedCache.size(); i++)
 	{
-		if (queue[i].status == DOWNLOAD_STATUS::QUEUED) counter++;
+		if (unfinishedCache[i].status == DOWNLOAD_STATUS::QUEUED) counter++;
 	}
 	return counter;
 }
 
-JsonArray DownloadManager::GetQueueAsJson()
+JsonArray DownloadManager::GetAlltimeCacheAsJson(time_t max_age, std::size_t max_num)
 {
 	JsonArray arr;
-	for (std::size_t i = 0; i < queue.size(); i++)
+	for (std::size_t i = 0; i < unfinishedCache.size(); i++)
 	{
-		arr += queue[i].GetAsJson();
+		arr += unfinishedCache[i].GetAsJson();
 	}
 
-	return arr;
+	arr.Merge(saveFileCache);
+	arr.Sort("queued_timestamp", JSON_ARRAY_SORT_MODE::NUM_DESC);
+
+	// Both limits are inifnite. Just return arr as is
+	if ((max_age == -1) && (max_num == (std::size_t)-1)) return arr;
+
+	JsonArray cutArr;
+	// If max_num is -1 (would mean inifnite) it would underflow to size_t::max
+	for (std::size_t i = 0; ((i < arr.Size()) && (i < max_num)); i++)
+	{
+		// If max_age is > 0, we have to check against the max age
+		if (max_age > 0)
+		{
+			if (arr[i].AsJson.DoesExist("queued_timestamp"))
+			{
+				if (arr[i]["queued_timestamp"].GetDataType() == JDType::INT)
+				{
+					if ((time(0) - arr[i]["queued_timestamp"].AsInt) < max_age)
+					{
+						cutArr += arr[i];
+					}
+				}
+			}
+		}
+		else // If not, just insert it
+		{
+			cutArr += arr[i];
+		}
+	}
+
+	return cutArr;
 }
 
 bool DownloadManager::DoesTubioIDExist(std::string tubioId)
 {
-	for (std::size_t i = 0; i < queue.size(); i++)
+	for (std::size_t i = 0; i < unfinishedCache.size(); i++)
 	{
-		if (queue[i].tubio_id == tubioId) return true;
+		if (unfinishedCache[i].tubio_id == tubioId) return true;
 	}
+	for (std::size_t i = 0; i < saveFileCache_Atomic.size(); i++)
+	{
+		if (saveFileCache_Atomic[i].tubio_id == tubioId) return true;
+	}
+
 	return false;
 }
 
 DownloadEntry& DownloadManager::GetDownloadEntryByTubioID(std::string tubioId)
 {
-	for (std::size_t i = 0; i < queue.size(); i++)
+	for (std::size_t i = 0; i < unfinishedCache.size(); i++)
 	{
-		if (queue[i].tubio_id == tubioId) return queue[i];
+		if (unfinishedCache[i].tubio_id == tubioId) return unfinishedCache[i];
 	}
+	for (std::size_t i = 0; i < saveFileCache_Atomic.size(); i++)
+	{
+		if (saveFileCache_Atomic[i].tubio_id == tubioId) return saveFileCache_Atomic[i];
+	}
+
 	throw std::exception("TubioID not found!");
 	std::terminate();
 }
@@ -298,7 +339,9 @@ bool DownloadManager::ClearDownloadCache()
 			FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/metadata");
 			FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/download");
 			FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/dlprogbuf");
-			queue.clear();
+			unfinishedCache.clear();
+			saveFileCache.Clear();
+			saveFileCache_Atomic.clear();
 		}
 
 		return true;
@@ -331,21 +374,23 @@ void DownloadManager::Save()
 		log->Flush();
 	}
 
-	JsonArray arr;
+	// Filecache has contain all and only nonsaved entries.
-	for (std::size_t i = 0; i < queue.size(); i++)
+	for (long long int i = unfinishedCache.size() - 1; i >= 0; i--)
 	{
-		if (queue[i].status == DOWNLOAD_STATUS::FINISHED)
+		if (unfinishedCache[i].status == DOWNLOAD_STATUS::FINISHED)
 		{
-			arr += queue[i].GetAsJson();
+			saveFileCache += unfinishedCache[i].GetAsJson();
+			unfinishedCache.erase(unfinishedCache.begin() + i);
 		}
 	}
 
-	Json j(arr);
+	Json j(saveFileCache);
 	if (!FileSystem::WriteFile(XGConfig::downloader.cachedir + "/index.json", j.Render()))
 	{
 		log->cout << log->Err() << "Unable to save download cache index file!";
 		log->Flush();
 	}
+	saveFileCache_Atomic = ParseJsonArrayToEntries(saveFileCache);
 
 	shouldSave = false;
 
@@ -375,70 +420,8 @@ void DownloadManager::Load()
 		
 		if (j.GetDataType() == JDType::ARRAY)
 		{
-			const JsonArray& cachedArr = j.AsArray;
+			saveFileCache.CloneFrom(j.AsArray);
-
+			saveFileCache_Atomic = ParseJsonArrayToEntries(saveFileCache);
-			for (std::size_t i = 0; i < cachedArr.Size(); i++)
-			{
-				JsonBlock iter = cachedArr[i].AsJson;
-				DownloadEntry newEntry;
-				newEntry.download_progress = 100;
-				newEntry.status = DOWNLOAD_STATUS::FINISHED; // All saved entries are finished...
-
-				if ((iter.DoesExist("title")) && (iter["title"].GetDataType() == JDType::STRING))
-				{
-					newEntry.title = iter["title"];
-				}
-
-				if ((iter.DoesExist("description")) && (iter["description"].GetDataType() == JDType::STRING))
-				{
-					newEntry.description = iter["description"];
-				}
-
-				if ((iter.DoesExist("uploader")) && (iter["uploader"].GetDataType() == JDType::STRING))
-				{
-					newEntry.uploader = iter["uploader"];
-				}
-
-				if ((iter.DoesExist("duration")) && (iter["duration"].GetDataType() == JDType::INT))
-				{
-					newEntry.duration = iter["duration"];
-				}
-
-				if ((iter.DoesExist("tubio_id")) && (iter["tubio_id"].GetDataType() == JDType::STRING))
-				{
-					newEntry.tubio_id = iter["tubio_id"];
-				}
-
-				if ((iter.DoesExist("webpage_url")) && (iter["webpage_url"].GetDataType() == JDType::STRING))
-				{
-					newEntry.webpage_url = iter["webpage_url"];
-				}
-
-				if ((iter.DoesExist("thumbnail_url")) && (iter["thumbnail_url"].GetDataType() == JDType::STRING))
-				{
-					newEntry.thumbnail_url = iter["thumbnail_url"];
-				}
-
-				if ((iter.DoesExist("download_url")) && (iter["download_url"].GetDataType() == JDType::STRING))
-				{
-					newEntry.download_url = iter["download_url"];
-				}
-
-				if ((iter.DoesExist("downloaded_filename")) && (iter["downloaded_filename"].GetDataType() == JDType::STRING))
-				{
-					newEntry.downloaded_filename = iter["downloaded_filename"];
-				}
-
-				if ((iter.DoesExist("mode")) && (iter["mode"].GetDataType() == JDType::STRING))
-				{
-					std::string cachedStrMode = iter["mode"];
-					if (cachedStrMode == "video") newEntry.mode = DOWNLOAD_MODE::VIDEO;
-					else if (cachedStrMode == "audio") newEntry.mode = DOWNLOAD_MODE::AUDIO;
-					else newEntry.mode = DOWNLOAD_MODE::VIDEO;
-				}
-
-				queue.push_back(newEntry);
-			}
 		}
 		else
 		{
@@ -455,6 +438,81 @@ void DownloadManager::Load()
 	return;
 }
 
+std::vector<DownloadEntry> DownloadManager::ParseJsonArrayToEntries(const JasonPP::JsonArray& arr)
+{
+	std::vector<DownloadEntry> entries;
+
+	for (std::size_t i = 0; i < arr.Size(); i++)
+	{
+		JsonBlock iter = arr[i].AsJson;
+		DownloadEntry newEntry;
+		newEntry.download_progress = 100;
+		newEntry.status = DOWNLOAD_STATUS::FINISHED; // All saved entries are finished...
+
+		if ((iter.DoesExist("title")) && (iter["title"].GetDataType() == JDType::STRING))
+		{
+			newEntry.title = iter["title"];
+		}
+
+		if ((iter.DoesExist("description")) && (iter["description"].GetDataType() == JDType::STRING))
+		{
+			newEntry.description = iter["description"];
+		}
+
+		if ((iter.DoesExist("uploader")) && (iter["uploader"].GetDataType() == JDType::STRING))
+		{
+			newEntry.uploader = iter["uploader"];
+		}
+
+		if ((iter.DoesExist("duration")) && (iter["duration"].GetDataType() == JDType::INT))
+		{
+			newEntry.duration = iter["duration"];
+		}
+
+		if ((iter.DoesExist("tubio_id")) && (iter["tubio_id"].GetDataType() == JDType::STRING))
+		{
+			newEntry.tubio_id = iter["tubio_id"];
+		}
+
+		if ((iter.DoesExist("webpage_url")) && (iter["webpage_url"].GetDataType() == JDType::STRING))
+		{
+			newEntry.webpage_url = iter["webpage_url"];
+		}
+
+		if ((iter.DoesExist("thumbnail_url")) && (iter["thumbnail_url"].GetDataType() == JDType::STRING))
+		{
+			newEntry.thumbnail_url = iter["thumbnail_url"];
+		}
+
+		if ((iter.DoesExist("download_url")) && (iter["download_url"].GetDataType() == JDType::STRING))
+		{
+			newEntry.download_url = iter["download_url"];
+		}
+
+		if ((iter.DoesExist("downloaded_filename")) && (iter["downloaded_filename"].GetDataType() == JDType::STRING))
+		{
+			newEntry.downloaded_filename = iter["downloaded_filename"];
+		}
+
+		if ((iter.DoesExist("queued_timestamp")) && (iter["queued_timestamp"].GetDataType() == JDType::INT))
+		{
+			newEntry.queued_timestamp = iter["queued_timestamp"];
+		}
+
+		if ((iter.DoesExist("mode")) && (iter["mode"].GetDataType() == JDType::STRING))
+		{
+			std::string cachedStrMode = iter["mode"];
+			if (cachedStrMode == "video") newEntry.mode = DOWNLOAD_MODE::VIDEO;
+			else if (cachedStrMode == "audio") newEntry.mode = DOWNLOAD_MODE::AUDIO;
+			else newEntry.mode = DOWNLOAD_MODE::VIDEO;
+		}
+
+		entries.push_back(newEntry);
+	}
+
+	return entries;
+}
+
 void DownloadManager::FetchInformation(std::string url, std::string tubId)
 {
 	std::stringstream ss;
@@ -476,10 +534,11 @@ std::string DownloadManager::CreateNewTubioID()
 		newId = Internal::Helpers::Base10_2_X(time(0), "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ");
 
 		isIdUnique = true;
-		for (std::size_t i = 0; i < queue.size(); i++)
+		if (unfinishedCache.size() > 0)
 		{
-			if (queue[i].tubio_id == newId) isIdUnique = false;
+			isIdUnique = unfinishedCache[unfinishedCache.size() - 1].tubio_id != newId;
 		}
+
 		counter++;
 	}
 
@@ -489,9 +548,9 @@ std::string DownloadManager::CreateNewTubioID()
 std::size_t DownloadManager::GetNumActiveDownloads()
 {
 	std::size_t counter = 0;
-	for (std::size_t i = 0; i < queue.size(); i++)
+	for (std::size_t i = 0; i < unfinishedCache.size(); i++)
 	{
-		if (queue[i].status == DOWNLOAD_STATUS::DOWNLOADING) counter++;
+		if (unfinishedCache[i].status == DOWNLOAD_STATUS::DOWNLOADING) counter++;
 	}
 	return counter;
 }
@@ -519,8 +578,10 @@ void DownloadManager::PostExit()
 }
 
 
-std::vector<DownloadEntry> DownloadManager::queue;
+std::vector<DownloadEntry> DownloadManager::unfinishedCache;
 std::vector<std::thread*> DownloadManager::downloadThreads;
+JasonPP::JsonArray DownloadManager::saveFileCache;
+std::vector<DownloadEntry> DownloadManager::saveFileCache_Atomic;
 ::Logging::Logger* DownloadManager::log;
 time_t DownloadManager::lastProgressCheck = 0;
 bool DownloadManager::shouldSave = false;
@@ -541,6 +602,7 @@ JsonBlock DownloadEntry::GetAsJson()
 	jb.Set(Ele("download_progress", download_progress));
 	jb.Set(Ele("downloaded_filename", downloaded_filename));
 	jb.Set(Ele("download_url", download_url));
+	jb.Set(Ele("queued_timestamp", (long long int)queued_timestamp));
 
 	switch (mode)
 	{

Tubio/DownloadManager.h

@@ -39,6 +39,7 @@ namespace Downloader
 		DOWNLOAD_STATUS status;
 		DOWNLOAD_MODE mode;
 		int download_progress;
+		time_t queued_timestamp;
 
 		JasonPP::JsonBlock GetAsJson();
 
@@ -67,10 +68,12 @@ namespace Downloader
 		static std::size_t GetQueueLength();
 
 		/// <summary>
-		/// Will return the whole queue in json format
+		/// Will return the whole cache in json format
 		/// </summary>
+		/// /// <param name="max_age">Maximum age of the entry in seconds. -1 = infinite</param>
+		/// /// <param name="max_num">Maximum of entries to fetch. -1 = infinite</param>
 		/// <returns></returns>
-		static JasonPP::JsonArray GetQueueAsJson();
+		static JasonPP::JsonArray GetAlltimeCacheAsJson(time_t max_age, std::size_t max_num);
 
 		/// <summary>
 		/// Returns whether or not a tubio id exists
@@ -94,6 +97,7 @@ namespace Downloader
 	private:
 		static void Save();
 		static void Load();
+		static std::vector<DownloadEntry> ParseJsonArrayToEntries(const JasonPP::JsonArray& arr);
 
 		static void FetchInformation(std::string url, std::string tubId);
 		static std::string CreateNewTubioID();
@@ -106,8 +110,10 @@ namespace Downloader
 		static void DownloadNext();
 		static void UpdateDownloadProgressPercentages();
 
-		static std::vector<DownloadEntry> queue;
+		static std::vector<DownloadEntry> unfinishedCache;
 		static std::vector<std::thread*> downloadThreads;
+		static JasonPP::JsonArray saveFileCache; // Content of the save-file
+		static std::vector<DownloadEntry> saveFileCache_Atomic; // Content of the save-file
 		static Logging::Logger* log;
 		// This gets set by other threads
 		static time_t lastProgressCheck;

Tubio/HttpServer.cpp

@@ -177,7 +177,7 @@ void HttpServer::ServeDownloadeableResource(mg_connection* pNc, int ev, void* p,
 	else
 	{
 		Json j;
-		j.CloneFrom(RestResponseTemplates::GetByCode(BAD_REQUEST, "Invalid file id!"));
+		j.CloneFrom(RestResponseTemplates::GetByCode(BAD_REQUEST, "Invalid tubio id!"));
 		ServeStringToConnection(pNc, j.Render(), BAD_REQUEST);
 	}
 

Tubio/RestQueryHandler.cpp

@@ -28,7 +28,8 @@ bool RestQueryHandler::ProcessQuery(const std::string clientAdress, const Json&
 
 	if (requestName == "kill_yourself") return KillYourself(requestBody, responseBody, responseCode);
 	else if (requestName == "queue_download") return QueueDownload(requestBody, responseBody, responseCode);
-	else if (requestName == "fetch_queue") return FetchQueue(requestBody, responseBody, responseCode);
+	else if (requestName == "fetch_session_cache") return FetchSessionCache(requestBody, responseBody, responseCode);
+	else if (requestName == "fetch_alltime_cache") return FetchAlltimeCache(requestBody, responseBody, responseCode);
 	else if (requestName == "clear_download_cache") return ClearDownloadCache(requestBody, responseBody, responseCode);
 	else if (requestName == "foo") return Example_Foo(requestBody, responseBody, responseCode);
 	else if (requestName == "show_console") return ShowConsole(requestBody, responseBody, responseCode);
@@ -91,18 +92,48 @@ bool RestQueryHandler::QueueDownload(const JsonBlock& request, JsonBlock& respon
 	responseBody.Set("message") = "Download queued!";
 	responseBody.Set("queue_position") = (long long int)DownloadManager::GetQueueLength();
 	responseBody.Set("tubio_id") = tubId;
-	responseBody.Set("queue") = DownloadManager::GetQueueAsJson();
+	JsonArray cache = DownloadManager::GetAlltimeCacheAsJson(time(0) - XGControl::boot_time, -1);
+	responseBody.Set("cache_size") = (long long int)cache.Size();
+	responseBody.Set("cache") = cache;
 	return true;
 }
 
-bool RestQueryHandler::FetchQueue(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode)
+bool RestQueryHandler::FetchSessionCache(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode)
 {
-	log->cout << "Asking for queue...";
+	log->cout << "Asking for session cache...";
+	log->Flush();
+
+	time_t max_age = time(0) - XGControl::boot_time; // Default max_age is session length
+	std::size_t max_num = (std::size_t )-1;			 // Default max_num is infinite
+
+	JsonBlock dummy;
+	if (ValidateField("max_age", JDType::INT, request, dummy))
+	{
+		max_age = request["max_age"].AsInt;
+	}
+	if (ValidateField("max_num", JDType::INT, request, dummy))
+	{
+		max_num = request["max_num"].AsInt;
+	}
+
+	responseCode = OK;
+	responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK));
+	JsonArray cache = DownloadManager::GetAlltimeCacheAsJson(max_age, max_num);
+	responseBody.Set("cache_size") = (long long int)cache.Size();
+	responseBody.Set("cache") = cache;
+	return true;
+}
+
+bool RestQueryHandler::FetchAlltimeCache(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode)
+{
+	log->cout << "Asking for whole cache...";
 	log->Flush();
 
 	responseCode = OK;
 	responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK));
-	responseBody.Set("queue") = DownloadManager::GetQueueAsJson();
+	JsonArray cache = DownloadManager::GetAlltimeCacheAsJson(-1, -1); // Get ALL the data
+	responseBody.Set("cache_size") = (long long int)cache.Size();
+	responseBody.Set("cache") = cache;
 	return true;
 }
 

Tubio/RestQueryHandler.h

@@ -19,7 +19,8 @@ namespace Rest
 
 	private:
 		static bool Example_Foo(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode);
-		static bool FetchQueue(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode);
+		static bool FetchSessionCache(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode);
+		static bool FetchAlltimeCache(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode);
 		static bool QueueDownload(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode);
 		static bool ClearDownloadCache(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode);
 		static bool KillYourself(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode);

Tubio/XGControl.cpp

@@ -1,3 +1,4 @@
 #include "XGControl.h"
 
 bool XGControl::keepServerRunning = false;
+time_t XGControl::boot_time = time(0);

Tubio/XGControl.h

@@ -1,5 +1,6 @@
 #pragma once
 #include <string>
+#include <ctime>
 
 /// <summary>
 /// Class to house control variables
@@ -8,5 +9,6 @@ class XGControl
 {
 public:
 	static bool keepServerRunning;
+	static time_t boot_time;
 };
 

tubio-frontend-nuxt-app/components/Layout/Header.vue

@@ -35,7 +35,6 @@ export default {
   width: 50px;
   stroke: theme("colors.purple-3");
   fill: theme("colors.purple-3");
-  -webkit-tap-highlight-color: #0000;
   transition:
     stroke 0.2s,
     fill 0.2s,
@@ -55,7 +54,6 @@ export default {
   width: 60px;
   fill: theme("colors.purple-3");
   stroke: theme("colors.purple-3");
-  -webkit-tap-highlight-color: #0000;
   transition:
     stroke 0.2s,
     fill 0.2s,