diff --git a/.gitignore b/.gitignore index c3dcaf9..e68f6ae 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,12 @@ config.json log.txt log.json +dlcache/ + +# Rediculously large dependencies +ffmpeg.exe +ffplay.exe +ffprobe.exe # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs diff --git a/README.md b/README.md index f130508..17e67e9 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,10 @@ -Please read all the LICENSE-files! - -This project is a semi-automatic youtube to mp3 converter. -It does only work, if you have the youtube-dl.exe and ffmpeg.exe, ffplay.exe and ffprobe.exe binaries in the root directory of the main executeable. -These are not mine, but they are licensed in a way that i can include them in the project, so no source code for them. - -You can leave this application running in the background. As soon as you copy any youtube link, it will automatically download it to mp3 format. -Mp3 files will be saved in a "Downloads" folder in the root directory of the main executeable. - -If you want to download an entire playlist at once, you just have to type PLAYLIST somewhere into the url of a playlist. -(This way you can quickly download single videos out of a playlist by just copying the url) - - - -This project is NOT entirely mine! My project is based on these four binaries that are NOT mine: -youtube-dl.exe binary is from https://rg3.github.io/youtube-dl/ and is released under public domain. -ffmpeg.exe, ffplay.exe and ffprobe.exe binaries are copyrighted by Fabrice Bellard and released under the gnu - general public license - - -The C++ project (without the before mentioned four binaries) are licensed under gnu - general public license. -You may redistribute it (under the same license), -you may display it, -you may copy it, -you may modify it, -as long as you do not sub-license it or say it is your own. - +# Setup: +## youtube-dl.exe +Download youtube-dl and Microsoft Visual C++ 2010 Redist x86 from here: +[http://ytdl-org.github.io/youtube-dl/download.html](http://ytdl-org.github.io/youtube-dl/download.html) +and put youtube-dl.exe into your build directory. It has to be in Tubio's working directory!!! +Youtube-dl should be already there, but it is always good to update it. + +## ffmpeg +Download the windows builds from ffmpeg from [https://ffmpeg.org/download.html#build-windows](https://ffmpeg.org/download.html#build-windows) +and put the executeables (ffmpeg.exe, ffplay.exe and ffprobe.exe) into your build directory. They have to be in Tubio's working directory!!! diff --git a/Tubio/DownloadManager.cpp b/Tubio/DownloadManager.cpp new file mode 100644 index 0000000..bebf3bf --- /dev/null +++ b/Tubio/DownloadManager.cpp @@ -0,0 +1,476 @@ +#include "DownloadManager.h" + +using namespace Downloader; +using namespace JasonPP; + +void DownloadManager::PreInit() +{ + log = new Logging::Logger("DownloadManager"); + + FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir); + FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/metadata"); + FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/download"); + FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/dlprogbuf"); + + Load(); + + return; +} + +std::string DownloadManager::QueueDownload(std::string url, DOWNLOAD_MODE mode) +{ + std::string tubioId = CreateNewTubioID(); + FetchInformation(url, tubioId); + + std::string jsString = FileSystem::ReadFile(XGConfig::downloader.cachedir + "/metadata/" + tubioId + ".json"); + + DownloadEntry newDownload; + newDownload.tubio_id = tubioId; + newDownload.status = DOWNLOAD_STATUS::QUEUED; + newDownload.mode = mode; + newDownload.download_progress = 0; + + Json j; + j.Parse(jsString); + if (j.GetDataType() != JDType::JSON) + { + newDownload.status = DOWNLOAD_STATUS::FAILED; + } + else + { + if ((j.AsJson.DoesExist("title")) && (j.AsJson["title"].GetDataType() == JDType::STRING)) + { + newDownload.title = j["title"]; + } + + if ((j.AsJson.DoesExist("description")) && (j.AsJson["description"].GetDataType() == JDType::STRING)) + { + newDownload.description = j["description"]; + } + + if ((j.AsJson.DoesExist("uploader")) && (j.AsJson["uploader"].GetDataType() == JDType::STRING)) + { + newDownload.uploader = j["uploader"]; + } + + if ((j.AsJson.DoesExist("duration")) && (j.AsJson["duration"].GetDataType() == JDType::INT)) + { + newDownload.duration = j["duration"]; + } + + if ((j.AsJson.DoesExist("webpage_url")) && (j.AsJson["webpage_url"].GetDataType() == JDType::STRING)) + { + newDownload.webpage_url = j["webpage_url"]; + } + + if ((j.AsJson.DoesExist("thumbnails")) && (j.AsJson["thumbnails"].GetDataType() == JDType::ARRAY)) + { + JsonArray& thumbnails = j.AsJson["thumbnails"].AsArray; + if (thumbnails.Size() > 0) + { + if (thumbnails.Size() > 1) + { + // If we have more than one thumbnail to choose from, choose the second-highes quality. + newDownload.thumbnail_url = thumbnails[thumbnails.Size() - 2]["url"]; + } + else + { + newDownload.thumbnail_url = thumbnails[thumbnails.Size() - 1]["url"]; + } + } + } + } + + queue.push_back(newDownload); + + Save(); + + return tubioId; +} + +void DownloadManager::Update() +{ + //if (shouldSave) Save(); + + std::size_t numActiveDownloads = GetNumActiveDownloads(); + + if (numActiveDownloads < XGConfig::downloader.num_threads) + { + DownloadNext(); + } + + // Check every second, non-blocking + if ((numActiveDownloads > 0) && (time(0) - lastProgressCheck > 2)) + { + UpdateDownloadProgressPercentages(); + } + + return; +} + +void DownloadManager::DownloadNext() +{ + if (GetQueueLength() == 0) return; + + DownloadEntry* next = nullptr; + + for (std::size_t i = 0; i < queue.size(); i++) + { + if (queue[i].status == DOWNLOAD_STATUS::QUEUED) + { + next = &queue[i]; + break; + } + } + next->status = DOWNLOAD_STATUS::DOWNLOADING; + + std::thread* downloadThread = new std::thread([=]() { + DownloadEntry* entry = next; + + std::stringstream ss; + if (entry->mode == DOWNLOAD_MODE::VIDEO) + { + ss << "youtube-dl --newline --no-call-home --no-playlist --limit-rate " << XGConfig::downloader.max_dlrate_per_thread + << " --no-mtime --no-cache-dir --format \"bestvideo[ext=mp4]+bestaudio\" --merge-output-format mp4" + << " -o \"" << XGConfig::downloader.cachedir << "/download/" << entry->tubio_id + << ".mp4\" " << entry->webpage_url << " > \"" << XGConfig::downloader.cachedir + << "/dlprogbuf/" << entry->tubio_id << ".buf" << "\""; + } + else // DOWNLOAD_MODE::AUDIO + { + ss << "youtube-dl --newline --no-call-home --no-playlist --limit-rate " << XGConfig::downloader.max_dlrate_per_thread + << " --no-mtime --no-cache-dir --audio-format mp3 --audio-quality 0 --extract-audio -o \"" + << XGConfig::downloader.cachedir << "/download/" << entry->tubio_id << ".%(ext)s\" " + << entry->webpage_url << " > \"" << XGConfig::downloader.cachedir + << "/dlprogbuf/" << entry->tubio_id << ".buf" << "\""; + } + + system(ss.str().c_str()); + + entry->status = DOWNLOAD_STATUS::FINISHED; + entry->download_progress = 100; + shouldSave = true; + return; + }); + downloadThreads.push_back(downloadThread); + + return; +} + +void DownloadManager::UpdateDownloadProgressPercentages() +{ + for (std::size_t i = 0; i < queue.size(); i++) + { + if (queue[i].status == DOWNLOAD_STATUS::DOWNLOADING) + { + std::string filePath = XGConfig::downloader.cachedir + "/dlprogbuf/" + queue[i].tubio_id + ".buf"; + if (FileSystem::Exists(filePath)) + { + std::ifstream ifs; + ifs.open(filePath, std::ios::in); + if (ifs.good()) + { + std::string lbuf; + while (std::getline(ifs, lbuf)) + { + if (lbuf.length() > 14) + { + if (lbuf.substr(0, 10) == "[download]") + { + std::string dirtyDigits = lbuf.substr(11, 3); + std::stringstream ss; + for (std::size_t j = 0; j < dirtyDigits.length(); j++) + { + if ((dirtyDigits[j] >= '0') && (dirtyDigits[j] <= '9')) ss << dirtyDigits[j]; + } + if (ss.str().length() > 0) + { + int newPercentage = std::stoi(ss.str()); + queue[i].download_progress = newPercentage; + } + } + } + } + } + ifs.close(); + } + } + } + + lastProgressCheck = time(0); + return; +} + +std::size_t DownloadManager::GetQueueLength() +{ + std::size_t counter = 0; + for (std::size_t i = 0; i < queue.size(); i++) + { + if (queue[i].status == DOWNLOAD_STATUS::QUEUED) counter++; + } + return counter; +} + +JsonArray DownloadManager::GetQueueAsJson() +{ + JsonArray arr; + for (std::size_t i = 0; i < queue.size(); i++) + { + arr += queue[i].GetAsJson(); + } + + return arr; +} + +void Downloader::DownloadManager::ClearDownloadCache() +{ + if (FileSystem::ExistsDirectory(XGConfig::downloader.cachedir)) + { + FileSystem::DeleteDirectory(XGConfig::downloader.cachedir); + FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/metadata"); + FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/download"); + FileSystem::CreateDirectoryIfNotExists(XGConfig::downloader.cachedir + "/dlprogbuf"); + queue.clear(); + } + + return; +} + +void DownloadManager::Save() +{ + JsonArray arr; + for (std::size_t i = 0; i < queue.size(); i++) + { + if (queue[i].status == DOWNLOAD_STATUS::FINISHED) + { + arr += queue[i].GetAsJson(); + } + } + + Json j(arr); + if (!FileSystem::WriteFile(XGConfig::downloader.cachedir + "/index.json", j.Render())) + { + log->cout << log->Err() << "Unable to save download cache index file!"; + log->Flush(); + } + + shouldSave = false; + + return; +} + +void DownloadManager::Load() +{ + // No file = nothing to load + if (!FileSystem::Exists(XGConfig::downloader.cachedir + "/index.json")) + { + log->cout << "Did not load download cache, because \"" << XGConfig::downloader.cachedir << "/index.json" + << "\" was not found."; + log->Flush(); + return; + } + + std::string fileContent = FileSystem::ReadFile(XGConfig::downloader.cachedir + "/index.json"); + + if (IsJsonValid(fileContent)) + { + Json j; + j.Parse(fileContent); + + if (j.GetDataType() == JDType::ARRAY) + { + const JsonArray& cachedArr = j.AsArray; + + for (std::size_t i = 0; i < cachedArr.Size(); i++) + { + JsonBlock iter = cachedArr[i].AsJson; + DownloadEntry newEntry; + newEntry.download_progress = -1; + newEntry.status = DOWNLOAD_STATUS::FINISHED; // All saved entries are finished... + + if ((iter.DoesExist("title")) && (iter["title"].GetDataType() == JDType::STRING)) + { + newEntry.title = iter["title"]; + } + + if ((iter.DoesExist("description")) && (iter["description"].GetDataType() == JDType::STRING)) + { + newEntry.description = iter["description"]; + } + + if ((iter.DoesExist("uploader")) && (iter["uploader"].GetDataType() == JDType::STRING)) + { + newEntry.uploader = iter["uploader"]; + } + + if ((iter.DoesExist("duration")) && (iter["duration"].GetDataType() == JDType::INT)) + { + newEntry.duration = iter["duration"]; + } + + if ((iter.DoesExist("tubio_id")) && (iter["tubio_id"].GetDataType() == JDType::STRING)) + { + newEntry.tubio_id = iter["tubio_id"]; + } + + if ((iter.DoesExist("webpage_url")) && (iter["webpage_url"].GetDataType() == JDType::STRING)) + { + newEntry.webpage_url = iter["webpage_url"]; + } + + if ((iter.DoesExist("thumbnail_url")) && (iter["thumbnail_url"].GetDataType() == JDType::STRING)) + { + newEntry.thumbnail_url = iter["thumbnail_url"]; + } + + if ((iter.DoesExist("mode")) && (iter["mode"].GetDataType() == JDType::STRING)) + { + std::string cachedStrMode = iter["mode"]; + if (cachedStrMode == "video") newEntry.mode = DOWNLOAD_MODE::VIDEO; + else if (cachedStrMode == "audio") newEntry.mode = DOWNLOAD_MODE::AUDIO; + else newEntry.mode = DOWNLOAD_MODE::VIDEO; + } + + queue.push_back(newEntry); + } + } + else + { + log->cout << log->Err() << "Unable to parse download cache index file! Not json-type array!"; + log->Flush(); + } + } + else + { + log->cout << log->Err() << "Unable to parse download cache index file! Invalid json!"; + log->Flush(); + } + + return; +} + +void DownloadManager::FetchInformation(std::string url, std::string tubId) +{ + std::stringstream ss; + ss << "youtube-dl.exe --skip-download --dump-json " << url << " > \"" << XGConfig::downloader.cachedir << "/metadata/" << tubId << ".json" << "\"" << std::endl; + system(ss.str().c_str()); + return; +} + +std::string DownloadManager::CreateNewTubioID() +{ + bool isIdUnique = false; + std::size_t counter = 0; + std::string newId; + + while (!isIdUnique) + { + if (counter > 100000000) throw std::exception("Tubio download id generator timeout"); + + newId = Internal::Helpers::Base10_2_X(time(0), "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"); + + isIdUnique = true; + for (std::size_t i = 0; i < queue.size(); i++) + { + if (queue[i].tubio_id == newId) isIdUnique = false; + } + counter++; + } + + return newId; +} + +std::size_t Downloader::DownloadManager::GetNumActiveDownloads() +{ + std::size_t counter = 0; + for (std::size_t i = 0; i < queue.size(); i++) + { + if (queue[i].status == DOWNLOAD_STATUS::DOWNLOADING) counter++; + } + return counter; +} + +void DownloadManager::OnExit() +{ + if (downloadThreads.size() > 0) + { + log->cout << "Waiting for active download threads to finish..."; + log->Flush(); + + for (std::size_t i = 0; i < downloadThreads.size(); i++) + { + downloadThreads[i]->join(); + delete downloadThreads[i]; + downloadThreads[i] = nullptr; + } + } + + // Clear dlprogbuf directory. + if (FileSystem::ExistsDirectory(XGConfig::downloader.cachedir + "/dlprogbuf")) + { + FileSystem::DeleteDirectory(XGConfig::downloader.cachedir + "/dlprogbuf"); + } + + + Save(); + return; +} + +void DownloadManager::PostExit() +{ + delete log; + + log = nullptr; + + return; +} + + +std::vector DownloadManager::queue; +std::vector DownloadManager::downloadThreads; +::Logging::Logger* DownloadManager::log; +bool DownloadManager::shouldSave = false; +time_t DownloadManager::lastProgressCheck = 0; + + + +JsonBlock DownloadEntry::GetAsJson() +{ + JsonBlock jb; + jb.Set(Ele("title",title)); + jb.Set(Ele("description", description)); + jb.Set(Ele("uploader", uploader)); + jb.Set(Ele("duration", duration)); + jb.Set(Ele("tubio_id", tubio_id)); + jb.Set(Ele("webpage_url", webpage_url)); + jb.Set(Ele("thumbnail_url", thumbnail_url)); + jb.Set(Ele("download_progress", download_progress)); + + switch (mode) + { + case DOWNLOAD_MODE::VIDEO: + jb.Set(Ele("mode", "video")); + break; + case DOWNLOAD_MODE::AUDIO: + jb.Set(Ele("mode", "audio")); + break; + } + + switch (status) + { + case DOWNLOAD_STATUS::QUEUED: + jb.Set(Ele("status", "queued")); + break; + case DOWNLOAD_STATUS::DOWNLOADING: + jb.Set(Ele("status", "downloading")); + break; + case DOWNLOAD_STATUS::FINISHED: + jb.Set(Ele("status", "finished")); + break; + case DOWNLOAD_STATUS::FAILED: + jb.Set(Ele("status", "failed")); + break; + } + + return jb; +} diff --git a/Tubio/DownloadManager.h b/Tubio/DownloadManager.h new file mode 100644 index 0000000..e093938 --- /dev/null +++ b/Tubio/DownloadManager.h @@ -0,0 +1,95 @@ +#pragma once +#include +#include +#include +#include +#include +#include "FileSystem.h" +#include "XGConfig.h" +#include "Logger.h" + +namespace Downloader +{ + enum class DOWNLOAD_MODE + { + VIDEO, + AUDIO + }; + enum class DOWNLOAD_STATUS + { + QUEUED, + DOWNLOADING, + FINISHED, + FAILED + }; + + class DownloadEntry + { + public: + std::string title; + std::string description; + std::string uploader; + int duration; + std::string tubio_id; + std::string webpage_url; + std::string thumbnail_url; + DOWNLOAD_STATUS status; + DOWNLOAD_MODE mode; + int download_progress; + + JasonPP::JsonBlock GetAsJson(); + + }; + + class DownloadManager + { + public: + static void PreInit(); + static void Update(); + static void OnExit(); + static void PostExit(); + + /// + /// Queues a video for download. Returns its tubio download id + /// + /// + /// If video or audio + /// Tubio download id + static std::string QueueDownload(std::string url, DOWNLOAD_MODE mode); + + /// + /// Returns the number of videos queued + /// + /// + static std::size_t GetQueueLength(); + + static JasonPP::JsonArray GetQueueAsJson(); + + /// + /// Will delete all cached downloads! + /// + static void ClearDownloadCache(); + + private: + static void Save(); + static void Load(); + + static void FetchInformation(std::string url, std::string tubId); + static std::string CreateNewTubioID(); + + static std::size_t GetNumActiveDownloads(); + + /// + /// Will start a download-thread for the next queue-entry with status "queued" + /// + static void DownloadNext(); + static void UpdateDownloadProgressPercentages(); + + static std::vector queue; + static std::vector downloadThreads; + static Logging::Logger* log; + // This gets set by other threads + static bool shouldSave; + static time_t lastProgressCheck; + }; +} diff --git a/Tubio/FileSystem.cpp b/Tubio/FileSystem.cpp index df12e11..39149a9 100644 --- a/Tubio/FileSystem.cpp +++ b/Tubio/FileSystem.cpp @@ -37,6 +37,27 @@ bool FileSystem::Exists(std::string filename) return true; } +bool FileSystem::ExistsDirectory(std::string name) +{ + return (!Exists(name) && (std::filesystem::exists(name))); +} + +bool FileSystem::CreateDirectory(std::string name) +{ + return std::filesystem::create_directories(name); +} + +bool FileSystem::CreateDirectoryIfNotExists(std::string name) +{ + if (!ExistsDirectory(name)) return CreateDirectory(name); + return false; +} + +bool FileSystem::DeleteDirectory(std::string name) +{ + return std::filesystem::remove_all(name); +} + bool FileSystem::Copy(std::string from, std::string to) { std::ifstream ifs; diff --git a/Tubio/FileSystem.h b/Tubio/FileSystem.h index 9ef2e48..b34365f 100644 --- a/Tubio/FileSystem.h +++ b/Tubio/FileSystem.h @@ -3,6 +3,7 @@ #include #include #include +#include class FileSystem @@ -13,7 +14,10 @@ public: static bool Exists(std::string filename); static bool Copy(std::string from, std::string to); static bool Delete(std::string filename); - + static bool ExistsDirectory(std::string name); + static bool CreateDirectory(std::string name); + static bool CreateDirectoryIfNotExists(std::string name); + static bool DeleteDirectory(std::string name); private: }; diff --git a/Tubio/Framework.cpp b/Tubio/Framework.cpp index 0cb742f..c26cd00 100644 --- a/Tubio/Framework.cpp +++ b/Tubio/Framework.cpp @@ -2,6 +2,7 @@ using namespace Logging; using namespace Rest; +using namespace Downloader; Framework::Framework() { @@ -16,7 +17,6 @@ Framework::Framework() PostInit(); XGControl::keepServerRunning = true; - Run(); return; } @@ -37,6 +37,7 @@ void Framework::Run() while (XGControl::keepServerRunning) { httpServer->Update(); + DownloadManager::Update(); } OnExit(); @@ -52,6 +53,7 @@ void Framework::PreInit() LogHistory::PreInit(); XGConfig::PreInit(); RestQueryHandler::PreInit(); + DownloadManager::PreInit(); return; } @@ -67,6 +69,7 @@ void Framework::PostInit() void Framework::OnExit() { httpServer->OnExit(); + DownloadManager::OnExit(); return; } @@ -76,6 +79,7 @@ void Framework::PostExit() XGConfig::PostExit(); RestQueryHandler::PostExit(); LogHistory::PostExit(); + DownloadManager::PostExit(); return; } diff --git a/Tubio/Framework.h b/Tubio/Framework.h index 49751b8..efb48de 100644 --- a/Tubio/Framework.h +++ b/Tubio/Framework.h @@ -2,6 +2,7 @@ #include "Logger.h" #include "LogHistory.h" #include "HttpServer.h" +#include "DownloadManager.h" #include "XGControl.h" #include "XGConfig.h" @@ -10,10 +11,9 @@ class Framework public: Framework(); ~Framework(); - -private: void Run(); +private: void PostInit(); void OnExit(); void PreInit(); diff --git a/Tubio/HttpServer.cpp b/Tubio/HttpServer.cpp index e63b925..b7b62d9 100644 --- a/Tubio/HttpServer.cpp +++ b/Tubio/HttpServer.cpp @@ -82,15 +82,30 @@ void HttpServer::EventHandler(mg_connection* pNc, int ev, void* p) http_message* hpm = (http_message*)p; std::string requestedUri = FixUnterminatedString(hpm->uri.p, hpm->uri.len); - if ((requestedUri == "/api")) + try { - ProcessAPIRequest(pNc, ev, p); + if (requestedUri == "/api") + { + ProcessAPIRequest(pNc, ev, p); + } + else + { + // Just serve the files requested + mg_serve_http(pNc, (struct http_message*)p, frontend_serve_opts); + } } - else + catch (std::exception& e) { - mg_serve_http(pNc, (struct http_message*)p, frontend_serve_opts); + Json j; + j.CloneFrom(RestResponseTemplates::GetByCode(INTERNAL_SERVER_ERROR, e.what())); + ServeStringToConnection(pNc, j.Render(), INTERNAL_SERVER_ERROR); + } + catch (...) + { + Json j; + j.CloneFrom(RestResponseTemplates::GetByCode(INTERNAL_SERVER_ERROR, "Das not gud")); + ServeStringToConnection(pNc, j.Render(), INTERNAL_SERVER_ERROR); } - break; } @@ -112,19 +127,20 @@ void HttpServer::ProcessAPIRequest(mg_connection* pNc, int ev, void* p) Json requestBody; requestBody.Parse(requestBodyRaw); - char addr[32]; - mg_sock_addr_to_str(&pNc->sa, addr, sizeof(addr), MG_SOCK_STRINGIFY_IP); + char peer_addr[32]; + mg_sock_addr_to_str(&pNc->sa, peer_addr, sizeof(peer_addr), MG_SOCK_STRINGIFY_IP); JsonBlock responseBody; HTTP_STATUS_CODE returnCode; - RestQueryHandler::ProcessQuery(std::string(addr), requestBody, responseBody, returnCode); + RestQueryHandler::ProcessQuery(std::string(peer_addr), requestBody, responseBody, returnCode); Json response(responseBody); ServeStringToConnection(pNc, response.Render(), returnCode); } else // return error message for invalid json { - Json errorJson = RestResponseTemplates::GetByCode(HTTP_STATUS_CODE::BAD_REQUEST, "Received json is fucked up"); + Json errorJson; + errorJson.CloneFrom(RestResponseTemplates::GetByCode(HTTP_STATUS_CODE::BAD_REQUEST, "Received json is fucked")); ServeStringToConnection(pNc, errorJson.Render(), HTTP_STATUS_CODE::BAD_REQUEST); } diff --git a/Tubio/JasonPP.cpp b/Tubio/JasonPP.cpp index 31697fa..3893d85 100644 --- a/Tubio/JasonPP.cpp +++ b/Tubio/JasonPP.cpp @@ -1,25 +1,25 @@ /* -MIT License - -JasonPP, Copyright (c) 2020, Leon Etienne - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +MIT License + +JasonPP, Copyright (c) 2020, Leon Etienne + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. */ #include "JasonPP.hpp" @@ -262,7 +262,7 @@ std::size_t JsonArray::RemoveSimilar(const JsonData reference) return counter; } -std::size_t JsonArray::RemoveAllOfType(const JDType type) +std::size_t JsonArray::RemoveAllOfType(const JSON_DATA_TYPE type) { std::size_t counter = 0; for (long long int i = content->size() - 1; i >= 0; i--) @@ -277,7 +277,7 @@ std::size_t JsonArray::RemoveAllOfType(const JDType type) return counter; } -std::size_t JsonArray::RemoveAllExceptType(const JDType type) +std::size_t JsonArray::RemoveAllExceptType(const JSON_DATA_TYPE type) { std::size_t counter = 0; for (long long int i = content->size() - 1; i >= 0; i--) @@ -524,7 +524,7 @@ void JsonArray::Sort(const std::string shorthandKey, const JSON_ARRAY_SORT_MODE const JsonData* b = &At(j + 1); // Check if they are of type json (this is the json sorter) (deep sort) - if ((a->GetDataType() == JDType::JSON) && (b->GetDataType() == JDType::JSON)) + if ((a->GetDataType() == JSON_DATA_TYPE::JSON) && (b->GetDataType() == JSON_DATA_TYPE::JSON)) { // Check if the requested key even exists if ((a->GetJsonData().DoesShorthandExist(shorthandKey, shorthandDelimiter)) && @@ -560,8 +560,8 @@ void JsonArray::Sort(const JSON_ARRAY_SORT_MODE mode) // Only if neither a or b's are neither of type json or array // This is the "shallow"-sort - if (!(((a.GetDataType() == JDType::JSON) || (a.GetDataType() == JDType::ARRAY)) && - ((b.GetDataType() == JDType::JSON) || (b.GetDataType() == JDType::ARRAY)))) + if (!(((a.GetDataType() == JSON_DATA_TYPE::JSON) || (a.GetDataType() == JSON_DATA_TYPE::ARRAY)) && + ((b.GetDataType() == JSON_DATA_TYPE::JSON) || (b.GetDataType() == JSON_DATA_TYPE::ARRAY)))) { if (Sort__Compare(a, b, mode)) { @@ -590,11 +590,11 @@ bool JsonArray::Sort__Compare(const JsonData& a, const JsonData& b, const JSON_A // If it's BOOL, INT or FLOAT, just get it's value as string. Eg "53", "53.2" or "false" // This way numerics can still be sorted alphabetically // Also allows for sorting after bools - if (a.GetDataType() == JDType::STRING) aStr = a.GetStringData(); - else if ((a.GetDataType() != JDType::JSON) && (a.GetDataType() != JDType::ARRAY)) aStr = a.Render(); + if (a.GetDataType() == JSON_DATA_TYPE::STRING) aStr = a.GetStringData(); + else if ((a.GetDataType() != JSON_DATA_TYPE::JSON) && (a.GetDataType() != JSON_DATA_TYPE::ARRAY)) aStr = a.Render(); else return true; // Datatype invalid. Swap, to keep the others in order. - if (b.GetDataType() == JDType::STRING) bStr = b.GetStringData(); - else if ((b.GetDataType() != JDType::JSON) && (b.GetDataType() != JDType::ARRAY)) bStr = b.Render(); + if (b.GetDataType() == JSON_DATA_TYPE::STRING) bStr = b.GetStringData(); + else if ((b.GetDataType() != JSON_DATA_TYPE::JSON) && (b.GetDataType() != JSON_DATA_TYPE::ARRAY)) bStr = b.Render(); else return true; // Datatype invalid. Swap, to keep the others in order. return StringHelpers::SortDescriminator_Alphabetically(aStr, bStr); @@ -612,11 +612,11 @@ bool JsonArray::Sort__Compare(const JsonData& a, const JsonData& b, const JSON_A // If it's BOOL, INT or FLOAT, just get it's value as string. Eg "53", "53.2" or "false" // This way numerics can still be sorted alphabetically // Also allows for sorting after bools - if (a.GetDataType() == JDType::STRING) aStr = a.GetStringData(); - else if ((a.GetDataType() != JDType::JSON) && (a.GetDataType() != JDType::ARRAY)) aStr = a.Render(); + if (a.GetDataType() == JSON_DATA_TYPE::STRING) aStr = a.GetStringData(); + else if ((a.GetDataType() != JSON_DATA_TYPE::JSON) && (a.GetDataType() != JSON_DATA_TYPE::ARRAY)) aStr = a.Render(); else return true; // Datatype invalid. Swap, to keep the others in order. - if (b.GetDataType() == JDType::STRING) bStr = b.GetStringData(); - else if ((b.GetDataType() != JDType::JSON) && (b.GetDataType() != JDType::ARRAY)) bStr = b.Render(); + if (b.GetDataType() == JSON_DATA_TYPE::STRING) bStr = b.GetStringData(); + else if ((b.GetDataType() != JSON_DATA_TYPE::JSON) && (b.GetDataType() != JSON_DATA_TYPE::ARRAY)) bStr = b.Render(); else return true; // Datatype invalid. Swap, to keep the others in order. return StringHelpers::SortDescriminator_Alphabetically(bStr, aStr); @@ -631,13 +631,13 @@ bool JsonArray::Sort__Compare(const JsonData& a, const JsonData& b, const JSON_A switch (a.GetDataType()) { - case JDType::INT: + case JSON_DATA_TYPE::INT: dataA = (long double)a.GetIntData(); break; - case JDType::FLOAT: + case JSON_DATA_TYPE::FLOAT: dataA = a.GetFloatData(); break; - case JDType::BOOL: + case JSON_DATA_TYPE::BOOL: dataA = a.GetBoolData() ? 1.0 : 0.0; break; default: @@ -645,13 +645,13 @@ bool JsonArray::Sort__Compare(const JsonData& a, const JsonData& b, const JSON_A } switch (b.GetDataType()) { - case JDType::INT: + case JSON_DATA_TYPE::INT: dataB = (long double)b.GetIntData(); break; - case JDType::FLOAT: + case JSON_DATA_TYPE::FLOAT: dataB = b.GetFloatData(); break; - case JDType::BOOL: + case JSON_DATA_TYPE::BOOL: dataB = b.GetBoolData() ? 1.0 : 0.0; break; default: @@ -669,13 +669,13 @@ bool JsonArray::Sort__Compare(const JsonData& a, const JsonData& b, const JSON_A switch (a.GetDataType()) { - case JDType::INT: + case JSON_DATA_TYPE::INT: dataA = (long double)a.GetIntData(); break; - case JDType::FLOAT: + case JSON_DATA_TYPE::FLOAT: dataA = a.GetFloatData(); break; - case JDType::BOOL: + case JSON_DATA_TYPE::BOOL: dataA = a.GetBoolData() ? 1.0 : 0.0; break; default: @@ -683,13 +683,13 @@ bool JsonArray::Sort__Compare(const JsonData& a, const JsonData& b, const JSON_A } switch (b.GetDataType()) { - case JDType::INT: + case JSON_DATA_TYPE::INT: dataB = (long double)b.GetIntData(); break; - case JDType::FLOAT: + case JSON_DATA_TYPE::FLOAT: dataB = b.GetFloatData(); break; - case JDType::BOOL: + case JSON_DATA_TYPE::BOOL: dataB = b.GetBoolData() ? 1.0 : 0.0; break; default: @@ -753,7 +753,7 @@ JsonArray& JsonArray::operator-=(const JsonData& data) return *this; } -JsonArray& JsonArray::operator-=(const JDType type) +JsonArray& JsonArray::operator-=(const JSON_DATA_TYPE type) { RemoveAllOfType(type); return *this; @@ -954,7 +954,8 @@ void JsonArray::Parse(const std::string jsonCode) { const char c = minifiedCode[i]; - if ((!areWeInString) && (!areWeInCode) && (arrayBracketLevel == 1) && (curlyBracketLevel == 0)) + if ((!areWeInString) && (!areWeInCode) && (arrayBracketLevel == 1) && + (curlyBracketLevel == 0) && (i < minifiedCode.length() - 1)) { start = i; areWeInCode = true; @@ -1031,29 +1032,29 @@ bool JasonPP::IsJsonValid(const std::string code) return true; } -std::string JasonPP::JsonDataType2String(const JDType type) +std::string JasonPP::JsonDataType2String(const JSON_DATA_TYPE type) { switch (type) { - case JDType::__NULL__: + case JSON_DATA_TYPE::__NULL__: return std::string("NULL"); - case JDType::BOOL: + case JSON_DATA_TYPE::BOOL: return std::string("BOOL"); - case JDType::INT: + case JSON_DATA_TYPE::INT: return std::string("INT"); - case JDType::FLOAT: + case JSON_DATA_TYPE::FLOAT: return std::string("FLOAT"); - case JDType::STRING: + case JSON_DATA_TYPE::STRING: return std::string("STRING"); - case JDType::JSON: + case JSON_DATA_TYPE::JSON: return std::string("JSON"); - case JDType::ARRAY: + case JSON_DATA_TYPE::ARRAY: return std::string("ARRAY"); } @@ -1457,37 +1458,37 @@ JsonData::JsonData() } // Set default data per type -JsonData::JsonData(const JDType type) +JsonData::JsonData(const JSON_DATA_TYPE type) { Init(); switch (type) { - case JDType::__NULL__: + case JSON_DATA_TYPE::__NULL__: // Default value is already NULL break; - case JDType::BOOL: + case JSON_DATA_TYPE::BOOL: SetBoolData(false); break; - case JDType::INT: + case JSON_DATA_TYPE::INT: SetIntData(0); break; - case JDType::FLOAT: + case JSON_DATA_TYPE::FLOAT: SetFloatData(0); break; - case JDType::STRING: + case JSON_DATA_TYPE::STRING: SetStringData(""); break; - case JDType::JSON: + case JSON_DATA_TYPE::JSON: SetJsonDataAsPointer(new JsonBlock()); break; - case JDType::ARRAY: + case JSON_DATA_TYPE::ARRAY: SetArrayDataAsPointer(new JsonArray()); break; } @@ -1602,7 +1603,7 @@ void JsonData::SetFloatPrecision(const double precision) JsonArray& JsonData::SetArrayDataAsPointer(JsonArray* p) { - dataType = JDType::ARRAY; + dataType = JSON_DATA_TYPE::ARRAY; if (arrayData != nullptr) { @@ -1617,7 +1618,7 @@ JsonArray& JsonData::SetArrayDataAsPointer(JsonArray* p) JsonBlock& JsonData::SetJsonDataAsPointer(JsonBlock* p) { - dataType = JDType::JSON; + dataType = JSON_DATA_TYPE::JSON; if (jsonData != nullptr) { @@ -1631,7 +1632,7 @@ JsonBlock& JsonData::SetJsonDataAsPointer(JsonBlock* p) void JsonData::Init() { - dataType = JDType::__NULL__; + dataType = JSON_DATA_TYPE::__NULL__; intData = 0l; floatData = 0.0f; stringData = std::string(); @@ -1686,12 +1687,12 @@ double JsonData::GetFloatPrecision() const bool JsonData::IsOfNumericType() const { - return (dataType == JDType::INT) || (dataType == JDType::FLOAT); + return (dataType == JSON_DATA_TYPE::INT) || (dataType == JSON_DATA_TYPE::FLOAT); } bool JsonData::GetBoolData() const { - JDType typeToGet = JDType::BOOL; + JSON_DATA_TYPE typeToGet = JSON_DATA_TYPE::BOOL; if (dataType != typeToGet) ThrowDataTypeException(typeToGet); @@ -1700,11 +1701,11 @@ bool JsonData::GetBoolData() const long long int JsonData::GetIntData() const { - JDType typeToGet = JDType::INT; + JSON_DATA_TYPE typeToGet = JSON_DATA_TYPE::INT; if (dataType != typeToGet) { - if (dataType == JDType::FLOAT) + if (dataType == JSON_DATA_TYPE::FLOAT) { return (long long int)floatData; } @@ -1719,11 +1720,11 @@ long long int JsonData::GetIntData() const long double JsonData::GetFloatData() const { - JDType typeToGet = JDType::FLOAT; + JSON_DATA_TYPE typeToGet = JSON_DATA_TYPE::FLOAT; if (dataType != typeToGet) { - if (dataType == JDType::INT) + if (dataType == JSON_DATA_TYPE::INT) { return (float)intData; } @@ -1738,7 +1739,7 @@ long double JsonData::GetFloatData() const std::string JsonData::GetStringData() const { - JDType typeToGet = JDType::STRING; + JSON_DATA_TYPE typeToGet = JSON_DATA_TYPE::STRING; if (dataType != typeToGet) ThrowDataTypeException(typeToGet); @@ -1747,35 +1748,35 @@ std::string JsonData::GetStringData() const JsonBlock& JsonData::GetJsonData() { - JDType typeToGet = JDType::JSON; + JSON_DATA_TYPE typeToGet = JSON_DATA_TYPE::JSON; if (dataType != typeToGet) ThrowDataTypeException(typeToGet); return *jsonData; } const JsonBlock& JsonData::GetJsonData() const { - JDType typeToGet = JDType::JSON; + JSON_DATA_TYPE typeToGet = JSON_DATA_TYPE::JSON; if (dataType != typeToGet) ThrowDataTypeException(typeToGet); return *jsonData; } JsonArray& JsonData::GetArrayData() { - JDType typeToGet = JDType::ARRAY; + JSON_DATA_TYPE typeToGet = JSON_DATA_TYPE::ARRAY; if (dataType != typeToGet) ThrowDataTypeException(typeToGet); return *arrayData; } const JsonArray& JsonData::GetArrayData() const { - JDType typeToGet = JDType::ARRAY; + JSON_DATA_TYPE typeToGet = JSON_DATA_TYPE::ARRAY; if (dataType != typeToGet) ThrowDataTypeException(typeToGet); return *arrayData; } short JsonData::GetNullData() { - JDType typeToGet = JDType::__NULL__; + JSON_DATA_TYPE typeToGet = JSON_DATA_TYPE::__NULL__; if (dataType != typeToGet) ThrowDataTypeException(typeToGet); @@ -1803,7 +1804,7 @@ bool JsonData::HasParent() const void JsonData::SetBoolData(const bool data) { - dataType = JDType::BOOL; + dataType = JSON_DATA_TYPE::BOOL; intData = (int)data; return; @@ -1811,7 +1812,7 @@ void JsonData::SetBoolData(const bool data) void JsonData::SetIntData(const long long int data) { - dataType = JDType::INT; + dataType = JSON_DATA_TYPE::INT; intData = data; return; @@ -1819,7 +1820,7 @@ void JsonData::SetIntData(const long long int data) void JsonData::SetIntData(const int data) { - dataType = JDType::INT; + dataType = JSON_DATA_TYPE::INT; intData = data; return; @@ -1827,7 +1828,7 @@ void JsonData::SetIntData(const int data) void JsonData::SetFloatData(const long double data) { - dataType = JDType::FLOAT; + dataType = JSON_DATA_TYPE::FLOAT; floatData = data; return; @@ -1835,7 +1836,7 @@ void JsonData::SetFloatData(const long double data) void JsonData::SetStringData(const std::string data) { - dataType = JDType::STRING; + dataType = JSON_DATA_TYPE::STRING; stringData = data; return; @@ -1848,7 +1849,7 @@ JsonBlock& JsonData::SetJsonData(const JsonBlock data) JsonArray& JsonData::SetArrayData(const std::vector data) { - dataType = JDType::ARRAY; + dataType = JSON_DATA_TYPE::ARRAY; JsonArray* newArr = new JsonArray; newArr->CopyJsonDataFromVector_Pointer(&data); // Slightly more performant than constructor return SetArrayDataAsPointer(newArr); @@ -1861,14 +1862,14 @@ JsonArray& JsonData::SetArrayData(const JsonArray data) void JsonData::SetNull() { - dataType = JDType::__NULL__; + dataType = JSON_DATA_TYPE::__NULL__; return; } /* MISC */ -void JsonData::ThrowDataTypeException(const JDType toFetch) const +void JsonData::ThrowDataTypeException(const JSON_DATA_TYPE toFetch) const { throw JsonWrongDataTypeException( JsonDataType2String(toFetch), @@ -1890,32 +1891,32 @@ std::string JsonData::Render(unsigned int num_tabs, const bool minify) const switch (dataType) { - case JDType::__NULL__: + case JSON_DATA_TYPE::__NULL__: ss << "null"; break; - case JDType::BOOL: + case JSON_DATA_TYPE::BOOL: ss << ((intData != 0) ? "true" : "false"); break; - case JDType::INT: + case JSON_DATA_TYPE::INT: ss << intData; break; - case JDType::FLOAT: + case JSON_DATA_TYPE::FLOAT: ss.precision((std::streamsize)((-log10(GetFloatPrecision())) + 1)); ss << floatData; break; - case JDType::STRING: + case JSON_DATA_TYPE::STRING: ss << "\"" << StringHelpers::Escape(stringData) << "\""; break; - case JDType::JSON: + case JSON_DATA_TYPE::JSON: ss << jsonData->Render(num_tabs, minify); break; - case JDType::ARRAY: + case JSON_DATA_TYPE::ARRAY: ss << arrayData->Render(num_tabs, minify); break; } @@ -2007,31 +2008,31 @@ void JsonData::CloneFrom(const JsonData& other) switch (other.dataType) { - case JDType::__NULL__: + case JSON_DATA_TYPE::__NULL__: // Default value is already NULL break; - case JDType::BOOL: + case JSON_DATA_TYPE::BOOL: SetBoolData(other.intData != 0); break; - case JDType::INT: + case JSON_DATA_TYPE::INT: SetIntData(other.intData); break; - case JDType::FLOAT: + case JSON_DATA_TYPE::FLOAT: SetFloatData(other.floatData); break; - case JDType::STRING: + case JSON_DATA_TYPE::STRING: SetStringData(other.stringData); break; - case JDType::ARRAY: + case JSON_DATA_TYPE::ARRAY: SetArrayData(*other.arrayData); break; - case JDType::JSON: + case JSON_DATA_TYPE::JSON: SetJsonData(*other.jsonData); break; } @@ -2042,8 +2043,8 @@ void JsonData::CloneFrom(const JsonData& other) bool JsonData::IsIdentical(const JsonData& other) const { // Special case for int/float implicit conversion - if (((dataType == JDType::INT) && (other.dataType == JDType::FLOAT)) || - ((other.dataType == JDType::INT) && (dataType == JDType::FLOAT))) + if (((dataType == JSON_DATA_TYPE::INT) && (other.dataType == JSON_DATA_TYPE::FLOAT)) || + ((other.dataType == JSON_DATA_TYPE::INT) && (dataType == JSON_DATA_TYPE::FLOAT))) { // Here we have to get the float value via the getter because of implicit conversion // Use the more precise precision of the two... @@ -2055,32 +2056,32 @@ bool JsonData::IsIdentical(const JsonData& other) const switch (dataType) { - case JDType::__NULL__: + case JSON_DATA_TYPE::__NULL__: return true; - case JDType::BOOL: + case JSON_DATA_TYPE::BOOL: // Values can't be of different type because of the check at the beginning of the function return intData == other.intData; break; - case JDType::INT: + case JSON_DATA_TYPE::INT: return intData == other.intData; break; - case JDType::FLOAT: + case JSON_DATA_TYPE::FLOAT: // Use the more precise precision of the two... return Helpers::AreSame(floatData, other.floatData, Helpers::Min(GetFloatPrecision(), other.GetFloatPrecision())); break; - case JDType::STRING: + case JSON_DATA_TYPE::STRING: return stringData == other.stringData; break; - case JDType::ARRAY: + case JSON_DATA_TYPE::ARRAY: return arrayData->IsIdentical(*other.arrayData); break; - case JDType::JSON: + case JSON_DATA_TYPE::JSON: return jsonData->IsIdentical(*other.jsonData); break; } @@ -2140,11 +2141,11 @@ bool JsonData::operator!=(const JsonData& other) const JsonData& JsonData::operator+=(const JsonElement ele) { - if (dataType == JDType::JSON) + if (dataType == JSON_DATA_TYPE::JSON) { return jsonData->Add(ele); } - ThrowDataTypeException(JDType::JSON); + ThrowDataTypeException(JSON_DATA_TYPE::JSON); std::terminate(); } @@ -2257,7 +2258,7 @@ JsonData::operator long double() const JsonData::operator std::string() const { - if (dataType == JDType::STRING) return GetStringData(); + if (dataType == JSON_DATA_TYPE::STRING) return GetStringData(); else return Render(JASONPP_STRINGCONV_MINIFY); } @@ -2266,7 +2267,7 @@ namespace JasonPP { std::ostream& operator<<(std::ostream& os, const JsonData& jd) { - if (jd.dataType == JDType::STRING) return os << jd.GetStringData(); + if (jd.dataType == JSON_DATA_TYPE::STRING) return os << jd.GetStringData(); else return os << jd.Render(JASONPP_STRINGCONV_MINIFY); } } @@ -2690,10 +2691,10 @@ bool JsonBlock::DoesShorthandExist(const std::string shorthand, const std::strin const JsonBlock* jb = &const_cast(*this); for (std::size_t i = 0; i < segments.size(); i++) { - if ((jb->DoesExist(segments[i])) && ((jb->Get(segments[i]).GetDataType() == JDType::JSON) || (i == segments.size() - 1))) + if ((jb->DoesExist(segments[i])) && ((jb->Get(segments[i]).GetDataType() == JSON_DATA_TYPE::JSON) || (i == segments.size() - 1))) { if (i == segments.size() - 1) return true; // We are at the end. Let's just return it - if (jb->Get(segments[i]).GetDataType() == JDType::JSON) jb = &jb->Get(segments[i]).GetJsonData(); + if (jb->Get(segments[i]).GetDataType() == JSON_DATA_TYPE::JSON) jb = &jb->Get(segments[i]).GetJsonData(); else return false; } else @@ -2718,10 +2719,10 @@ const JsonData& JsonBlock::ShorthandGet(const std::string shorthand, const std:: const JsonBlock* jb = &const_cast(*this); for (std::size_t i = 0; i < segments.size(); i++) { - if ((jb->DoesExist(segments[i])) && ((jb->Get(segments[i]).GetDataType() == JDType::JSON) || (i == segments.size() - 1))) + if ((jb->DoesExist(segments[i])) && ((jb->Get(segments[i]).GetDataType() == JSON_DATA_TYPE::JSON) || (i == segments.size() - 1))) { if (i == segments.size() - 1) return jb->Get(segments[i]); // We are at the end. Let's just return it - if (jb->Get(segments[i]).GetDataType() == JDType::JSON) jb = &jb->Get(segments[i]).GetJsonData(); + if (jb->Get(segments[i]).GetDataType() == JSON_DATA_TYPE::JSON) jb = &jb->Get(segments[i]).GetJsonData(); else throw JsonShorthandInvalidException(shorthand); } else @@ -2757,7 +2758,7 @@ JsonData& JsonBlock::ShorthandAdd(const std::string shorthand, const std::string } else { - if (jb->Get(segments[i]).GetDataType() != JDType::JSON) throw JsonShorthandInvalidException(shorthand, "A path segment already exists and is not of type json!"); + if (jb->Get(segments[i]).GetDataType() != JSON_DATA_TYPE::JSON) throw JsonShorthandInvalidException(shorthand, "A path segment already exists and is not of type json!"); jb = &jb->Get(segments[i]).GetJsonData(); } } @@ -2804,7 +2805,7 @@ void JsonBlock::ShorthandRemove(const std::string shorthand, const std::string d JsonData* dt = &ShorthandGet(shorthandParent, delimiter); // Is the parent object of the object to be deleted even of type json? - if (dt->GetDataType() != JDType::JSON) throw JsonShorthandInvalidException(shorthand, "The parent of the object to be deleted is not of type json!"); + if (dt->GetDataType() != JSON_DATA_TYPE::JSON) throw JsonShorthandInvalidException(shorthand, "The parent of the object to be deleted is not of type json!"); parentJson = &dt->GetJsonData(); } diff --git a/Tubio/JasonPP.hpp b/Tubio/JasonPP.hpp index c5746cd..a9cf535 100644 --- a/Tubio/JasonPP.hpp +++ b/Tubio/JasonPP.hpp @@ -2142,7 +2142,7 @@ namespace JasonPP }; } -#define JASONPP_VERSION (1.021) +#define JASONPP_VERSION (1.0215) namespace JasonPP { diff --git a/Tubio/RestQueryHandler.cpp b/Tubio/RestQueryHandler.cpp index 28e9f7f..3a37191 100644 --- a/Tubio/RestQueryHandler.cpp +++ b/Tubio/RestQueryHandler.cpp @@ -2,6 +2,7 @@ using namespace Rest; using namespace Logging; +using namespace Downloader; using namespace JasonPP; void RestQueryHandler::PreInit() @@ -26,6 +27,9 @@ bool RestQueryHandler::ProcessQuery(const std::string clientAdress, const Json& if (requestName == "kill_yourself") return KillYourself(requestBody, responseBody, responseCode); + else if (requestName == "queue_download") return QueueDownload(requestBody, responseBody, responseCode); + else if (requestName == "fetch_queue") return FetchQueue(requestBody, responseBody, responseCode); + else if (requestName == "clear_download_cache") return ClearDownloadCache(requestBody, responseBody, responseCode); else if (requestName == "foo") return Example_Foo(requestBody, responseBody, responseCode); @@ -52,6 +56,57 @@ bool RestQueryHandler::Example_Foo(const JsonBlock& request, JsonBlock& response return true; } +bool RestQueryHandler::QueueDownload(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode) +{ + if ((!ValidateField("video_url", JDType::STRING, request, responseBody)) || + (!ValidateField("mode", JDType::STRING, request, responseBody))) + { + responseCode = BAD_REQUEST; + return false; + } + + std::string modeParam = request.Get("mode").AsString; + std::string videoUrl = request.Get("video_url").AsString; + DOWNLOAD_MODE mode; + if (modeParam == "video") mode = DOWNLOAD_MODE::VIDEO; + else if (modeParam == "audio") mode = DOWNLOAD_MODE::AUDIO; + else + { + responseCode = BAD_REQUEST; + responseBody.CloneFrom(RestResponseTemplates::GetByCode(BAD_REQUEST, "Parameter 'mode' is of wrong value. Should be either 'video' or 'audio'.")); + return false; + } + + log->cout << "Queued video \"" << videoUrl << "\"..."; + log->Flush(); + + std::string tubId = DownloadManager::QueueDownload(videoUrl, mode); + + responseCode = OK; + responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK)); + responseBody.Set("message") = "Download queued!"; + responseBody.Set("queue_position") = (long long int)DownloadManager::GetQueueLength(); + responseBody.Set("tubio_id") = tubId; + responseBody.Set("queue") = DownloadManager::GetQueueAsJson(); + return true; +} + +bool RestQueryHandler::FetchQueue(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode) +{ + responseCode = OK; + responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK)); + responseBody.Set("queue") = DownloadManager::GetQueueAsJson(); + return true; +} + +bool RestQueryHandler::ClearDownloadCache(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode) +{ + responseCode = OK; + responseBody.CloneFrom(RestResponseTemplates::GetByCode(OK)); + DownloadManager::ClearDownloadCache(); + return true; +} + bool RestQueryHandler::KillYourself(const JsonBlock& request, JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode) { XGControl::keepServerRunning = false; @@ -65,6 +120,23 @@ bool RestQueryHandler::KillYourself(const JsonBlock& request, JsonBlock& respons return true; } + + + + + + + + + + + + + + + + + bool RestQueryHandler::ValidateField(const std::string name, const JasonPP::JDType type, const JasonPP::Json& checkThat, JasonPP::JsonBlock& putErrorResponseHere) { if (checkThat.GetDataType() != JDType::JSON) @@ -103,4 +175,4 @@ bool RestQueryHandler::ValidateField(const std::string name, const JasonPP::JDTy return false; } -Logger* RestQueryHandler::log; \ No newline at end of file +Logger* RestQueryHandler::log; diff --git a/Tubio/RestQueryHandler.h b/Tubio/RestQueryHandler.h index 8984c13..acbed23 100644 --- a/Tubio/RestQueryHandler.h +++ b/Tubio/RestQueryHandler.h @@ -3,6 +3,7 @@ #include "RestResponseTemplates.h" #include "XGControl.h" #include "Logger.h" +#include "DownloadManager.h" namespace Rest { @@ -17,6 +18,9 @@ namespace Rest private: static bool Example_Foo(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode); + static bool FetchQueue(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode); + static bool QueueDownload(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode); + static bool ClearDownloadCache(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode); static bool KillYourself(const JasonPP::JsonBlock& request, JasonPP::JsonBlock& responseBody, HTTP_STATUS_CODE& responseCode); static bool ValidateField(const std::string name, const JasonPP::JDType type, const JasonPP::Json& checkThat, JasonPP::JsonBlock& putErrorResponseHere); diff --git a/Tubio/Tubio.vcxproj b/Tubio/Tubio.vcxproj index 5ad5a1c..0098af9 100644 --- a/Tubio/Tubio.vcxproj +++ b/Tubio/Tubio.vcxproj @@ -116,6 +116,7 @@ true _DEBUG;_CONSOLE;%(PreprocessorDefinitions) true + stdcpp17 Console @@ -139,6 +140,7 @@ + @@ -153,6 +155,7 @@ + diff --git a/Tubio/Tubio.vcxproj.filters b/Tubio/Tubio.vcxproj.filters index b01d533..f4ecddd 100644 --- a/Tubio/Tubio.vcxproj.filters +++ b/Tubio/Tubio.vcxproj.filters @@ -54,6 +54,9 @@ Quelldateien + + Quelldateien + @@ -92,5 +95,8 @@ Headerdateien + + Headerdateien + \ No newline at end of file diff --git a/Tubio/XGConfig.cpp b/Tubio/XGConfig.cpp index 86b7c4d..65b1ab1 100644 --- a/Tubio/XGConfig.cpp +++ b/Tubio/XGConfig.cpp @@ -39,6 +39,15 @@ void XGConfig::InitializeDefaultValues() logging.logfile_text = "log.txt"; logging.logfile_json = "log.json"; + downloader.cachedir = "dlcache"; + downloader.max_dlrate_per_thread = "100M"; + downloader.num_threads = 1; + + downloader.loginCredentials.use_account = false; + downloader.loginCredentials.username = ""; + downloader.loginCredentials.password = ""; + downloader.loginCredentials.twofactor = ""; + return; } @@ -71,6 +80,37 @@ void XGConfig::LoadFromJson(const JasonPP::JsonBlock& json) httpServer.rootdir = json.ShorthandGet("httpServer.rootdir").AsString; } + + + if (IsJsonFieldValid(json, "downloader.cachedir", JDType::STRING)) + { + downloader.cachedir = json.ShorthandGet("downloader.cachedir").AsString; + } + if (IsJsonFieldValid(json, "downloader.num_threads", JDType::INT)) + { + downloader.num_threads = json.ShorthandGet("downloader.num_threads").AsInt; + } + if (IsJsonFieldValid(json, "downloader.max_dlrate_per_thread", JDType::STRING)) + { + downloader.max_dlrate_per_thread = json.ShorthandGet("downloader.max_dlrate_per_thread").AsString; + } + + if (IsJsonFieldValid(json, "downloader.loginCredentials.use_account", JDType::BOOL)) + { + downloader.loginCredentials.use_account = json.ShorthandGet("downloader.loginCredentials.use_account").AsBool; + } + if (IsJsonFieldValid(json, "downloader.loginCredentials.username", JDType::STRING)) + { + downloader.loginCredentials.username = json.ShorthandGet("downloader.loginCredentials.username").AsString; + } + if (IsJsonFieldValid(json, "downloader.loginCredentials.password", JDType::STRING)) + { + downloader.loginCredentials.password = json.ShorthandGet("downloader.loginCredentials.password").AsString; + } + if (IsJsonFieldValid(json, "downloader.loginCredentials.twofactor", JDType::STRING)) + { + downloader.loginCredentials.twofactor = json.ShorthandGet("downloader.loginCredentials.twofactor").AsString; + } return; } @@ -86,6 +126,17 @@ JsonBlock XGConfig::CreateJson() Ele("logging", JsonBlock({ Ele("logfile_text", logging.logfile_text), Ele("logfile_json", logging.logfile_json) + })), + Ele("downloader", JsonBlock({ + Ele("cachedir", downloader.cachedir), + Ele("max_dlrate_per_thread", downloader.max_dlrate_per_thread), + Ele("num_threads", downloader.num_threads), + Ele("login_credentials", JsonBlock({ + Ele("use_account", downloader.loginCredentials.use_account), + Ele("username", downloader.loginCredentials.username), + Ele("password", downloader.loginCredentials.password), + Ele("twofactor", downloader.loginCredentials.twofactor) + })) })) }); } @@ -197,4 +248,5 @@ void XGConfig::Load() XGConfig::HttpServer XGConfig::httpServer; XGConfig::Logging XGConfig::logging; +XGConfig::Downloader XGConfig::downloader; ::Logging::Logger* XGConfig::log; diff --git a/Tubio/XGConfig.h b/Tubio/XGConfig.h index 8a6c44d..2c8a241 100644 --- a/Tubio/XGConfig.h +++ b/Tubio/XGConfig.h @@ -20,7 +20,22 @@ public: std::string logfile_text; std::string logfile_json; }; + struct Downloader + { + struct LoginCredentials + { + bool use_account; + std::string username; + std::string password; + std::string twofactor; + }; + std::string cachedir; + std::string max_dlrate_per_thread; + LoginCredentials loginCredentials; + + int num_threads; + }; static void PreInit(); static void Save(); @@ -28,6 +43,7 @@ public: static HttpServer httpServer; static XGConfig::Logging logging; + static XGConfig::Downloader downloader; static ::Logging::Logger* log; diff --git a/Tubio/main.cpp b/Tubio/main.cpp index be185c6..c203008 100644 --- a/Tubio/main.cpp +++ b/Tubio/main.cpp @@ -3,6 +3,7 @@ int main() { Framework framework; + framework.Run(); return 0; } diff --git a/Tubio/youtube-dl.exe b/Tubio/youtube-dl.exe new file mode 100644 index 0000000..799a240 Binary files /dev/null and b/Tubio/youtube-dl.exe differ