From 08675d678bb5ee291eb1de44a23bd9d987a9f53b Mon Sep 17 00:00:00 2001 From: Mike Nolan Date: Wed, 6 Jul 2022 17:59:50 -0500 Subject: [PATCH] Added File Download, abstracted JSON --- .gitignore | 2 + ...s.YouTubeDownloader.ExtensionLoader.csproj | 6 +- .../Config.cs | 24 +++ .../Program.cs | 23 +++ ...esses.YouTubeDownloader.ServerProxy.csproj | 19 +++ .../proxy.json | 4 + .../StringUtils.cs | 18 +++ .../SymlinkGenerator.cs | 96 +++++++++++ .../TYTDContext.cs | 34 ++++ ...sses.YouTubeDownloader.Tools.Common.csproj | 14 ++ .../Program.cs | 83 ++++++++++ ...YouTubeDownloader.Tools.LinkCreator.csproj | 14 ++ Tesses.YouTubeDownloader.Server/Class1.cs | 151 +++++++++++++++++- .../Tesses.YouTubeDownloader.Server.csproj | 8 +- Tesses.YouTubeDownloader/B64.cs | 58 +++++++ Tesses.YouTubeDownloader/BestStreams.cs | 30 ++-- Tesses.YouTubeDownloader/ConvertLegacy.cs | 2 +- Tesses.YouTubeDownloader/DownloadLoop.cs | 99 +++++++++++- Tesses.YouTubeDownloader/IDownloader.cs | 3 +- Tesses.YouTubeDownloader/IStorage.cs | 2 +- Tesses.YouTubeDownloader/ITYTDBase.cs | 3 +- Tesses.YouTubeDownloader/PreMediaContext.cs | 96 +++++++++-- Tesses.YouTubeDownloader/TYTD.cs | 37 +++-- Tesses.YouTubeDownloader/TYTDBase.cs | 82 +++++++--- Tesses.YouTubeDownloader/TYTDClient.cs | 10 +- .../TYTDIDownloaderStorageProxy.cs | 32 ++++ .../Tesses.YouTubeDownloader.csproj | 6 +- 27 files changed, 874 insertions(+), 82 deletions(-) create mode 100644 Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/Config.cs create mode 100644 Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/Program.cs create mode 100644 Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/Tesses.YouTubeDownloader.ServerProxy.csproj create mode 100644 Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/proxy.json create mode 100644 Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/StringUtils.cs create mode 100644 Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/SymlinkGenerator.cs create mode 100644 Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/TYTDContext.cs create mode 100644 Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/Tesses.YouTubeDownloader.Tools.Common.csproj create mode 100644 Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.LinkCreator/Program.cs create mode 100644 Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.LinkCreator/Tesses.YouTubeDownloader.Tools.LinkCreator.csproj create mode 100644 Tesses.YouTubeDownloader/B64.cs diff --git a/.gitignore b/.gitignore index da4ad96..c54a85f 100644 --- a/.gitignore +++ b/.gitignore @@ -143,4 +143,6 @@ Tesses.YouTubeDownloader.Net6/config/ Tesses.YouTubeDownloader.Net6/Playlist/ Tesses.YouTubeDownloader.Net6/Channel/ Tesses.YouTubeDownloader.Net6/Subscriptions/ +Tesses.YouTubeDownloader.Net6/Download/ +Tesses.YouTubeDownloader.Net6/FileInfo/ push diff --git a/Tesses.YouTubeDownloader.ExtensionLoader/Tesses.YouTubeDownloader.ExtensionLoader.csproj b/Tesses.YouTubeDownloader.ExtensionLoader/Tesses.YouTubeDownloader.ExtensionLoader.csproj index 5babc7d..2040300 100644 --- a/Tesses.YouTubeDownloader.ExtensionLoader/Tesses.YouTubeDownloader.ExtensionLoader.csproj +++ b/Tesses.YouTubeDownloader.ExtensionLoader/Tesses.YouTubeDownloader.ExtensionLoader.csproj @@ -11,9 +11,9 @@ Tesses.YouTubeDownloader.ExtensionLoader Mike Nolan Tesses - 1.1.1 - 1.1.1 - 1.1.1 + 1.1.2 + 1.1.2 + 1.1.2 Load Extensions into TYTD (Not Tested) LGPL-3.0-only true diff --git a/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/Config.cs b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/Config.cs new file mode 100644 index 0000000..8f2ad8c --- /dev/null +++ b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/Config.cs @@ -0,0 +1,24 @@ +using Newtonsoft.Json; + +public class TYTDConfiguration +{ + public TYTDConfiguration() + { + Url = "http://127.0.0.1:3252/"; + LocalFiles=Environment.CurrentDirectory; + } + public string Url {get;set;} + + public string LocalFiles {get;set;} + + public static TYTDConfiguration Load() + { + if(!File.Exists("proxy.json")) return new TYTDConfiguration(); + var res= JsonConvert.DeserializeObject(File.ReadAllText("proxy.json")); + if(res != null) + { + return res; + } + return new TYTDConfiguration(); + } +} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/Program.cs b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/Program.cs new file mode 100644 index 0000000..5eee806 --- /dev/null +++ b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/Program.cs @@ -0,0 +1,23 @@ +using Tesses.YouTubeDownloader; +using Tesses.YouTubeDownloader.Server; +using Tesses.WebServer; +using Newtonsoft.Json; + +var config=TYTDConfiguration.Load(); +Environment.CurrentDirectory=config.LocalFiles; +var c=new HttpClient(); +TYTDCurrentDirectory currentDirectory=new TYTDCurrentDirectory(c); +TYTDClient client=new TYTDClient(c,config.Url); + +TYTDDownloaderStorageProxy proxy=new TYTDDownloaderStorageProxy(); +proxy.Storage = currentDirectory; +proxy.Downloader=client; + +TYTDServer server=new TYTDServer(proxy); +server.RootServer.Server=new StaticServer("WebSite"); +currentDirectory.CanDownload=false; +HttpServerListener listener=new HttpServerListener(new System.Net.IPEndPoint(System.Net.IPAddress.Any,3252),server.InnerServer); +currentDirectory.StartLoop(); +TYTDStorage.FFmpeg ="/usr/bin/ffmpeg"; +Console.WriteLine("Almost Ready to Listen"); +await listener.ListenAsync(CancellationToken.None); \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/Tesses.YouTubeDownloader.ServerProxy.csproj b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/Tesses.YouTubeDownloader.ServerProxy.csproj new file mode 100644 index 0000000..12432f2 --- /dev/null +++ b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/Tesses.YouTubeDownloader.ServerProxy.csproj @@ -0,0 +1,19 @@ + + + + + + + + + + + + + Exe + net6.0 + enable + enable + + + diff --git a/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/proxy.json b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/proxy.json new file mode 100644 index 0000000..8b424a3 --- /dev/null +++ b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.ServerProxy/proxy.json @@ -0,0 +1,4 @@ +{ + "Url": "http://10.137.42.142:3252/", + "LocalFiles": "/media/mike/PhotoDrive/wii-vids/working" +} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/StringUtils.cs b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/StringUtils.cs new file mode 100644 index 0000000..3dbf34e --- /dev/null +++ b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/StringUtils.cs @@ -0,0 +1,18 @@ +using System.Text; + +namespace Tesses.YouTubeDownloader.Tools.Common +{ + public static class StringUtils + { + public static string GetSafeFileName(this string filename) + { + StringBuilder b=new StringBuilder(filename); + foreach(var badChr in "\\\"\'/?*<>|:") + { + b.Replace(badChr.ToString(),""); + } + if(b.Length == 0) return "file"; + return b.ToString(); + } + } +} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/SymlinkGenerator.cs b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/SymlinkGenerator.cs new file mode 100644 index 0000000..208dfdd --- /dev/null +++ b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/SymlinkGenerator.cs @@ -0,0 +1,96 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; +using System; +namespace Tesses.YouTubeDownloader.Tools.Common +{ + public static class SymlinkGenerator + { + [DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Unicode)] + static extern bool CreateHardLink(string lpFileName, string lpExistingFileName, IntPtr lpSecurityAttributes); + private static void createHardLink(string destPath,string srcPath) + {if(RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + CreateHardLink(destPath,srcPath,IntPtr.Zero); + }else{ + using(var p=new Process()) + { + + p.StartInfo.FileName = "ln"; + p.StartInfo.ArgumentList.Add(srcPath); + p.StartInfo.ArgumentList.Add(destPath); + + if(p.Start()){ p.WaitForExit();} + } + } + } + public static async Task GenerateHardLinks(TYTDStorage storage,string dest="GoodFileNames",Resolution res=Resolution.PreMuxed,bool verbose=false) + { + Directory.CreateDirectory(dest); + await foreach(var item in storage.GetVideosAsync()) + { + if(await item.VideoExistsAsync(storage,res)) + { + var (path,delete)= await storage.GetRealUrlOrPathAsync(await BestStreams.GetPathResolution(storage,item,res)); + string? ext=Path.GetExtension(path); + + string defaultExt = res == Resolution.Mux ? ".mkv" : ".mp4"; + if(string.IsNullOrWhiteSpace(ext)) + { + ext=defaultExt; + } + string destPathMkv=Path.Combine(dest,$"{item.Title.GetSafeFileName()}-{item.Id}{defaultExt}"); + string destPath=Path.Combine(dest,$"{item.Title.GetSafeFileName()}-{item.Id}{ext}"); + + if(File.Exists(destPathMkv) && destPathMkv != destPath) + { + File.Delete(destPathMkv); + createHardLink(destPath,path); + if(verbose) + Console.WriteLine($"Changed: {item.Title} {defaultExt} -> {ext}"); + } + if(!File.Exists(destPath)) + { + createHardLink(destPath,path); + if(verbose) + Console.WriteLine(item.Title); + } + } + } + + } + public static async Task GenerateSymlinks(TYTDBase storage,string dest="GoodFileNames",Resolution res=Resolution.PreMuxed,bool verbose=false) + { + Directory.CreateDirectory(dest); + await foreach(var item in storage.GetVideosAsync()) + { + if(await item.VideoExistsAsync(storage,res)) + { + var (path,delete)= await storage.GetRealUrlOrPathAsync(await BestStreams.GetPathResolution(storage,item,res)); + string? ext=Path.GetExtension(path); + + string defaultExt = res == Resolution.Mux ? ".mkv" : ".mp4"; + if(string.IsNullOrWhiteSpace(ext)) + { + ext=defaultExt; + } + string destPathMkv=Path.Combine(dest,$"{item.Title.GetSafeFileName()}-{item.Id}{defaultExt}"); + string destPath=Path.Combine(dest,$"{item.Title.GetSafeFileName()}-{item.Id}{ext}"); + + if(File.Exists(destPathMkv) && destPathMkv != destPath) + { + File.Delete(destPathMkv); + File.CreateSymbolicLink(destPath,path); + if(verbose) + Console.WriteLine($"Changed: {item.Title} {defaultExt} -> {ext}"); + } + if(!File.Exists(destPath)) + { + File.CreateSymbolicLink(destPath,path); + if(verbose) + Console.WriteLine(item.Title); + } + } + } + } + } +} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/TYTDContext.cs b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/TYTDContext.cs new file mode 100644 index 0000000..3eaa6a9 --- /dev/null +++ b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/TYTDContext.cs @@ -0,0 +1,34 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; +using System; +using Tesses.YouTubeDownloader.SFTP; +namespace Tesses.YouTubeDownloader.Tools.Common +{ + public static class TYTDOpener + { + public static TYTDBase? GetTYTDBase(string p) + { + Uri? uri; + + if(Uri.TryCreate(p,UriKind.Absolute,out uri)) + { + if(uri.IsFile) + { + return new TYTDPathDirectory(uri.LocalPath); + } + if(uri.Scheme == "sftp") + { + return new SSHFS(uri); + } + if(uri.Scheme == "http" || uri.Scheme == "https") + { + return new TYTDClient(uri); + } + }else{ + if(!string.IsNullOrWhiteSpace(p)) return new TYTDPathDirectory(p); + } + return null; + } + + } +} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/Tesses.YouTubeDownloader.Tools.Common.csproj b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/Tesses.YouTubeDownloader.Tools.Common.csproj new file mode 100644 index 0000000..681133a --- /dev/null +++ b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.Common/Tesses.YouTubeDownloader.Tools.Common.csproj @@ -0,0 +1,14 @@ + + + + + + + + + net6.0 + enable + enable + + + diff --git a/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.LinkCreator/Program.cs b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.LinkCreator/Program.cs new file mode 100644 index 0000000..8be3e2d --- /dev/null +++ b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.LinkCreator/Program.cs @@ -0,0 +1,83 @@ +using Tesses.YouTubeDownloader.Tools.Common; +using Tesses.YouTubeDownloader; +using System; +using System.IO; + +Resolution res=Resolution.PreMuxed; + +bool verbose=false; +bool isSymlink = false; +List _args=new List(); +foreach(var arg in args) +{ + bool any=false; + if(arg == "-h" || arg == "--help") + { + _args.Clear(); + break; + } + if( (arg.Length >= 2 && arg[1] != '-' && arg[0] == '-' && arg.Contains("s") )|| arg == "--symbolic") + { + any=true; + isSymlink=true; + } + if((arg.Length >= 2 && arg[1] != '-' && arg[0] == '-' && arg.Contains("m") ) || arg == "--mux" ) + { + any=true; + res = Resolution.Mux; + } + if((arg.Length >= 2 && arg[1] != '-' && arg[0] == '-' && arg.Contains("a") ) || arg == "--audio-only") + { + any=true; + res = Resolution.AudioOnly; + + } + if ((arg.Length >= 2 && arg[1] != '-' && arg[0] == '-' && arg.Contains("V") ) || arg=="--video-only") + { + any=true; + res = Resolution.VideoOnly; + + } + if ((arg.Length >= 2 && arg[1] != '-' && arg[0] == '-' && arg.Contains("v") ) || arg=="--verbose") + { + any=true; + verbose=true; + + } + if(!any) + _args.Add(arg); + +} +string[] argv = _args.ToArray(); +if(argv.Length < 2) +{ + string app = Path.GetFileNameWithoutExtension(Environment.GetCommandLineArgs()[0]); + + Console.WriteLine($"usage: {app} [-smaVv] []"); + Console.WriteLine(); + Console.WriteLine("Options:"); + Console.WriteLine(" -s, --symbolic make symbolic links instead of hard links"); + Console.WriteLine(" -m, --mux set resolution to Mux"); + Console.WriteLine(" -a, --audio-only set resolution to AudioOnly"); + Console.WriteLine(" -V, --video-only set resolution to VideoOnly"); + Console.WriteLine(" -h, --help show this help"); + Console.WriteLine(" -v, --verbose print video names"); + Console.WriteLine(); + Console.WriteLine("Positional Arguments:"); + Console.WriteLine(" Working the folder containing the Info Directory for TYTD. (required)"); + Console.WriteLine(" Destination the folder to create links within. (required)"); + + +}else{ + + +Environment.CurrentDirectory=argv[0]; +TYTDCurrentDirectory currentDirectory=new TYTDCurrentDirectory(); +currentDirectory.CanDownload=false; + +if(isSymlink){ +await SymlinkGenerator.GenerateSymlinks(currentDirectory,argv[1],res,verbose); +}else{ +await SymlinkGenerator.GenerateHardLinks(currentDirectory,argv[1],res,verbose); +} +} diff --git a/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.LinkCreator/Tesses.YouTubeDownloader.Tools.LinkCreator.csproj b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.LinkCreator/Tesses.YouTubeDownloader.Tools.LinkCreator.csproj new file mode 100644 index 0000000..a3577b1 --- /dev/null +++ b/Tesses.YouTubeDownloader.Extras/Tesses.YouTubeDownloader.Tools/Tesses.YouTubeDownloader.Tools.LinkCreator/Tesses.YouTubeDownloader.Tools.LinkCreator.csproj @@ -0,0 +1,14 @@ + + + + + + + + Exe + net6.0 + enable + enable + + + diff --git a/Tesses.YouTubeDownloader.Server/Class1.cs b/Tesses.YouTubeDownloader.Server/Class1.cs index 11a2c68..adbf4f5 100644 --- a/Tesses.YouTubeDownloader.Server/Class1.cs +++ b/Tesses.YouTubeDownloader.Server/Class1.cs @@ -25,6 +25,79 @@ namespace Tesses.YouTubeDownloader.Server public override async Task GetAsync(ServerContext ctx) { string path=ctx.UrlAndQuery; + + if(path.StartsWith("/AddPlaylistRes/")) + { string id_res=path.Substring(16); + string[] id_res_split = id_res.Split(new char[] {'/'},2,StringSplitOptions.RemoveEmptyEntries); + if(id_res_split.Length ==2) + { + int num; + if(int.TryParse(id_res_split[0],out num)) + { + if(num < 0) num=1; + if(num > 3) num=1; + + await downloader1.AddPlaylistAsync(id_res_split[1],(Resolution)num); + } + } + } + if(path.StartsWith("/AddChannelRes/")) + { string id_res=path.Substring(15); + string[] id_res_split = id_res.Split(new char[] {'/'},2,StringSplitOptions.RemoveEmptyEntries); + if(id_res_split.Length ==2) + { + int num; + if(int.TryParse(id_res_split[0],out num)) + { + if(num < 0) num=1; + if(num > 3) num=1; + + await downloader1.AddChannelAsync(id_res_split[1],(Resolution)num); + } + } + } + + if(path.StartsWith("/AddChannel/")) + { + await downloader1.AddChannelAsync(path.Substring(12),Resolution.PreMuxed); + + } + if(path.StartsWith("/AddPlaylist/")) + { + await downloader1.AddPlaylistAsync(path.Substring(13),Resolution.PreMuxed); + + } + if(path.StartsWith("/AddVideoRes/")) + { string id_res=path.Substring(13); + string[] id_res_split = id_res.Split(new char[] {'/'},2,StringSplitOptions.RemoveEmptyEntries); + if(id_res_split.Length ==2) + { + int num; + if(int.TryParse(id_res_split[0],out num)) + { + if(num < 0) num=1; + if(num > 3) num=1; + + await downloader1.AddVideoAsync(id_res_split[1],(Resolution)num); + } + } + } + + if(path.StartsWith("/AddVideo/")) + { + //string id_res=path.Substring(12); + //string[] id_res_split = id_res.Split(new char[] {'/'},2,StringSplitOptions.RemoveEmptyEntries); + //if(id_res_split.Length ==2) + //{ + + await downloader1.AddVideoAsync(path.Substring(10),Resolution.PreMuxed); + + // } + // await ctx.SendTextAsync( + // $"

You Will Be Redirected in 5 Sec

\n" + //); + await ctx.SendRedirectAsync("/"); + } if(path.StartsWith("/AddItemRes/")) { string id_res=path.Substring(12); @@ -43,7 +116,7 @@ namespace Tesses.YouTubeDownloader.Server // await ctx.SendTextAsync( // $"

You Will Be Redirected in 5 Sec

\n" //); - await ctx.SendRedirectAsync("/"); + } if(path.StartsWith("/AddItem/")) { @@ -58,8 +131,59 @@ namespace Tesses.YouTubeDownloader.Server // await ctx.SendTextAsync( // $"

You Will Be Redirected in 5 Sec

\n" //); - await ctx.SendRedirectAsync("/"); + } + if(path.StartsWith("/AddUserRes/")) + { + string id_res=path.Substring(12); + string[] id_res_split = id_res.Split(new char[] {'/'},2,StringSplitOptions.RemoveEmptyEntries); + if(id_res_split.Length ==2) + { + int num; + if(int.TryParse(id_res_split[0],out num)) + { + if(num < 0) num=1; + if(num > 3) num=1; + + await downloader1.AddUserAsync(id_res_split[1],(Resolution)num); + } + } + // await ctx.SendTextAsync( + // $"

You Will Be Redirected in 5 Sec

\n" + //); + + } + if(path.StartsWith("/AddUser/")) + { + //string id_res=path.Substring(12); + //string[] id_res_split = id_res.Split(new char[] {'/'},2,StringSplitOptions.RemoveEmptyEntries); + //if(id_res_split.Length ==2) + //{ + + await downloader1.AddUserAsync(path.Substring(9),Resolution.PreMuxed); + + // } + // await ctx.SendTextAsync( + // $"

You Will Be Redirected in 5 Sec

\n" + //); + + } + if(path.StartsWith("/AddFile/")) + { + //string id_res=path.Substring(12); + //string[] id_res_split = id_res.Split(new char[] {'/'},2,StringSplitOptions.RemoveEmptyEntries); + //if(id_res_split.Length ==2) + //{ + + await downloader1.AddFileAsync(path.Substring(9)); + + // } + // await ctx.SendTextAsync( + // $"

You Will Be Redirected in 5 Sec

\n" + //); + + } + await ctx.SendRedirectAsync("/"); } } internal class ApiStorage : Tesses.WebServer.Server @@ -332,6 +456,7 @@ namespace Tesses.YouTubeDownloader.Server AddBoth("/AddUser",AddUser); AddBoth("/AddPlaylist",AddPlaylist); AddBoth("/AddVideo",AddVideo); + AddBoth("/AddFile",AddFile); AddBoth("/Progress",ProgressFunc); AddBoth("/QueueList",QueueList); AddBoth("/subscribe",Subscribe); @@ -641,6 +766,28 @@ namespace Tesses.YouTubeDownloader.Server public async Task ProgressFunc(ServerContext ctx) { await ctx.SendJsonAsync(Downloader.GetProgress()); + } + public async Task AddFile(ServerContext ctx) + { + string url; + string downloadStr; + bool download=true; + if(ctx.QueryParams.TryGetFirst("url",out url)) + { + if(ctx.QueryParams.TryGetFirst("download",out downloadStr)) + { + bool dl; + if(bool.TryParse(downloadStr,out dl)) + { + download=dl; + } + } + + await Downloader.AddFileAsync(url,download); + await ctx.SendTextAsync( + $"

You Will Be Redirected in 5 Sec

\n" + ); + } } public async Task AddVideo(ServerContext ctx) { diff --git a/Tesses.YouTubeDownloader.Server/Tesses.YouTubeDownloader.Server.csproj b/Tesses.YouTubeDownloader.Server/Tesses.YouTubeDownloader.Server.csproj index f2cbd3d..8ee06f7 100644 --- a/Tesses.YouTubeDownloader.Server/Tesses.YouTubeDownloader.Server.csproj +++ b/Tesses.YouTubeDownloader.Server/Tesses.YouTubeDownloader.Server.csproj @@ -5,7 +5,7 @@ - + @@ -15,9 +15,9 @@ Tesses.YouTubeDownloader.Server Mike Nolan Tesses - 1.1.3 - 1.1.3 - 1.1.3 + 1.1.4 + 1.1.4 + 1.1.4 Adds WebServer to TYTD LGPL-3.0-only true diff --git a/Tesses.YouTubeDownloader/B64.cs b/Tesses.YouTubeDownloader/B64.cs new file mode 100644 index 0000000..3887679 --- /dev/null +++ b/Tesses.YouTubeDownloader/B64.cs @@ -0,0 +1,58 @@ +using System; +namespace Tesses.YouTubeDownloader +{ +internal static class B64 +{ + public static string Base64UrlEncodes(string arg) + { + return Base64UrlEncode(System.Text.Encoding.UTF8.GetBytes(arg)); + } + + public static string Base64Encode(byte[] arg) + { + return Convert.ToBase64String(arg); + } + public static byte[] Base64Decode(string arg) + { + return Convert.FromBase64String(arg); + } + + public static string Base64Encodes(string arg) + { + return Base64Encode(System.Text.Encoding.UTF8.GetBytes(arg)); + } + + public static string Base64UrlEncode(byte[] arg) + { + string s = Convert.ToBase64String(arg); // Regular base64 encoder + s = s.Split('=')[0]; // Remove any trailing '='s + s = s.Replace('+', '-'); // 62nd char of encoding + s = s.Replace('/', '_'); // 63rd char of encoding + return s; + } + public static string Base64Decodes(string arg) + { + return System.Text.Encoding.UTF8.GetString(Base64Decode(arg)); + } + public static string Base64UrlDecodes(string arg) + { + return System.Text.Encoding.UTF8.GetString(Base64UrlDecode(arg)); + } + public static byte[] Base64UrlDecode(string arg) + { + string s = arg; + s = s.Replace('-', '+'); // 62nd char of encoding + s = s.Replace('_', '/'); // 63rd char of encoding + switch (s.Length % 4) // Pad with trailing '='s + { + case 0: break; // No pad chars in this case + case 2: s += "=="; break; // Two pad chars + case 3: s += "="; break; // One pad char + default: throw new System.Exception( + "Illegal base64url string!"); + } + return Convert.FromBase64String(s); // Standard base64 decoder + } + +} +} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader/BestStreams.cs b/Tesses.YouTubeDownloader/BestStreams.cs index e40a42b..622ebd8 100644 --- a/Tesses.YouTubeDownloader/BestStreams.cs +++ b/Tesses.YouTubeDownloader/BestStreams.cs @@ -32,7 +32,7 @@ namespace Tesses.YouTubeDownloader var f= await BestStreamInfo.GetBestStreams(storage,video.Id); if(f ==null) - return ""; + return resolution == Resolution.NoDownload ? "" : resolution == Resolution.Mux ? $"Mux/{video.Id}.mkv" : $"{TYTDManager.ResolutionToDirectory(resolution)}/{video.Id}.mp4"; if(f.VideoFrozen) { @@ -61,13 +61,12 @@ namespace Tesses.YouTubeDownloader public static async Task GetBestStreams(ITYTDBase storage,VideoId id) { //Console.WriteLine("IN FUNC"); - if(storage.DirectoryExists("StreamInfo")) - { + //Console.WriteLine("DIR"); - if(storage.FileExists($"StreamInfo/{id.Value}.json")) + if(storage.BestStreamInfoExists(id)) { //Console.WriteLine("STREAMS"); - BestStreamsSerialized serialization=JsonConvert.DeserializeObject(await storage.ReadAllTextAsync($"StreamInfo/{id.Value}.json")); + BestStreamsSerialized serialization=await storage.GetBestStreamInfoAsync(id); BestStreams streams=new BestStreams(); streams.VideoOnlyStreamInfo = new BestStreamInfo(serialization.VideoOnly); @@ -76,16 +75,16 @@ namespace Tesses.YouTubeDownloader return streams; } - } + return null; } public static async Task GetBestStreams(IStorage storage,VideoId id,CancellationToken token=default(CancellationToken),bool expire_check=true) { - if(storage.DirectoryExists("StreamInfo")) - { - if(storage.FileExists($"StreamInfo/{id.Value}.json")) + + + if(storage.BestStreamInfoExists(id)) { - BestStreamsSerialized serialization=JsonConvert.DeserializeObject(await storage.ReadAllTextAsync($"StreamInfo/{id.Value}.json")); + BestStreamsSerialized serialization=await storage.GetBestStreamInfoAsync(id); if(DateTime.Now < serialization.Expires || !expire_check) { BestStreams streams=new BestStreams(); @@ -95,9 +94,7 @@ namespace Tesses.YouTubeDownloader return streams; } } - }else{ - storage.CreateDirectory("StreamInfo"); - } + DateTime expires=DateTime.Now.AddHours(6); try{ if(storage.VideoInfoExists(id)) @@ -126,8 +123,7 @@ namespace Tesses.YouTubeDownloader streams1.MuxedStreamInfo =new BestStreamInfo(); streams1.MuxedStreamInfo.SetInfo(muxed); serialized.Muxed = streams1.MuxedStreamInfo.Serialization; - - await storage.WriteAllTextAsync($"StreamInfo/{id.Value}.json",JsonConvert.SerializeObject(serialized)); + await storage.WriteBestStreamInfoAsync(id,serialized); return streams1; }catch(YoutubeExplodeException ex) { @@ -135,7 +131,7 @@ namespace Tesses.YouTubeDownloader return null; } } - private class BestStreamsSerialized + public class BestStreamsSerialized { public DateTime Expires {get;set;} public BestStreamInfoSerialization VideoOnly {get;set;} @@ -202,7 +198,7 @@ namespace Tesses.YouTubeDownloader } } - internal class BestStreamInfoSerialization + public class BestStreamInfoSerialization { public string AudioCodec {get;set;} public int FrameRate {get;set;} diff --git a/Tesses.YouTubeDownloader/ConvertLegacy.cs b/Tesses.YouTubeDownloader/ConvertLegacy.cs index 7e72345..0dcef97 100644 --- a/Tesses.YouTubeDownloader/ConvertLegacy.cs +++ b/Tesses.YouTubeDownloader/ConvertLegacy.cs @@ -133,7 +133,7 @@ namespace Tesses.YouTubeDownloader int count = videos.Count; foreach(var v in videos) { - await storage1.WriteAllTextAsync($"Info/{v.Id}.json",JsonConvert.SerializeObject(v)); + await storage1.WriteVideoInfoAsync(v); } diff --git a/Tesses.YouTubeDownloader/DownloadLoop.cs b/Tesses.YouTubeDownloader/DownloadLoop.cs index adc776d..c788861 100644 --- a/Tesses.YouTubeDownloader/DownloadLoop.cs +++ b/Tesses.YouTubeDownloader/DownloadLoop.cs @@ -112,6 +112,8 @@ namespace Tesses.YouTubeDownloader } + + public async Task GetSavedVideoAsync(VideoId id) { VideoMediaContext context=new VideoMediaContext(id,Resolution.PreMuxed); @@ -128,6 +130,8 @@ namespace Tesses.YouTubeDownloader private async Task DownloadVideoAsync(SavedVideo video, Resolution resolution, CancellationToken token=default(CancellationToken),IProgress progress=null,bool report=true) { try{ + if(video.DownloadFrom == "YouTube") + { switch (resolution) { case Resolution.Mux: @@ -143,12 +147,103 @@ namespace Tesses.YouTubeDownloader await DownloadVideoOnlyAsync(video,token,progress,report); break; } + }else if(video.DownloadFrom.StartsWith("NormalDownload,Length=")) + { + await DownloadFileAsync(video,token,progress,report); + } }catch(Exception ex) { - await GetLogger().WriteAsync(ex,video.Id); + VideoId? id=VideoId.TryParse(video.Id); + if(id.HasValue){ + await GetLogger().WriteAsync(ex,id.Value); + }else{ + await GetLogger().WriteAsync(ex); + } } } + + private async Task DownloadFileAsync(SavedVideo video, CancellationToken token, IProgress progress, bool report) + { + string incomplete_file_path = $"Download/{B64.Base64UrlEncodes(video.Id)}-incomplete.part"; + string file_path = $"Download/{B64.Base64UrlEncodes(video.Id)}.bin"; + string url = video.Id; + bool canSeek=false; + long length=0; + + foreach(var kvp in video.DownloadFrom.Split(',').Select>((e)=>{ + if(!e.Contains('=')) return new KeyValuePair("",""); + string[] keyVP = e.Split(new char[]{'='},2); + + + return new KeyValuePair(keyVP[0],keyVP[1]);})) + { + switch(kvp.Key) + { + case "CanSeek": + bool.TryParse(kvp.Value,out canSeek); + break; + case "Length": + long len; + if(long.TryParse(kvp.Value,out len)) + { + length=len; + } + break; + } + } + await ReportStartVideo(video,Resolution.PreMuxed,length); + Func> openDownload = async(e)=>{ + HttpRequestMessage msg=new HttpRequestMessage(HttpMethod.Get,url); + if(e > 0) + { + msg.Headers.Range.Ranges.Add(new System.Net.Http.Headers.RangeItemHeaderValue(e,null)); + } + + var res=await HttpClient.SendAsync(msg); + return await res.Content.ReadAsStreamAsync(); + }; + + if(await Continue(file_path)) + { + bool deleteAndRestart=false; + + using(var file = await OpenOrCreateAsync(incomplete_file_path)) + { + if(file.Length > 0 && !canSeek) + { + deleteAndRestart = true; + } + if(!deleteAndRestart) + { + Stream strm=await openDownload(file.Length); + bool res=await CopyStreamAsync(strm,file,0,length,4096,progress,token); + if(res) + { + RenameFile(incomplete_file_path,file_path); + if(report) + await ReportEndVideo(video, Resolution.PreMuxed); + } + } + } + if(deleteAndRestart){ + DeleteFile(incomplete_file_path); + using(var file = await OpenOrCreateAsync(incomplete_file_path)) + { + Stream strm=await openDownload(0); + bool res=await CopyStreamAsync(strm,file,0,length,4096,progress,token); + if(res) + { + RenameFile(incomplete_file_path,file_path); + if(report) + await ReportEndVideo(video, Resolution.PreMuxed); + } + } + } + + } + } + public async Task Continue(string path) { @@ -282,7 +377,7 @@ namespace Tesses.YouTubeDownloader } curPos+=read; await dest.WriteAsync(buffer,0,read); - if(progress != null) + if(progress != null && len != 0) { progress.Report(curPos / len); } diff --git a/Tesses.YouTubeDownloader/IDownloader.cs b/Tesses.YouTubeDownloader/IDownloader.cs index b095357..966aba6 100644 --- a/Tesses.YouTubeDownloader/IDownloader.cs +++ b/Tesses.YouTubeDownloader/IDownloader.cs @@ -17,7 +17,7 @@ namespace Tesses.YouTubeDownloader Task AddChannelAsync(ChannelId id,Resolution resolution=Resolution.PreMuxed); Task AddUserAsync(UserName userName,Resolution resolution=Resolution.PreMuxed); - + Task AddFileAsync(string url,bool download=true); IReadOnlyList<(SavedVideo Video,Resolution Resolution)> GetQueueList(); SavedVideoProgress GetProgress(); IAsyncEnumerable GetSubscriptionsAsync(); @@ -26,5 +26,6 @@ namespace Tesses.YouTubeDownloader Task SubscribeAsync(UserName name,ChannelBellInfo info=ChannelBellInfo.NotifyAndDownload); Task ResubscribeAsync(ChannelId id,ChannelBellInfo info=ChannelBellInfo.NotifyAndDownload); void DeletePersonalPlaylist(string name); + } } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader/IStorage.cs b/Tesses.YouTubeDownloader/IStorage.cs index 79c688d..e56ac37 100644 --- a/Tesses.YouTubeDownloader/IStorage.cs +++ b/Tesses.YouTubeDownloader/IStorage.cs @@ -15,7 +15,7 @@ namespace Tesses.YouTubeDownloader { public interface IStorage : IWritable, IDownloader, ITYTDBase { - + Task WriteBestStreamInfoAsync(VideoId id,BestStreamInfo.BestStreamsSerialized serialized); Task MuxVideosAsync(SavedVideo video,string videoSrc,string audioSrc,string videoDest,IProgress progress=null,CancellationToken token=default(CancellationToken)); Task Continue(string path); Task WriteVideoInfoAsync(SavedVideo channel); diff --git a/Tesses.YouTubeDownloader/ITYTDBase.cs b/Tesses.YouTubeDownloader/ITYTDBase.cs index 890e622..3c9ae3a 100644 --- a/Tesses.YouTubeDownloader/ITYTDBase.cs +++ b/Tesses.YouTubeDownloader/ITYTDBase.cs @@ -15,7 +15,8 @@ namespace Tesses.YouTubeDownloader { public interface ITYTDBase : IPersonalPlaylistGet { - + Task GetBestStreamInfoAsync(VideoId id); + bool BestStreamInfoExists(VideoId id); IAsyncEnumerable GetPersonalPlaylistsAsync(); Task<(String Path,bool Delete)> GetRealUrlOrPathAsync(string path); diff --git a/Tesses.YouTubeDownloader/PreMediaContext.cs b/Tesses.YouTubeDownloader/PreMediaContext.cs index d9db14b..9d50124 100644 --- a/Tesses.YouTubeDownloader/PreMediaContext.cs +++ b/Tesses.YouTubeDownloader/PreMediaContext.cs @@ -37,13 +37,13 @@ namespace Tesses.YouTubeDownloader SavedChannel channel; if(Id.HasValue) //dont check for if(Id != null) hince I was looking for several minutes for the bug { - string path=$"Channel/{Id.Value}.json"; - if(await storage.Continue(path)) + //string path=$"Channel/{Id.Value}.json"; + if(!storage.ChannelInfoExists(Id.Value)) { try{ channel=await DownloadThumbnails(storage,await storage.YoutubeClient.Channels.GetAsync(Id.Value)); //channel=new SavedChannel(i); - await storage.WriteAllTextAsync(path,JsonConvert.SerializeObject(channel)); + await storage.WriteChannelInfoAsync(channel); }catch(Exception ex) { await storage.GetLogger().WriteAsync(ex); @@ -51,16 +51,16 @@ namespace Tesses.YouTubeDownloader } return channel; }else{ - var j=JsonConvert.DeserializeObject(await storage.ReadAllTextAsync(path)); + var j=await storage.GetChannelInfoAsync(Id.Value); return j; } }else{ var c=await storage.YoutubeClient.Channels.GetByUserAsync(name1); channel=await DownloadThumbnails(storage,c); - string path=$"Channel/{c.Id.Value}.json"; - if(await storage.Continue(path)) + //string path=$"Channel/{c.Id.Value}.json"; + if(!storage.ChannelInfoExists(c.Id.Value)) { - await storage.WriteAllTextAsync(path,JsonConvert.SerializeObject(channel)); + await storage.WriteChannelInfoAsync(channel); } return channel; @@ -118,7 +118,7 @@ namespace Tesses.YouTubeDownloader public async Task FillQueue(TYTDStorage storage, List<(SavedVideo video, Resolution resolution)> Queue) { - string path=$"Playlist/{Id}.json"; + // string path=$"Playlist/{Id}.json"; List videos=new List(); try{ @@ -134,7 +134,7 @@ namespace Tesses.YouTubeDownloader await cmc.GetChannel(storage); } - await storage.WriteAllTextAsync(path,JsonConvert.SerializeObject(p)); + await storage.WritePlaylistInfoAsync(p); }catch(Exception ex) { await storage.GetLogger().WriteAsync(ex); @@ -148,6 +148,76 @@ namespace Tesses.YouTubeDownloader } } + internal class NormalDownloadMediaContext : IMediaContext + { + public NormalDownloadMediaContext(string url,bool download=true) + { + this.url=url; + this.download=download; + } + bool download; + string url; + public async Task FillQueue(TYTDStorage storage, List<(SavedVideo video, Resolution resolution)> Queue) + { + + + SavedVideo video=new SavedVideo(); + if(storage.DownloadExists(url)){ + video = await storage.GetDownloadInfoAsync(url); + }else{ + video.Id = url; + + await GetFileNameAsync(storage,video); + } + lock(Queue){ + Queue.Add((video,Resolution.PreMuxed)); + } + } + private async Task GetFileNameAsync(TYTDStorage storage,SavedVideo video) + { + string[] uri0=url.Split(new char[]{'?'},2,StringSplitOptions.None); + string filename=Path.GetFileName(uri0[0]); + System.Net.Http.HttpRequestMessage message=new System.Net.Http.HttpRequestMessage(System.Net.Http.HttpMethod.Head,url); + message.Headers.Add("Range","bytes=0-"); + + var head=await storage.HttpClient.SendAsync(message); + if(head.Content.Headers.ContentDisposition != null && !string.IsNullOrWhiteSpace(head.Content.Headers.ContentDisposition.FileName)) + { + filename = head.Content.Headers.ContentDisposition.FileName; + } + long length = 0; + if(head.Content.Headers.ContentLength.HasValue) + { + length = head.Content.Headers.ContentLength.Value; + } + video.Title = filename; + var res=head.StatusCode == System.Net.HttpStatusCode.PartialContent ? "true" : "false"; + video.DownloadFrom=$"NormalDownload,Length={length},CanSeek={res}"; + video.AuthorTitle = "NotYouTube"; + video.AuthorChannelId = "TYTD_FILEDOWNLOAD"; + List hdrs=new List(); + foreach(var hdr in head.Content.Headers) + { + foreach(var item in hdr.Value){ + hdrs.Add($"{hdr.Key}: {item}"); + } + } + string headers=string.Join("\n",hdrs); + video.Description=$"File Download on \"{DateTime.Now.ToShortDateString()}\" at \"{DateTime.Now.ToShortTimeString()}\"\nHeaders:\n{headers}"; + video.Likes=42; + video.Dislikes=42; + video.Views=42; + video.Duration = new TimeSpan(0,0,0); + video.Keywords = new string[] {"FILE"}; + if(head.Headers.Date.HasValue) + { + video.UploadDate = head.Headers.Date.Value.DateTime; + } + + await storage.WriteVideoInfoAsync(video); + + } + } internal class VideoMediaContext : IMediaContext { VideoId Id; @@ -161,15 +231,15 @@ namespace Tesses.YouTubeDownloader } public async Task FillQueue(TYTDStorage storage,List<(SavedVideo,Resolution)> queue) { - string path=$"Info/{Id}.json"; + SavedVideo video; - if(await storage.Continue(path)) + if(!storage.VideoInfoExists(Id)) { try{ video = new SavedVideo(await storage.YoutubeClient.Videos.GetAsync(Id)); storage.SendBeforeSaveInfo(video); - await storage.WriteAllTextAsync(path,JsonConvert.SerializeObject(video)); + await storage.WriteVideoInfoAsync(video); await video.DownloadThumbnails(storage); }catch(Exception ex) { @@ -179,7 +249,7 @@ namespace Tesses.YouTubeDownloader } }else{ - video = JsonConvert.DeserializeObject(await storage.ReadAllTextAsync(path)); + video = await storage.GetVideoInfoAsync(Id); } if(storage.GetLoggerProperties().AlwaysDownloadChannel) { diff --git a/Tesses.YouTubeDownloader/TYTD.cs b/Tesses.YouTubeDownloader/TYTD.cs index 336ba48..9fb04ef 100644 --- a/Tesses.YouTubeDownloader/TYTD.cs +++ b/Tesses.YouTubeDownloader/TYTD.cs @@ -50,17 +50,23 @@ namespace Tesses.YouTubeDownloader public abstract void MoveDirectory(string src,string dest); public abstract void DeleteFile(string file); public abstract void DeleteDirectory(string dir,bool recursive=false); - - public async Task WriteVideoInfoAsync(SavedVideo info) + public virtual async Task WriteBestStreamInfoAsync(VideoId id,BestStreamInfo.BestStreamsSerialized serialized) { - string file = $"Info/{info.Id}.json"; + await WriteAllTextAsync($"StreamInfo/{id.Value}.json",JsonConvert.SerializeObject(serialized)); + + } + + public virtual async Task WriteVideoInfoAsync(SavedVideo info) + { + + string file = info.DownloadFrom.StartsWith("NormalDownload,Length=") ? $"FileInfo/{B64.Base64UrlEncodes(info.Id)}.json" : $"Info/{info.Id}.json"; if(!FileExists(file)) { await WriteAllTextAsync(file,JsonConvert.SerializeObject(info)); } } - public async Task WritePlaylistInfoAsync(SavedPlaylist info) + public virtual async Task WritePlaylistInfoAsync(SavedPlaylist info) { string file = $"Playlist/{info.Id}.json"; if(!FileExists(file)) @@ -68,7 +74,7 @@ namespace Tesses.YouTubeDownloader await WriteAllTextAsync(file,JsonConvert.SerializeObject(info)); } } - public async Task WriteChannelInfoAsync(SavedChannel info) + public virtual async Task WriteChannelInfoAsync(SavedChannel info) { string file = $"Channel/{info.Id}.json"; if(!FileExists(file)) @@ -111,6 +117,14 @@ namespace Tesses.YouTubeDownloader } await Task.FromResult(0); } + public async Task AddFileAsync(string url,bool download=true) + { + lock(Temporary) + { + Temporary.Add(new NormalDownloadMediaContext(url,download)); + } + await Task.FromResult(0); + } public void CreateDirectoryIfNotExist(string dir) { if(!DirectoryExists(dir)) @@ -151,6 +165,9 @@ namespace Tesses.YouTubeDownloader CreateDirectoryIfNotExist("Thumbnails"); CreateDirectoryIfNotExist("config"); CreateDirectoryIfNotExist("config/logs"); + CreateDirectoryIfNotExist("FileInfo"); + CreateDirectoryIfNotExist("Download"); + CreateDirectoryIfNotExist("StreamInfo"); } public void StartLoop(CancellationToken token = default(CancellationToken)) { @@ -179,7 +196,7 @@ namespace Tesses.YouTubeDownloader } } - public async Task AddToPersonalPlaylistAsync(string name, IEnumerable items) + public virtual async Task AddToPersonalPlaylistAsync(string name, IEnumerable items) { List items0=new List(); await foreach(var item in GetPersonalPlaylistContentsAsync(name)) @@ -191,18 +208,18 @@ namespace Tesses.YouTubeDownloader } - public async Task ReplacePersonalPlaylistAsync(string name, IEnumerable items) + public virtual async Task ReplacePersonalPlaylistAsync(string name, IEnumerable items) { await WriteAllTextAsync($"PersonalPlaylist/{name}.json",JsonConvert.SerializeObject(items.ToList())); } - public void DeletePersonalPlaylist(string name) + public virtual void DeletePersonalPlaylist(string name) { DeleteFile($"PersonalPlaylist/{name}.json"); } - public async Task RemoveItemFromPersonalPlaylistAsync(string name, VideoId id) + public virtual async Task RemoveItemFromPersonalPlaylistAsync(string name, VideoId id) { List items0=new List(); await foreach(var item in GetPersonalPlaylistContentsAsync(name)) @@ -217,7 +234,7 @@ namespace Tesses.YouTubeDownloader } - public async Task SetResolutionForItemInPersonalPlaylistAsync(string name, VideoId id, Resolution resolution) + public virtual async Task SetResolutionForItemInPersonalPlaylistAsync(string name, VideoId id, Resolution resolution) { List items0=new List(); await foreach(var item in GetPersonalPlaylistContentsAsync(name)) diff --git a/Tesses.YouTubeDownloader/TYTDBase.cs b/Tesses.YouTubeDownloader/TYTDBase.cs index a011c75..1487322 100644 --- a/Tesses.YouTubeDownloader/TYTDBase.cs +++ b/Tesses.YouTubeDownloader/TYTDBase.cs @@ -17,11 +17,11 @@ namespace Tesses.YouTubeDownloader public abstract class TYTDBase : ITYTDBase { - public bool PersonalPlaylistExists(string name) + public virtual bool PersonalPlaylistExists(string name) { return FileExists($"PersonalPlaylist/{name}.json"); } - public async IAsyncEnumerable GetPersonalPlaylistContentsAsync(string playlist) + public virtual async IAsyncEnumerable GetPersonalPlaylistContentsAsync(string playlist) { if(!PersonalPlaylistExists(playlist)) yield break; var ls=JsonConvert.DeserializeObject>(await ReadAllTextAsync($"PersonalPlaylist/{playlist}.json")); @@ -30,7 +30,7 @@ namespace Tesses.YouTubeDownloader yield return await Task.FromResult(item); } } - public async IAsyncEnumerable GetPersonalPlaylistsAsync() + public virtual async IAsyncEnumerable GetPersonalPlaylistsAsync() { await foreach(var item in EnumerateFilesAsync("PersonalPlaylist")) { @@ -62,7 +62,7 @@ namespace Tesses.YouTubeDownloader return FileExistsAsync(path).GetAwaiter().GetResult(); } - public async IAsyncEnumerable GetVideoIdsAsync() + public virtual async IAsyncEnumerable GetVideoIdsAsync() { await foreach(var item in EnumerateFilesAsync("Info")) { @@ -73,13 +73,13 @@ namespace Tesses.YouTubeDownloader } } - public async Task GetVideoInfoAsync(VideoId id) + public virtual async Task GetVideoInfoAsync(VideoId id) { return JsonConvert.DeserializeObject(await ReadAllTextAsync($"Info/{id}.json")); } - public async IAsyncEnumerable GetVideosAsync() + public virtual async IAsyncEnumerable GetVideosAsync() { await foreach(var item in GetVideoIdsAsync()) { @@ -90,7 +90,7 @@ namespace Tesses.YouTubeDownloader } } } - public async IAsyncEnumerable GetLegacyVideosAsync() + public virtual async IAsyncEnumerable GetLegacyVideosAsync() { await foreach(var item in GetVideoIdsAsync()) { @@ -101,11 +101,11 @@ namespace Tesses.YouTubeDownloader } } } - public async Task GetLegacyVideoInfoAsync(VideoId id) + public virtual async Task GetLegacyVideoInfoAsync(VideoId id) { return JsonConvert.DeserializeObject(await ReadAllTextAsync($"Info/{id}.json")); } - public async IAsyncEnumerable GetPlaylistsAsync() + public virtual async IAsyncEnumerable GetPlaylistsAsync() { await foreach(var item in GetPlaylistIdsAsync()) { @@ -131,7 +131,7 @@ namespace Tesses.YouTubeDownloader } } - public async IAsyncEnumerable GetPlaylistIdsAsync() + public virtual async IAsyncEnumerable GetPlaylistIdsAsync() { await foreach(var item in EnumerateFilesAsync("Playlist")) { @@ -141,7 +141,7 @@ namespace Tesses.YouTubeDownloader } } } - public async IAsyncEnumerable GetChannelIdsAsync() + public virtual async IAsyncEnumerable GetChannelIdsAsync() { await foreach(var item in EnumerateFilesAsync("Channel")) { @@ -151,7 +151,7 @@ namespace Tesses.YouTubeDownloader } } } - public async IAsyncEnumerable GetYouTubeExplodeVideoIdsAsync() + public virtual async IAsyncEnumerable GetYouTubeExplodeVideoIdsAsync() { await foreach(var item in GetVideoIdsAsync()) { @@ -159,11 +159,11 @@ namespace Tesses.YouTubeDownloader if(id.HasValue) yield return id.Value; } } - public async Task GetChannelInfoAsync(ChannelId id) + public virtual async Task GetChannelInfoAsync(ChannelId id) { return JsonConvert.DeserializeObject(await ReadAllTextAsync($"Channel/{id}.json")); } - public async IAsyncEnumerable GetChannelsAsync() + public virtual async IAsyncEnumerable GetChannelsAsync() { await foreach(var item in GetChannelIdsAsync()) { @@ -175,24 +175,60 @@ namespace Tesses.YouTubeDownloader } } - public bool PlaylistInfoExists(PlaylistId id) + public virtual async IAsyncEnumerable GetDownloadsAsync() + { + await foreach(var item in GetDownloadUrlsAsync()) + { + yield return await GetDownloadInfoAsync(item); + } + } + public virtual async IAsyncEnumerable GetDownloadUrlsAsync() + { + await foreach(var item in EnumerateFilesAsync("FileInfo")) + { + if(Path.GetExtension(item).Equals(".json",StringComparison.Ordinal)) + { + yield return B64.Base64UrlDecodes(Path.GetFileNameWithoutExtension(item)); + } + } + } + public virtual async Task GetBestStreamInfoAsync(VideoId id) + { + return JsonConvert.DeserializeObject(await ReadAllTextAsync($"StreamInfo/{id.Value}.json")); + } + public virtual bool BestStreamInfoExists(VideoId id) + { + return FileExists($"StreamInfo/{id.Value}.json"); + } + public virtual async Task GetDownloadInfoAsync(string url) + { + string enc=$"FileInfo/{B64.Base64UrlEncodes(url)}.json"; + return JsonConvert.DeserializeObject(await ReadAllTextAsync(enc)); + + } + public virtual bool DownloadExists(string url) + { + string enc=$"FileInfo/{B64.Base64UrlEncodes(url)}.json"; + return FileExists(enc); + } + public virtual bool PlaylistInfoExists(PlaylistId id) { return FileExists($"Playlist/{id}.json"); } - public bool VideoInfoExists(VideoId id) + public virtual bool VideoInfoExists(VideoId id) { return FileExists($"Info/{id}.json"); } - public bool ChannelInfoExists(ChannelId id) + public virtual bool ChannelInfoExists(ChannelId id) { return FileExists($"Channel/{id}.json"); } - public async Task GetPlaylistInfoAsync(PlaylistId id) + public virtual async Task GetPlaylistInfoAsync(PlaylistId id) { return JsonConvert.DeserializeObject(await ReadAllTextAsync($"Playlist/{id}.json")); } - public async Task ReadAllTextAsync(string file) + public virtual async Task ReadAllTextAsync(string file) { using(var s = await OpenReadAsync(file)) { @@ -203,12 +239,12 @@ namespace Tesses.YouTubeDownloader } } - public bool DirectoryExists(string path) + public virtual bool DirectoryExists(string path) { return DirectoryExistsAsync(path).GetAwaiter().GetResult(); } - public IEnumerable EnumerateFiles(string path) + public virtual IEnumerable EnumerateFiles(string path) { var e = EnumerateFilesAsync(path).GetAsyncEnumerator(); while(e.MoveNextAsync().GetAwaiter().GetResult()) @@ -216,7 +252,7 @@ namespace Tesses.YouTubeDownloader yield return e.Current; } } - public IEnumerable EnumerateDirectories(string path) + public virtual IEnumerable EnumerateDirectories(string path) { var e = EnumerateDirectoriesAsync(path).GetAsyncEnumerator(); while(e.MoveNextAsync().GetAwaiter().GetResult()) @@ -748,7 +784,7 @@ namespace Tesses.YouTubeDownloader public interface IWritable : IPersonalPlaylistGet, IPersonalPlaylistSet { public Task WriteAllTextAsync(string path,string data); - } + } } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader/TYTDClient.cs b/Tesses.YouTubeDownloader/TYTDClient.cs index 2dcf0ca..1d7694a 100644 --- a/Tesses.YouTubeDownloader/TYTDClient.cs +++ b/Tesses.YouTubeDownloader/TYTDClient.cs @@ -256,7 +256,15 @@ internal class SegmentedHttpStream : Stream _=ex; } } - + public async Task AddFileAsync(string url,bool download=true) + { + try{ + await client.GetStringAsync($"{url}api/v2/AddFile?url={WebUtility.UrlEncode(url)}&download={download}"); + }catch(Exception ex) + { + _=ex; + } + } public override async Task DirectoryExistsAsync(string path) { try{ diff --git a/Tesses.YouTubeDownloader/TYTDIDownloaderStorageProxy.cs b/Tesses.YouTubeDownloader/TYTDIDownloaderStorageProxy.cs index f8a29d8..e8b91af 100644 --- a/Tesses.YouTubeDownloader/TYTDIDownloaderStorageProxy.cs +++ b/Tesses.YouTubeDownloader/TYTDIDownloaderStorageProxy.cs @@ -346,6 +346,11 @@ namespace Tesses.YouTubeDownloader if(Downloader != null) await Downloader.AddVideoAsync(id,resolution); } + public async Task AddFileAsync(string url,bool download=true) + { + if(Downloader != null) + await Downloader.AddFileAsync(url,download); + } public async Task AddPlaylistAsync(PlaylistId id, Resolution resolution = Resolution.PreMuxed) { @@ -660,6 +665,33 @@ namespace Tesses.YouTubeDownloader e.DeletePersonalPlaylist(name); }); } + + + public async Task WriteBestStreamInfoAsync(VideoId id, BestStreamInfo.BestStreamsSerialized serialized) + { + await StorageAsStorageAsync(async(e)=>{ + await e.WriteBestStreamInfoAsync(id,serialized); + }); + } + + public async Task GetBestStreamInfoAsync(VideoId id) + { + BestStreamInfo.BestStreamsSerialized s=null; + await StorageAsStorageAsync(async(e)=>{ + s=await e.GetBestStreamInfoAsync(id); + }); + + return s; + } + + public bool BestStreamInfoExists(VideoId id) + { + bool res=false; + StorageAsStorage((e)=>{ + res=e.BestStreamInfoExists(id); + }); + return res; + } } public class DownloaderMigration diff --git a/Tesses.YouTubeDownloader/Tesses.YouTubeDownloader.csproj b/Tesses.YouTubeDownloader/Tesses.YouTubeDownloader.csproj index c8cc549..c62ed09 100644 --- a/Tesses.YouTubeDownloader/Tesses.YouTubeDownloader.csproj +++ b/Tesses.YouTubeDownloader/Tesses.YouTubeDownloader.csproj @@ -7,9 +7,9 @@ Tesses.YouTubeDownloader Mike Nolan Tesses - 1.1.5 - 1.1.5 - 1.1.5 + 1.1.6 + 1.1.6 + 1.1.6 A YouTube Downloader LGPL-3.0-only true