tytd/Tesses.YouTubeDownloader/TYTD.cs

475 lines
18 KiB
C#

using System;
using YoutubeExplode;
using YoutubeExplode.Videos;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Net.Http;
using System.IO;
using YoutubeExplode.Playlists;
using YoutubeExplode.Channels;
using Newtonsoft.Json;
using System.Text;
namespace Tesses.YouTubeDownloader
{
public abstract partial class TYTDStorage : TYTDBase, IStorage
{
public new virtual async Task<byte[]> ReadThumbnailAsync(VideoId videoId,string res,CancellationToken token=default)
{
CreateDirectoryIfNotExist($"Thumbnails/{videoId.Value}");
if(await ThumbnailExistsAsync(videoId,res))
{
return await ReadAllBytesAsync($"Thumbnails/{videoId.Value}/{res}.jpg",token);
}else{
var result= await HttpClient.GetByteArrayAsync($"https://s.ytimg.com/vi/{videoId.Value}/{res}.jpg");
await WriteThumbnailAsync(videoId,res,result,token);
return result;
}
}
public virtual async Task WriteThumbnailAsync(VideoId videoId,string res,byte[] data,CancellationToken token=default)
{
CreateDirectoryIfNotExist($"Thumbnails/{videoId.Value}");
await WriteAllBytesAsync($"Thumbnails/{videoId.Value}/{res}.jpg",data,token);
}
public override ExtraData GetExtraData()
{
ExtraData data=new ExtraData();
lock(Temporary){
data.ItemsInInfoQueue=Temporary.Count;
}
lock(QueueList)
{
data.ItemsInQueue=QueueList.Count;
}
return data;
}
internal class ConsoleWriterCLS : TextWriter
{
Action<string> cls;
public ConsoleWriterCLS(Action<string> cls)
{
this.cls=cls;
}
public override Encoding Encoding => Encoding.UTF8;
public override void Write(string value)
{
cls(value);
}
}
private static readonly HttpClient _default = new HttpClient();
public abstract Task<Stream> CreateAsync(string path);
public abstract void CreateDirectory(string path);
public static bool UseConsole = true;
public TYTDStorage(HttpClient clt)
{
HttpClient=clt;
YoutubeClient=new YoutubeClient(HttpClient);
ExtensionContext=null;
ConsoleWriter=new ConsoleWriterCLS((e)=>{
ConsoleWrite?.Invoke(this,new ConsoleWriteEventArgs(e));
});
}
public TYTDStorage()
{
HttpClient=_default;
YoutubeClient=new YoutubeClient(HttpClient);
ExtensionContext=null;
ConsoleWriter=new ConsoleWriterCLS((e)=>{
ConsoleWrite?.Invoke(this,new ConsoleWriteEventArgs(e));
});
}
public async Task WriteAllBytesAsync(string path,byte[] data,CancellationToken token=default(CancellationToken))
{
MemoryStream ms = new MemoryStream(data);
using(var s=await CreateAsync(path))
{
await ms.CopyToAsync(s,4096,token);
}
}
public EventHandler<ConsoleWriteEventArgs> ConsoleWrite;
public TextWriter ConsoleWriter {get; private set;}
public static string TYTDTag {get {return _tytd_tag;}}
public static void SetTYTDTag(string tag)
{
//for use on mobile phones
_tytd_tag= tag;
}
private static string _tytd_tag=_getTYTDTag();
private static string _getTYTDTag()
{
string tag=Environment.GetEnvironmentVariable("TYTD_TAG");
if(string.IsNullOrWhiteSpace(tag)) return "UnknownPC";
return tag;
}
bool can_download=true;
public bool CanDownload {get {return can_download;} set {can_download=value;}}
public IExtensionContext ExtensionContext {get;set;}
public HttpClient HttpClient {get;set;}
public YoutubeClient YoutubeClient {get;set;}
public abstract void MoveDirectory(string src,string dest);
public abstract void DeleteFile(string file);
public abstract void DeleteDirectory(string dir,bool recursive=false);
public virtual async Task WriteBestStreamInfoAsync(VideoId id,BestStreamInfo.BestStreamsSerialized serialized)
{
await WriteAllTextAsync($"StreamInfo/{id.Value}.json",JsonConvert.SerializeObject(serialized));
}
public virtual async Task WriteVideoInfoAsync(SavedVideo info)
{
string file = info.DownloadFrom.StartsWith("NormalDownload,Length=") ? $"DownloadsInfo/{HashDownloadUrl(info.Id)}.json" : $"Info/{info.Id}.json";
if(!await FileExistsAsync(file))
{
await WriteAllTextAsync(file,JsonConvert.SerializeObject(info));
}
}
public virtual async Task WritePlaylistInfoAsync(SavedPlaylist info)
{
string file = $"Playlist/{info.Id}.json";
if(!FileExists(file))
{
await WriteAllTextAsync(file,JsonConvert.SerializeObject(info));
}
}
public virtual async Task WriteChannelInfoAsync(SavedChannel info)
{
string file = $"Channel/{info.Id}.json";
if(!await FileExistsAsync(file))
{
await WriteAllTextAsync(file,JsonConvert.SerializeObject(info));
}
}
public async Task AddPlaylistAsync(PlaylistId id,Resolution resolution=Resolution.PreMuxed)
{
lock(Temporary)
{
GetLogger().WriteVideoLog($"https://www.youtube.com/playlist?list={id.Value}");
Temporary.Add( new PlaylistMediaContext(id,resolution));
}
await Task.FromResult(0);
}
public async Task AddChannelAsync(ChannelId id,Resolution resolution=Resolution.PreMuxed)
{
lock(Temporary)
{
GetLogger().WriteVideoLog($"https://www.youtube.com/channel/{id.Value}");
Temporary.Add(new ChannelMediaContext(id,resolution));
}
await Task.FromResult(0);
}
public async Task AddSlugAsync(ChannelSlug slug,Resolution resolution=Resolution.PreMuxed)
{
lock(Temporary)
{
GetLogger().WriteVideoLog($"https://www.youtube.com/c/{slug.Value}");
Temporary.Add(new ChannelMediaContext(slug,resolution));
}
await Task.FromResult(0);
}
public async Task AddHandleAsync(ChannelHandle handle,Resolution resolution=Resolution.PreMuxed)
{
lock(Temporary)
{
GetLogger().WriteVideoLog($"https://www.youtube.com/@{handle.Value}");
Temporary.Add(new ChannelMediaContext(handle,resolution));
}
await Task.FromResult(0);
}
public async Task AddUserAsync(UserName name,Resolution resolution=Resolution.PreMuxed)
{
lock(Temporary)
{
GetLogger().WriteVideoLog($"https://www.youtube.com/user/{name.Value}");
Temporary.Add(new ChannelMediaContext(name,resolution));
}
await Task.FromResult(0);
}
public async Task AddVideoAsync(VideoId videoId,Resolution res=Resolution.PreMuxed)
{
lock(Temporary)
{
GetLogger().WriteVideoLog($"https://www.youtube.com/watch?v={videoId.Value}");
Temporary.Add(new VideoMediaContext(videoId,res));
}
await Task.FromResult(0);
}
public async Task AddFileAsync(string url,bool download=true)
{
lock(Temporary)
{
GetLogger().WriteVideoLog(url);
Temporary.Add(new NormalDownloadMediaContext(url,download));
}
await Task.FromResult(0);
}
public void CreateDirectoryIfNotExist(string dir)
{
if(!DirectoryExists(dir))
{
CreateDirectory(dir);
}
}
public async Task DownloadThumbnails(VideoId id)
{
if(!can_download) return;
string Id=id.Value;
string[] res=new string[] {"default.jpg","sddefault.jpg","mqdefault.jpg","hqdefault.jpg","maxresdefault.jpg"};
CreateDirectoryIfNotExist($"Thumbnails/{Id}");
foreach(var reso in res)
{
if(await Continue($"Thumbnails/{Id}/{reso}"))
{
try{
var data=await HttpClient.GetByteArrayAsync($"https://s.ytimg.com/vi/{Id}/{reso}");
await WriteAllBytesAsync($"Thumbnails/{Id}/{reso}",data);
}catch(Exception ex)
{
_=ex;
}
}
}
}
public void CreateDirectories()
{
CreateDirectoryIfNotExist("Channel");
CreateDirectoryIfNotExist("Playlist");
CreateDirectoryIfNotExist("Subscriptions");
CreateDirectoryIfNotExist("VideoOnly");
CreateDirectoryIfNotExist("AudioOnly");
CreateDirectoryIfNotExist("Muxed");
CreateDirectoryIfNotExist("PreMuxed");
CreateDirectoryIfNotExist("Info");
CreateDirectoryIfNotExist("Thumbnails");
CreateDirectoryIfNotExist("config");
CreateDirectoryIfNotExist("config/logs");
CreateDirectoryIfNotExist("config/addlog");
CreateDirectoryIfNotExist("DownloadsInfo");
CreateDirectoryIfNotExist("Downloads");
CreateDirectoryIfNotExist("StreamInfo");
CreateDirectoryIfNotExist("PersonalPlaylist");
}
public void StartLoop(CancellationToken token = default(CancellationToken))
{
CreateDirectories();
if(DirectoryExists("Download") && DirectoryExists("FileInfo"))
Task.Run(MigrateDownloads).Wait();
Thread thread0=new Thread(()=>{
DownloadLoop(token).Wait();
});
thread0.Start();
Thread thread1=new Thread(()=>{
QueueLoop(token).Wait();
});
thread1.Start();
}
private async IAsyncEnumerable<SavedVideo> GetDownloadsLegacyAsync()
{
await foreach(var item in EnumerateFilesAsync("FileInfo"))
{
if(Path.GetExtension(item).Equals(".json",StringComparison.Ordinal))
{
var res= JsonConvert.DeserializeObject<SavedVideo>(await ReadAllTextAsync(item));
DeleteFile(item);
yield return res;
}
}
}
private async Task MigrateDownloads()
{
await GetLogger().WriteAsync("Migrating File Downloads (Please Don't close TYTD)",true);
await foreach(var dl in GetDownloadsLegacyAsync())
{
await MoveLegacyDownload(dl);
}
int files=0;
await foreach(var f in EnumerateDirectoriesAsync("FileInfo"))
{
files++;
break;
}
if(files==0)
{
await foreach(var f in EnumerateFilesAsync("FileInfo"))
{
files++;
break;
}
}
if(files>0)
{
MoveDirectory("FileInfo","DownloadsInfoLegacy");
await GetLogger().WriteAsync("WARNING: there were still files/folders in FileInfo so they are stored in DownloadsInfoLegacy",true);
}else{
DeleteDirectory("FileInfo");
}
files=0;
await foreach(var f in EnumerateDirectoriesAsync("Download"))
{
files++;
break;
}
if(files==0)
{
await foreach(var f in EnumerateFilesAsync("Download"))
{
files++;
break;
}
}
if(files>0)
{
MoveDirectory("Download","DownloadsLegacy");
await GetLogger().WriteAsync("WARNING: there were still files/folders in Download so they are stored in DownloadsLegacy",true);
}else{
DeleteDirectory("Download");
}
await GetLogger().WriteAsync("Migrating Downloads Complete",true);
}
private async Task MoveLegacyDownload(SavedVideo dl)
{
await WriteVideoInfoAsync(dl);
string old_incomplete_file_path = $"Download/{B64.Base64UrlEncodes(dl.Id)}-incomplete.part";
string old_file_path = $"Download/{B64.Base64UrlEncodes(dl.Id)}.bin";
string incomplete_file_path = $"Downloads/{HashDownloadUrl(dl.Id)}-incomplete.part";
string file_path = $"Downloads/{HashDownloadUrl(dl.Id)}.bin";
bool complete = FileExists(old_file_path);
bool missing = !complete && !FileExists(old_incomplete_file_path);
string alreadyStr ="";
if(!missing)
{
if(complete)
{
//migrate complete
if(!FileExists(file_path))
{
RenameFile(old_file_path,file_path);
}else{
alreadyStr="Already ";
}
}
else
{
//migrate incomplete
if(!FileExists(incomplete_file_path))
{
RenameFile(old_incomplete_file_path,incomplete_file_path);
}else{
alreadyStr="Already ";
}
}
}
await GetLogger().WriteAsync($"{alreadyStr}Migrated {(missing? "missing" : (complete ? "complete" : "incomplete"))} download: {dl.Title} with Url: {dl.Id}",true);
}
internal void ThrowError(TYTDErrorEventArgs e)
{
Error?.Invoke(this,e);
}
public async Task WriteAllTextAsync(string path,string data)
{
using(var dstStrm= await CreateAsync(path))
{
using(var sw = new StreamWriter(dstStrm))
{
await sw.WriteAsync(data);
}
}
}
public virtual async Task AddToPersonalPlaylistAsync(string name, IEnumerable<ListContentItem> items)
{
List<ListContentItem> items0=new List<ListContentItem>();
await foreach(var item in GetPersonalPlaylistContentsAsync(name))
{
items0.Add(item);
}
items0.AddRange(items);
await WriteAllTextAsync($"PersonalPlaylist/{name}.json",JsonConvert.SerializeObject(items0));
}
public virtual async Task ReplacePersonalPlaylistAsync(string name, IEnumerable<ListContentItem> items)
{
await WriteAllTextAsync($"PersonalPlaylist/{name}.json",JsonConvert.SerializeObject(items.ToList()));
}
public virtual void DeletePersonalPlaylist(string name)
{
DeleteFile($"PersonalPlaylist/{name}.json");
}
public virtual async Task RemoveItemFromPersonalPlaylistAsync(string name, VideoId id)
{
List<ListContentItem> items0=new List<ListContentItem>();
await foreach(var item in GetPersonalPlaylistContentsAsync(name))
{
if(item.Id != id)
{
items0.Add(item);
}
}
await WriteAllTextAsync($"PersonalPlaylist/{name}.json",JsonConvert.SerializeObject(items0));
}
public virtual async Task SetResolutionForItemInPersonalPlaylistAsync(string name, VideoId id, Resolution resolution)
{
List<ListContentItem> items0=new List<ListContentItem>();
await foreach(var item in GetPersonalPlaylistContentsAsync(name))
{
if(item.Id != id)
{
items0.Add(item);
}else{
items0.Add(new ListContentItem(item.Id,resolution));
}
}
await WriteAllTextAsync($"PersonalPlaylist/{name}.json",JsonConvert.SerializeObject(items0));
}
public void RegisterVideoStarted(EventHandler<VideoStartedEventArgs> vs)
{
VideoStarted+=vs;
}
public void RegisterVideoFinished(EventHandler<VideoFinishedEventArgs> vf)
{
VideoFinished+=vf;
}
public void RegisterVideoProgress(EventHandler<VideoProgressEventArgs> vp)
{
VideoProgress+=vp;
}
}
}