using System; using YoutubeExplode; using YoutubeExplode.Videos; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Net.Http; using System.IO; using YoutubeExplode.Playlists; using YoutubeExplode.Channels; using Newtonsoft.Json; using System.Text; namespace Tesses.YouTubeDownloader { public abstract partial class TYTDStorage : TYTDBase, IStorage { public new virtual async Task ReadThumbnailAsync(VideoId videoId,string res,CancellationToken token=default) { CreateDirectoryIfNotExist($"Thumbnails/{videoId.Value}"); if(await ThumbnailExistsAsync(videoId,res)) { return await ReadAllBytesAsync($"Thumbnails/{videoId.Value}/{res}.jpg",token); }else{ var result= await HttpClient.GetByteArrayAsync($"https://s.ytimg.com/vi/{videoId.Value}/{res}.jpg"); await WriteThumbnailAsync(videoId,res,result,token); return result; } } public virtual async Task WriteThumbnailAsync(VideoId videoId,string res,byte[] data,CancellationToken token=default) { CreateDirectoryIfNotExist($"Thumbnails/{videoId.Value}"); await WriteAllBytesAsync($"Thumbnails/{videoId.Value}/{res}.jpg",data,token); } public override ExtraData GetExtraData() { ExtraData data=new ExtraData(); lock(Temporary){ data.ItemsInInfoQueue=Temporary.Count; } lock(QueueList) { data.ItemsInQueue=QueueList.Count; } return data; } internal class ConsoleWriterCLS : TextWriter { Action cls; public ConsoleWriterCLS(Action cls) { this.cls=cls; } public override Encoding Encoding => Encoding.UTF8; public override void Write(string value) { cls(value); } } private static readonly HttpClient _default = new HttpClient(); public abstract Task CreateAsync(string path); public abstract void CreateDirectory(string path); public static bool UseConsole = true; public TYTDStorage(HttpClient clt) { HttpClient=clt; YoutubeClient=new YoutubeClient(HttpClient); ExtensionContext=null; ConsoleWriter=new ConsoleWriterCLS((e)=>{ ConsoleWrite?.Invoke(this,new ConsoleWriteEventArgs(e)); }); } public TYTDStorage() { HttpClient=_default; YoutubeClient=new YoutubeClient(HttpClient); ExtensionContext=null; ConsoleWriter=new ConsoleWriterCLS((e)=>{ ConsoleWrite?.Invoke(this,new ConsoleWriteEventArgs(e)); }); } public async Task WriteAllBytesAsync(string path,byte[] data,CancellationToken token=default(CancellationToken)) { using(var s=await CreateAsync(path)) { await s.WriteAsync(data,0,data.Length,token); } } public EventHandler ConsoleWrite; public TextWriter ConsoleWriter {get; private set;} public static string TYTDTag {get {return _tytd_tag;}} public static void SetTYTDTag(string tag) { //for use on mobile phones _tytd_tag= tag; } private static string _tytd_tag=_getTYTDTag(); private static string _getTYTDTag() { string tag=Environment.GetEnvironmentVariable("TYTD_TAG"); if(string.IsNullOrWhiteSpace(tag)) return "UnknownPC"; return tag; } bool can_download=true; public bool CanDownload {get {return can_download;} set {can_download=value;}} public IExtensionContext ExtensionContext {get;set;} public HttpClient HttpClient {get;set;} public YoutubeClient YoutubeClient {get;set;} public abstract void MoveDirectory(string src,string dest); public abstract void DeleteFile(string file); public abstract void DeleteDirectory(string dir,bool recursive=false); public virtual async Task WriteBestStreamInfoAsync(VideoId id,BestStreamInfo.BestStreamsSerialized serialized) { await WriteAllTextAsync($"StreamInfo/{id.Value}.json",JsonConvert.SerializeObject(serialized)); } public virtual async Task WriteVideoInfoAsync(SavedVideo info) { string file = info.DownloadFrom.StartsWith("NormalDownload,Length=") ? $"DownloadsInfo/{HashDownloadUrl(info.Id)}.json" : $"Info/{info.Id}.json"; if(!await FileExistsAsync(file)) { await WriteAllTextAsync(file,JsonConvert.SerializeObject(info)); } } public virtual async Task WritePlaylistInfoAsync(SavedPlaylist info) { string file = $"Playlist/{info.Id}.json"; if(!FileExists(file)) { await WriteAllTextAsync(file,JsonConvert.SerializeObject(info)); } } public virtual async Task WriteChannelInfoAsync(SavedChannel info) { string file = $"Channel/{info.Id}.json"; if(!await FileExistsAsync(file)) { await WriteAllTextAsync(file,JsonConvert.SerializeObject(info)); } } public async Task AddPlaylistAsync(PlaylistId id,Resolution resolution=Resolution.PreMuxed) { lock(Temporary) { GetLogger().WriteVideoLog($"https://www.youtube.com/playlist?list={id.Value}"); Temporary.Add( new PlaylistMediaContext(id,resolution)); } await Task.FromResult(0); } public async Task AddChannelAsync(ChannelId id,Resolution resolution=Resolution.PreMuxed) { lock(Temporary) { GetLogger().WriteVideoLog($"https://www.youtube.com/channel/{id.Value}"); Temporary.Add(new ChannelMediaContext(id,resolution)); } await Task.FromResult(0); } public async Task AddSlugAsync(ChannelSlug slug,Resolution resolution=Resolution.PreMuxed) { lock(Temporary) { GetLogger().WriteVideoLog($"https://www.youtube.com/c/{slug.Value}"); Temporary.Add(new ChannelMediaContext(slug,resolution)); } await Task.FromResult(0); } public async Task AddHandleAsync(ChannelHandle handle,Resolution resolution=Resolution.PreMuxed) { lock(Temporary) { GetLogger().WriteVideoLog($"https://www.youtube.com/@{handle.Value}"); Temporary.Add(new ChannelMediaContext(handle,resolution)); } await Task.FromResult(0); } public async Task AddUserAsync(UserName name,Resolution resolution=Resolution.PreMuxed) { lock(Temporary) { GetLogger().WriteVideoLog($"https://www.youtube.com/user/{name.Value}"); Temporary.Add(new ChannelMediaContext(name,resolution)); } await Task.FromResult(0); } public async Task AddVideoAsync(VideoId videoId,Resolution res=Resolution.PreMuxed) { lock(Temporary) { GetLogger().WriteVideoLog($"https://www.youtube.com/watch?v={videoId.Value}"); Temporary.Add(new VideoMediaContext(videoId,res)); } await Task.FromResult(0); } public async Task AddFileAsync(string url,bool download=true) { lock(Temporary) { GetLogger().WriteVideoLog(url); Temporary.Add(new NormalDownloadMediaContext(url,download)); } await Task.FromResult(0); } public void CreateDirectoryIfNotExist(string dir) { if(!DirectoryExists(dir)) { CreateDirectory(dir); } } public async Task DownloadThumbnails(VideoId id) { if(!can_download) return; string Id=id.Value; string[] res=new string[] {"default.jpg","sddefault.jpg","mqdefault.jpg","hqdefault.jpg","maxresdefault.jpg"}; CreateDirectoryIfNotExist($"Thumbnails/{Id}"); foreach(var reso in res) { if(await Continue($"Thumbnails/{Id}/{reso}")) { try{ var data=await HttpClient.GetByteArrayAsync($"https://s.ytimg.com/vi/{Id}/{reso}"); await WriteAllBytesAsync($"Thumbnails/{Id}/{reso}",data); }catch(Exception ex) { _=ex; } } } } public void CreateDirectories() { CreateDirectoryIfNotExist("Channel"); CreateDirectoryIfNotExist("Playlist"); CreateDirectoryIfNotExist("Subscriptions"); CreateDirectoryIfNotExist("VideoOnly"); CreateDirectoryIfNotExist("AudioOnly"); CreateDirectoryIfNotExist("Muxed"); CreateDirectoryIfNotExist("PreMuxed"); CreateDirectoryIfNotExist("Info"); CreateDirectoryIfNotExist("Thumbnails"); CreateDirectoryIfNotExist("config"); CreateDirectoryIfNotExist("config/logs"); CreateDirectoryIfNotExist("config/addlog"); CreateDirectoryIfNotExist("DownloadsInfo"); CreateDirectoryIfNotExist("Downloads"); CreateDirectoryIfNotExist("StreamInfo"); CreateDirectoryIfNotExist("PersonalPlaylist"); } public void StartLoop(CancellationToken token = default(CancellationToken)) { CreateDirectories(); if(DirectoryExists("Download") && DirectoryExists("FileInfo")) Task.Run(MigrateDownloads).Wait(); Thread thread0=new Thread(()=>{ DownloadLoop(token).Wait(); }); thread0.Start(); Thread thread1=new Thread(()=>{ QueueLoop(token).Wait(); }); thread1.Start(); } private async IAsyncEnumerable GetDownloadsLegacyAsync() { await foreach(var item in EnumerateFilesAsync("FileInfo")) { if(Path.GetExtension(item).Equals(".json",StringComparison.Ordinal)) { var res= JsonConvert.DeserializeObject(await ReadAllTextAsync(item)); DeleteFile(item); yield return res; } } } private async Task MigrateDownloads() { await GetLogger().WriteAsync("Migrating File Downloads (Please Don't close TYTD)",true); await foreach(var dl in GetDownloadsLegacyAsync()) { await MoveLegacyDownload(dl); } int files=0; await foreach(var f in EnumerateDirectoriesAsync("FileInfo")) { files++; break; } if(files==0) { await foreach(var f in EnumerateFilesAsync("FileInfo")) { files++; break; } } if(files>0) { MoveDirectory("FileInfo","DownloadsInfoLegacy"); await GetLogger().WriteAsync("WARNING: there were still files/folders in FileInfo so they are stored in DownloadsInfoLegacy",true); }else{ DeleteDirectory("FileInfo"); } files=0; await foreach(var f in EnumerateDirectoriesAsync("Download")) { files++; break; } if(files==0) { await foreach(var f in EnumerateFilesAsync("Download")) { files++; break; } } if(files>0) { MoveDirectory("Download","DownloadsLegacy"); await GetLogger().WriteAsync("WARNING: there were still files/folders in Download so they are stored in DownloadsLegacy",true); }else{ DeleteDirectory("Download"); } await GetLogger().WriteAsync("Migrating Downloads Complete",true); } private async Task MoveLegacyDownload(SavedVideo dl) { await WriteVideoInfoAsync(dl); string old_incomplete_file_path = $"Download/{B64.Base64UrlEncodes(dl.Id)}-incomplete.part"; string old_file_path = $"Download/{B64.Base64UrlEncodes(dl.Id)}.bin"; string incomplete_file_path = $"Downloads/{HashDownloadUrl(dl.Id)}-incomplete.part"; string file_path = $"Downloads/{HashDownloadUrl(dl.Id)}.bin"; bool complete = FileExists(old_file_path); bool missing = !complete && !FileExists(old_incomplete_file_path); string alreadyStr =""; if(!missing) { if(complete) { //migrate complete if(!FileExists(file_path)) { RenameFile(old_file_path,file_path); }else{ alreadyStr="Already "; } } else { //migrate incomplete if(!FileExists(incomplete_file_path)) { RenameFile(old_incomplete_file_path,incomplete_file_path); }else{ alreadyStr="Already "; } } } await GetLogger().WriteAsync($"{alreadyStr}Migrated {(missing? "missing" : (complete ? "complete" : "incomplete"))} download: {dl.Title} with Url: {dl.Id}",true); } internal void ThrowError(TYTDErrorEventArgs e) { Error?.Invoke(this,e); } public async Task WriteAllTextAsync(string path,string data) { using(var dstStrm= await CreateAsync(path)) { using(var sw = new StreamWriter(dstStrm)) { await sw.WriteAsync(data); } } } public virtual async Task AddToPersonalPlaylistAsync(string name, IEnumerable items) { List items0=new List(); await foreach(var item in GetPersonalPlaylistContentsAsync(name)) { items0.Add(item); } items0.AddRange(items); await WriteAllTextAsync($"PersonalPlaylist/{name}.json",JsonConvert.SerializeObject(items0)); } public virtual async Task ReplacePersonalPlaylistAsync(string name, IEnumerable items) { await WriteAllTextAsync($"PersonalPlaylist/{name}.json",JsonConvert.SerializeObject(items.ToList())); } public virtual void DeletePersonalPlaylist(string name) { DeleteFile($"PersonalPlaylist/{name}.json"); } public virtual async Task RemoveItemFromPersonalPlaylistAsync(string name, VideoId id) { List items0=new List(); await foreach(var item in GetPersonalPlaylistContentsAsync(name)) { if(item.Id != id) { items0.Add(item); } } await WriteAllTextAsync($"PersonalPlaylist/{name}.json",JsonConvert.SerializeObject(items0)); } public virtual async Task SetResolutionForItemInPersonalPlaylistAsync(string name, VideoId id, Resolution resolution) { List items0=new List(); await foreach(var item in GetPersonalPlaylistContentsAsync(name)) { if(item.Id != id) { items0.Add(item); }else{ items0.Add(new ListContentItem(item.Id,resolution)); } } await WriteAllTextAsync($"PersonalPlaylist/{name}.json",JsonConvert.SerializeObject(items0)); } public void RegisterVideoStarted(EventHandler vs) { VideoStarted+=vs; } public void RegisterVideoFinished(EventHandler vf) { VideoFinished+=vf; } public void RegisterVideoProgress(EventHandler vp) { VideoProgress+=vp; } } }