tytd/Tesses.YouTubeDownloader/TYTD.cs

using System;
using YoutubeExplode;
using YoutubeExplode.Videos;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Net.Http;
using System.IO;
using YoutubeExplode.Playlists;
using YoutubeExplode.Channels;
using Newtonsoft.Json;
using System.Text;

namespace Tesses.YouTubeDownloader
{
    
    public abstract partial class TYTDStorage : TYTDBase, IStorage
    {
        public new virtual async Task<byte[]> ReadThumbnailAsync(VideoId videoId,string res,CancellationToken token=default)
        {
            CreateDirectoryIfNotExist($"Thumbnails/{videoId.Value}");
            if(await ThumbnailExistsAsync(videoId,res))
            {
                return await ReadAllBytesAsync($"Thumbnails/{videoId.Value}/{res}.jpg",token);
            }else{
               var result= await HttpClient.GetByteArrayAsync($"https://s.ytimg.com/vi/{videoId.Value}/{res}.jpg");
               await WriteThumbnailAsync(videoId,res,result,token);
               return result;
            }
        }
        public virtual async Task WriteThumbnailAsync(VideoId videoId,string res,byte[] data,CancellationToken token=default)
        {
            CreateDirectoryIfNotExist($"Thumbnails/{videoId.Value}");
            await WriteAllBytesAsync($"Thumbnails/{videoId.Value}/{res}.jpg",data,token);
        }
        public override ExtraData GetExtraData()
        {
            ExtraData data=new ExtraData();
            lock(Temporary){
                data.ItemsInInfoQueue=Temporary.Count;
            }
            lock(QueueList)
            {
                data.ItemsInQueue=QueueList.Count;
            }
            
            return data;
        }
        internal class ConsoleWriterCLS : TextWriter
        {
            Action<string> cls;
            public ConsoleWriterCLS(Action<string> cls)
            {
                this.cls=cls;
            }

            public override Encoding Encoding => Encoding.UTF8;
            public override void Write(string value)
            {
                cls(value);
            }
        }
        private static readonly HttpClient _default = new HttpClient();
        public abstract Task<Stream> CreateAsync(string path);

        public abstract void CreateDirectory(string path);

        public static bool UseConsole = true;

        public TYTDStorage(HttpClient clt)
        {
            
            HttpClient=clt;
            YoutubeClient=new YoutubeClient(HttpClient);
            ExtensionContext=null;
            ConsoleWriter=new ConsoleWriterCLS((e)=>{
                ConsoleWrite?.Invoke(this,new ConsoleWriteEventArgs(e));
            });
        }
        public TYTDStorage()
        {
            HttpClient=_default;
             YoutubeClient=new YoutubeClient(HttpClient);
            ExtensionContext=null;
             ConsoleWriter=new ConsoleWriterCLS((e)=>{
                ConsoleWrite?.Invoke(this,new ConsoleWriteEventArgs(e));
            });
        }
        public async Task WriteAllBytesAsync(string path,byte[] data,CancellationToken token=default(CancellationToken))
        {
            MemoryStream ms = new MemoryStream(data);
            using(var s=await CreateAsync(path))
            {
                await ms.CopyToAsync(s,4096,token);
            }
        }
        public EventHandler<ConsoleWriteEventArgs> ConsoleWrite;
        public TextWriter ConsoleWriter {get; private set;}
         public static string TYTDTag {get {return _tytd_tag;}}
         public static void SetTYTDTag(string tag)
         {
            //for use on mobile phones
            _tytd_tag= tag;
         }
         private static string _tytd_tag=_getTYTDTag();
         private static string _getTYTDTag()
         {
            string tag=Environment.GetEnvironmentVariable("TYTD_TAG");
            if(string.IsNullOrWhiteSpace(tag)) return "UnknownPC";
            return tag;
         }
        bool can_download=true;
        public bool CanDownload {get {return can_download;} set {can_download=value;}}
         public IExtensionContext ExtensionContext {get;set;}
        public HttpClient HttpClient {get;set;}
        public YoutubeClient YoutubeClient {get;set;}
        public abstract void MoveDirectory(string src,string dest);
        public abstract void DeleteFile(string file);
        public abstract void DeleteDirectory(string dir,bool recursive=false);

        public virtual async Task WriteBestStreamInfoAsync(VideoId id,BestStreamInfo.BestStreamsSerialized serialized)
        {
            await WriteAllTextAsync($"StreamInfo/{id.Value}.json",JsonConvert.SerializeObject(serialized));
            
        }

        public virtual async Task WriteVideoInfoAsync(SavedVideo info)
        {
            
            string file = info.DownloadFrom.StartsWith("NormalDownload,Length=") ? $"DownloadsInfo/{HashDownloadUrl(info.Id)}.json" : $"Info/{info.Id}.json";
            if(!await FileExistsAsync(file))
            {
               await WriteAllTextAsync(file,JsonConvert.SerializeObject(info));
            }
        }
        public virtual async Task WritePlaylistInfoAsync(SavedPlaylist info)
        {
             string file = $"Playlist/{info.Id}.json";
            if(!FileExists(file))
            {
               await WriteAllTextAsync(file,JsonConvert.SerializeObject(info));
            }
        }
        public virtual async Task WriteChannelInfoAsync(SavedChannel info)
        {
             string file = $"Channel/{info.Id}.json";
            if(!await FileExistsAsync(file))
            {
               await WriteAllTextAsync(file,JsonConvert.SerializeObject(info));
            }
        }
        public async Task AddPlaylistAsync(PlaylistId id,Resolution resolution=Resolution.PreMuxed)
        {
            lock(Temporary)
            {
                GetLogger().WriteVideoLog($"https://www.youtube.com/playlist?list={id.Value}");
                Temporary.Add( new PlaylistMediaContext(id,resolution));
            }
          

            await Task.FromResult(0);
        }
        public async Task AddChannelAsync(ChannelId id,Resolution resolution=Resolution.PreMuxed)
        {
            lock(Temporary)
            {
                GetLogger().WriteVideoLog($"https://www.youtube.com/channel/{id.Value}");
                Temporary.Add(new ChannelMediaContext(id,resolution));
            }
            await Task.FromResult(0);
        }
        public async Task AddSlugAsync(ChannelSlug slug,Resolution resolution=Resolution.PreMuxed)
        {
            lock(Temporary)
            {
                GetLogger().WriteVideoLog($"https://www.youtube.com/c/{slug.Value}");
               
                Temporary.Add(new ChannelMediaContext(slug,resolution));
            }
            await Task.FromResult(0);
        }
        public async Task AddHandleAsync(ChannelHandle handle,Resolution resolution=Resolution.PreMuxed)
        {
            lock(Temporary)
            {
                 GetLogger().WriteVideoLog($"https://www.youtube.com/@{handle.Value}");
                Temporary.Add(new ChannelMediaContext(handle,resolution));
            }
            await Task.FromResult(0);
        }
        public async Task AddUserAsync(UserName name,Resolution resolution=Resolution.PreMuxed)
        {
            lock(Temporary)
            {
                 GetLogger().WriteVideoLog($"https://www.youtube.com/user/{name.Value}");
                Temporary.Add(new ChannelMediaContext(name,resolution));
            }
            await Task.FromResult(0);
        }
        
        public async Task AddVideoAsync(VideoId videoId,Resolution res=Resolution.PreMuxed)
        {
            lock(Temporary)
            {
                GetLogger().WriteVideoLog($"https://www.youtube.com/watch?v={videoId.Value}");
                Temporary.Add(new VideoMediaContext(videoId,res));
            }
            await Task.FromResult(0);
        }
        public async Task AddFileAsync(string url,bool download=true)
        {
            lock(Temporary)
            {
                GetLogger().WriteVideoLog(url);
                Temporary.Add(new NormalDownloadMediaContext(url,download));
            }
            await Task.FromResult(0);
        }
        public void CreateDirectoryIfNotExist(string dir)
        {
            if(!DirectoryExists(dir))
            {
                CreateDirectory(dir);
            }
        }
        public async Task DownloadThumbnails(VideoId id)
        {
            if(!can_download) return;
            string Id=id.Value;
            string[] res=new string[] {"default.jpg","sddefault.jpg","mqdefault.jpg","hqdefault.jpg","maxresdefault.jpg"};
            CreateDirectoryIfNotExist($"Thumbnails/{Id}");
            foreach(var reso in res)
            {
                if(await Continue($"Thumbnails/{Id}/{reso}"))
                {
                    try{
                        var data=await HttpClient.GetByteArrayAsync($"https://s.ytimg.com/vi/{Id}/{reso}");
                        await WriteAllBytesAsync($"Thumbnails/{Id}/{reso}",data);
                    }catch(Exception ex)
                    {
                        _=ex;
                    }
                }
            }
        }
        public void CreateDirectories()
        {
            CreateDirectoryIfNotExist("Channel");
            CreateDirectoryIfNotExist("Playlist");
            CreateDirectoryIfNotExist("Subscriptions");
            CreateDirectoryIfNotExist("VideoOnly");
            CreateDirectoryIfNotExist("AudioOnly");
            CreateDirectoryIfNotExist("Muxed");
            CreateDirectoryIfNotExist("PreMuxed");
            CreateDirectoryIfNotExist("Info");
            CreateDirectoryIfNotExist("Thumbnails");
            CreateDirectoryIfNotExist("config");
            CreateDirectoryIfNotExist("config/logs");
            CreateDirectoryIfNotExist("config/addlog");
            CreateDirectoryIfNotExist("DownloadsInfo");
            CreateDirectoryIfNotExist("Downloads");
            CreateDirectoryIfNotExist("StreamInfo");
            CreateDirectoryIfNotExist("PersonalPlaylist");
        }
       
        public void StartLoop(CancellationToken token = default(CancellationToken))
        {
          
           CreateDirectories();
           if(DirectoryExists("Download") && DirectoryExists("FileInfo"))
           Task.Run(MigrateDownloads).Wait();
            Thread thread0=new Thread(()=>{
                 DownloadLoop(token).Wait();
            });
            thread0.Start();
            Thread thread1=new Thread(()=>{
                 QueueLoop(token).Wait();
            });
           thread1.Start();
        }
        private async IAsyncEnumerable<SavedVideo> GetDownloadsLegacyAsync()
        {
           await foreach(var item in EnumerateFilesAsync("FileInfo"))
            {
                if(Path.GetExtension(item).Equals(".json",StringComparison.Ordinal))
                {
                    var res= JsonConvert.DeserializeObject<SavedVideo>(await ReadAllTextAsync(item));
                    DeleteFile(item);
                    yield return res;
                }
            }
          
        }
        private async Task MigrateDownloads()
        {
             await  GetLogger().WriteAsync("Migrating File Downloads (Please Don't close TYTD)",true);
           
            await foreach(var dl in GetDownloadsLegacyAsync())
            {
                await MoveLegacyDownload(dl);
            }
              int files=0;
            await foreach(var f in EnumerateDirectoriesAsync("FileInfo"))
            {
                files++;
                break;
            }
            if(files==0)
            {
                await foreach(var f in EnumerateFilesAsync("FileInfo"))
                {
                    files++;
                    break;
                }
            }
            if(files>0)
            {
                MoveDirectory("FileInfo","DownloadsInfoLegacy");
                await GetLogger().WriteAsync("WARNING: there were still files/folders in FileInfo so they are stored in DownloadsInfoLegacy",true);
            }else{
                DeleteDirectory("FileInfo");
            }
            files=0;
             await foreach(var f in EnumerateDirectoriesAsync("Download"))
            {
                files++;
                break;
            }
            if(files==0)
            {
                await foreach(var f in EnumerateFilesAsync("Download"))
                {
                    files++;
                    break;
                }
            }
            if(files>0)
            {
                MoveDirectory("Download","DownloadsLegacy");
                await GetLogger().WriteAsync("WARNING: there were still files/folders in Download so they are stored in DownloadsLegacy",true);
            }else{
                DeleteDirectory("Download");
            }
            

            await GetLogger().WriteAsync("Migrating Downloads Complete",true);
        }
        private async Task MoveLegacyDownload(SavedVideo dl)
        {
            await WriteVideoInfoAsync(dl);
            string old_incomplete_file_path = $"Download/{B64.Base64UrlEncodes(dl.Id)}-incomplete.part";
            string old_file_path = $"Download/{B64.Base64UrlEncodes(dl.Id)}.bin";

            string incomplete_file_path = $"Downloads/{HashDownloadUrl(dl.Id)}-incomplete.part";
            string file_path = $"Downloads/{HashDownloadUrl(dl.Id)}.bin";

            bool complete = FileExists(old_file_path);
            bool missing = !complete && !FileExists(old_incomplete_file_path);
            string alreadyStr ="";
            if(!missing)
            {
                if(complete)
                {
                     //migrate complete
                    if(!FileExists(file_path))
                    {
                        RenameFile(old_file_path,file_path);
                    }else{
                        alreadyStr="Already ";
                    }
                }
                else
                {
                    //migrate incomplete
                    if(!FileExists(incomplete_file_path))
                    {
                        RenameFile(old_incomplete_file_path,incomplete_file_path);
                    }else{
                        alreadyStr="Already ";
                    }
                }
            }


            await  GetLogger().WriteAsync($"{alreadyStr}Migrated {(missing? "missing" : (complete ? "complete" : "incomplete"))} download: {dl.Title} with Url: {dl.Id}",true);
           
        }
        internal void ThrowError(TYTDErrorEventArgs e)
        {
            Error?.Invoke(this,e);
        }
        public async Task WriteAllTextAsync(string path,string data)
        {
            using(var dstStrm= await CreateAsync(path))
            {
                using(var sw = new StreamWriter(dstStrm))
                {
                    await sw.WriteAsync(data);
                }
            }
        }

        public virtual async Task AddToPersonalPlaylistAsync(string name, IEnumerable<ListContentItem> items)
        {
             List<ListContentItem> items0=new List<ListContentItem>();
            await foreach(var item in GetPersonalPlaylistContentsAsync(name))
            {
                items0.Add(item);
            }
            items0.AddRange(items);
            await WriteAllTextAsync($"PersonalPlaylist/{name}.json",JsonConvert.SerializeObject(items0));
            
        }

        public virtual async Task ReplacePersonalPlaylistAsync(string name, IEnumerable<ListContentItem> items)
        {
            
            await WriteAllTextAsync($"PersonalPlaylist/{name}.json",JsonConvert.SerializeObject(items.ToList()));
            
        }
        public virtual void DeletePersonalPlaylist(string name)
        {
            DeleteFile($"PersonalPlaylist/{name}.json");
        }

        public virtual async Task RemoveItemFromPersonalPlaylistAsync(string name, VideoId id)
        {
              List<ListContentItem> items0=new List<ListContentItem>();
            await foreach(var item in GetPersonalPlaylistContentsAsync(name))
            {
                if(item.Id != id)
                {
                    items0.Add(item);
                }
            }
           
            await WriteAllTextAsync($"PersonalPlaylist/{name}.json",JsonConvert.SerializeObject(items0));
            
        }

        public virtual async Task SetResolutionForItemInPersonalPlaylistAsync(string name, VideoId id, Resolution resolution)
        {
             List<ListContentItem> items0=new List<ListContentItem>();
            await foreach(var item in GetPersonalPlaylistContentsAsync(name))
            {
                if(item.Id != id)
                {
                    items0.Add(item);
                }else{
                    items0.Add(new ListContentItem(item.Id,resolution));
                }
            }
           
            await WriteAllTextAsync($"PersonalPlaylist/{name}.json",JsonConvert.SerializeObject(items0));
            
        }
       
        public void RegisterVideoStarted(EventHandler<VideoStartedEventArgs> vs)
        {
            VideoStarted+=vs;
        }

        public void RegisterVideoFinished(EventHandler<VideoFinishedEventArgs> vf)
        {
            VideoFinished+=vf;
        }

        public void RegisterVideoProgress(EventHandler<VideoProgressEventArgs> vp)
        {
            VideoProgress+=vp;
        }
    }
}