using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using YoutubeExplode.Videos; using YoutubeExplode; using System.Net.Http; using System.Net; using System.IO; using TYTD.Server.Models; using Newtonsoft.Json; using YoutubeExplode.Videos.Streams; using YoutubeExplode.Channels; using YoutubeExplode.Playlists; using Dasync.Collections; namespace TYTD.Server.Functions { public class Downloader { public Downloader() { // TessesYoutubeDownloader.Server.Functions.Downloader.DL.DownloadThread().GetAwaiter().GetResult(); } public static bool DownloadIt = true; public static bool RedownloadIt = false; public List infoQueue = new List(); public static YoutubeClient CreateYoutubeClient() { ServicePointManager .ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; HttpClientHandler handler = new HttpClientHandler(); handler.ServerCertificateCustomValidationCallback = (sender, cert, chain, sslPolicyErrors) => true; Directory.CreateDirectory("config"); string cookiesFile = Path.Combine("config", "cookies.txt"); if (File.Exists(cookiesFile)) { var cookies = CookiesTxt.Parser.ParseFileAsCookieCollection(cookiesFile); handler.CookieContainer.Add(cookies); } Http = new HttpClient(handler); return new YoutubeClient(Http); } public static VideoDownloadProgress GetProgress() { return P; } static HttpClient Http; internal YoutubeClient ytc = CreateYoutubeClient(); static VideoDownloadProgress P = new VideoDownloadProgress(); const int NUM_WAITS = 5; static int WAITS = 0; Progress DownloadP = new Progress((e) => { P.Progress = (int)(e * 100.0); P.ProgressRaw = e; }); public List Queue = new List(); public static string GetQueue() { string q; lock (DL.Queue) { q = JsonConvert.SerializeObject(DL.Queue.Select(o => o.Video)); } return q; } public async Task ListenForQueueItem() { do { InfomationQueueItem item; bool canAdd = false; lock (infoQueue) { canAdd = infoQueue.Count > 0; if (canAdd) { item = infoQueue[0]; infoQueue.RemoveAt(0); } else { item = null; } } if (canAdd) { try { var items = await item.DownloadData(); if (item.DownloadActualDataAfterwards) { _DownloadVideos(items); } } catch (Exception ex) { Console.WriteLine(ex.Message); _ = ex; } } } while (true); } public static void ModQueue2(string mvto, string index) { try { //?mv=up|down|top|bottom|remove,int&i=videoId;URL lock (DL.Queue) { int index2 = -1; for(int i=0;i 0) { var v = DL.Queue[index2]; DL.Queue.Remove(v); DL.Queue.Insert(index2 - 1, v); } } else if (mvto == "down") { if (index2 < DL.Queue.Count - 1) { var v = DL.Queue[index2]; DL.Queue.Remove(v); DL.Queue.Insert(index2 + 1, v); } } else { int n1; if (int.TryParse(mvto, out n1)) { var v = DL.Queue[index2]; DL.Queue.Remove(v); if (n1 > index2) { DL.Queue.Insert(n1 - 1, v); } else { DL.Queue.Insert(n1, v); } } } } } catch (Exception ex) { Console.WriteLine(ex.Message); _ = ex; } } public static void ModQueue(string mvto, string index) { try { //?mv=up|down|top|bottom|remove,int&i=0,last lock (DL.Queue) { int index2 = 0; if (index == "last") { index2 = DL.Queue.Count - 1; } else { if (!int.TryParse(index, out index2)) { index2 = 0; } } if (index2 >= DL.Queue.Count) { index2 = DL.Queue.Count - 1; } if (mvto == "top") { var v = DL.Queue[index2]; DL.Queue.Remove(v); DL.Queue.Insert(0, v); } else if (mvto == "bottom") { var v = DL.Queue[index2]; DL.Queue.Remove(v); DL.Queue.Add(v); } else if (mvto == "remove") { var v = DL.Queue[index2]; DL.Queue.Remove(v); } else if (mvto == "up") { if (index2 > 0) { var v = DL.Queue[index2]; DL.Queue.Remove(v); DL.Queue.Insert(index2 - 1, v); } } else if (mvto == "down") { if (index2 < DL.Queue.Count - 1) { var v = DL.Queue[index2]; DL.Queue.Remove(v); DL.Queue.Insert(index2 + 1, v); } } else { int n1; if (int.TryParse(mvto, out n1)) { var v = DL.Queue[index2]; DL.Queue.Remove(v); if (n1 > index2) { DL.Queue.Insert(n1 - 1, v); } else { DL.Queue.Insert(n1, v); } } } } } catch (Exception ex) { Console.WriteLine(ex.Message); _ = ex; } } public static void DownloadChannelOnly(string id) { ChannelId? v = ChannelId.TryParse(id); if (v.HasValue) { InfomationQueueItem item = new InfomationQueueItem(v.Value, Resolution.NoConvert, false); lock (DL.infoQueue) { DL.infoQueue.Insert(0, item); } } } public static void DownloadItems(List id) { List items = new List(); foreach (var item in id) { var iqi = item.ToInfomationQueueItem(); if (iqi != null) { items.Add(iqi); } } if (items.Count > 0) { lock (DL.infoQueue) { DL.infoQueue.InsertRange(0, items); } } } public static void DownloadUserOnly(string id) { UserName? v = UserName.TryParse(id); if (v.HasValue) { InfomationQueueItem item = new InfomationQueueItem(v.Value, Resolution.NoConvert, false); lock (DL.infoQueue) { DL.infoQueue.Insert(0, item); } } } public bool Continue(string v) { if (File.Exists(v)) { using (var f = File.OpenRead(v)) { return f.Length == 0; } } return true; } private string gStorageLocation() { if (File.Exists("loc.txt")) { string loc = File.ReadAllText("loc.txt"); try { Directory.CreateDirectory(loc); if (Directory.Exists(loc)) { return loc; } } catch (Exception ex) { Console.WriteLine(ex.Message); _ = ex; } } return Environment.CurrentDirectory; } public string StorageLocation { get { return gStorageLocation(); } } private void _DownloadVideos(SavedVideoObject[] items) { lock (Queue) { foreach (var item in items) { Queue.Insert(0, item); //new elements get added to begining } } } public bool FileExists(string nameSrc,ref string nameDest,ref int i) { if(i == 1) { i++; return File.Exists(nameSrc); } i++; nameDest= Path.Combine(Path.GetDirectoryName(nameSrc), $"{Path.GetFileNameWithoutExtension(nameSrc)} ({i}){Path.GetExtension(nameSrc)}"); return File.Exists(nameDest); } public static Func> GetManifest; public async Task DownloadThread() { do { bool canDownload = false; SavedVideoObject v; lock (Queue) { canDownload = Queue.Count > 0; if (canDownload) { v = Queue[0]; Queue.RemoveAt(0); P.Saved = v.Video; if (v.RegularFile) { Console.WriteLine($"Download: {v.Video.Id}"); } else { Console.WriteLine($"Download: {v.Video.Title}"); } } else { v = null; } } if (canDownload) { DownloadStartEventArgs evt = new DownloadStartEventArgs(); evt.Cancel = false; evt.RegularFile = v.RegularFile; evt.Video = v.Video; ApiLoader.DownloadStarted(this, evt); if(evt.Cancel) { DownloadIt = true; goto nope; } redownload: try { if (v.RegularFile) { var req=await Http.GetAsync(v.Video.Id); long? Len=req.Content.Headers.ContentLength; Uri u = new Uri(v.Video.Id); string abs = u.AbsolutePath; string name = "file.bin"; if(abs.Contains("/")) { name=abs.Substring(abs.LastIndexOf('/') + 1); } if (req.Content.Headers.Contains("Content-Disposition")) { name=req.Content.Headers.ContentDisposition.FileName; } int fileI=1; P.Saved.Title = name; name=GetPath(true, "Download", name); string filename=name; while (FileExists(name, ref filename, ref fileI)) ; long Len2 = long.MaxValue; if(Len.HasValue) { if(Len.Value > 0) { Len2 = Len.Value; } } P.Length = Len2; long Pos = 0; byte[] buffer = new byte[4096]; int Cycles = 0; IProgress p = DownloadP; int CYCLES_BETWEEN_REPORT = 1; using (var srcFile = await req.Content.ReadAsStreamAsync()) { using (var destFile = File.Create(filename)) { int read; do { read = await srcFile.ReadAsync(buffer, 0, buffer.Length); await destFile.WriteAsync(buffer, 0, read); Pos += read; Cycles++; if (Cycles > CYCLES_BETWEEN_REPORT) { Cycles = 0; p.Report(Pos / Len2); } } while (read > 0 && DownloadIt); } } p.Report(1); } else { switch (v.Resolution) { case Resolution.Convert: string mypath = GetPath(true, "Converted", v.Video.Id + "-vidonly.bkp"); string mypathaudio = GetPath(true, "Audio", v.Video.Id + "incomplete.mp4"); string mypathCompleteAudio = GetPath(true, "Audio", v.Video.Id + ".mp4"); string mypathComplete = GetPath(true, "Converted", v.Video.Id + ".mp4"); string mypathIncompleteConverting = GetPath(true, "Converted", "conv.mkv"); if (Continue(mypathComplete)) { var s3 = await GetManifest(ytc,v.Video.Id); var best2 = s3.GetAudioOnlyStreams().GetWithHighestBitrate(); var best = s3.GetVideoOnlyStreams().GetWithHighestVideoQuality(); P.Length = best.Size.Bytes + best2.Size.Bytes; ProgressTwo p = new ProgressTwo(best.Size.Bytes, best2.Size.Bytes, DownloadP); using (var destStrm = System.IO.File.Open(mypath, FileMode.OpenOrCreate, FileAccess.ReadWrite)) { long pos = 0; long len = 0; using (var srcStrm = await ytc.Videos.Streams.GetAsync(best)) { len = srcStrm.Length; pos = destStrm.Length; IProgress myProgress = p.Video; if (pos >= len) { myProgress.Report(1); } /* This is why videos get corrupted */ srcStrm.Seek(destStrm.Length, SeekOrigin.Begin); destStrm.Seek(destStrm.Length, SeekOrigin.Begin); byte[] buffer = new byte[4096]; int read = 0; do { read = srcStrm.Read(buffer, 0, buffer.Length); destStrm.Write(buffer, 0, read); pos += read; double myP = (double)pos / (double)len; myProgress.Report(myP); } while (read > 0 && DownloadIt ) ; } } IProgress pv = p.Video; pv.Report(1); if (Continue(mypathCompleteAudio)) { long pos = 0; long len = 0; using (var destStrm = File.Open(mypathaudio, FileMode.OpenOrCreate, FileAccess.ReadWrite)) { using (var srcStrm = await ytc.Videos.Streams.GetAsync(best2)) { len = srcStrm.Length; pos = destStrm.Length; IProgress myProgress = p.Audio; if (pos >= len) { myProgress.Report(1); } /* This is why videos get corrupted */ srcStrm.Seek(destStrm.Length, SeekOrigin.Begin); destStrm.Seek(destStrm.Length, SeekOrigin.Begin); byte[] buffer = new byte[4096]; int read = 0; do { read = srcStrm.Read(buffer, 0, buffer.Length); destStrm.Write(buffer, 0, read); pos += read; double myP = (double)pos / (double)len; myProgress.Report(myP); } while (read > 0 && DownloadIt); } } if(DownloadIt) { File.Move(mypathaudio, mypathCompleteAudio); } } IProgress pa = p.Video; if (DownloadIt) { pa.Report(1); ffmpeg.mux(mypath, mypathCompleteAudio, mypathIncompleteConverting); File.Move(mypathIncompleteConverting, mypathComplete); } } break; case Resolution.NoConvert: string mypath2 = GetPath(true, "NotConverted", v.Video.Id + "incomplete.mp4"); string mypath2Complete = GetPath(true, "NotConverted", v.Video.Id + ".mp4"); if (Continue(mypath2Complete)) { var s = await GetManifest(ytc,v.Video.Id); var best = s.GetMuxedStreams().GetWithHighestVideoQuality(); P.Length = best.Size.Bytes; long pos = 0; long len = 0; using (var destStrm = System.IO.File.Open(mypath2, FileMode.OpenOrCreate, FileAccess.ReadWrite)) { using (var srcStrm = await ytc.Videos.Streams.GetAsync(best)) { len = srcStrm.Length; pos = destStrm.Length; IProgress myProgress = DownloadP; if (pos >= len) { myProgress.Report(1); } /* This is why videos get corrupted */ srcStrm.Seek(destStrm.Length, SeekOrigin.Begin); destStrm.Seek(destStrm.Length, SeekOrigin.Begin); byte[] buffer = new byte[4096]; int read = 0; do { read = srcStrm.Read(buffer, 0, buffer.Length); destStrm.Write(buffer, 0, read); pos += read; double myP = (double)pos / (double)len; myProgress.Report(myP); } while (read > 0 && DownloadIt); } } if (DownloadIt) { File.Move(mypath2, mypath2Complete); } } break; case Resolution.Audio: string mypath3 = GetPath(true, "Audio", v.Video.Id + "incomplete.mp4"); string mypath3Complete = GetPath(true, "Audio", v.Video.Id + ".mp4"); if (Continue(mypath3Complete)) { var s2 = await GetManifest(ytc,v.Video.Id); var best2 = s2.GetAudioOnlyStreams().GetWithHighestBitrate(); P.Length = best2.Size.Bytes; long pos = 0; long len = 0; using (var destStrm = System.IO.File.Open(mypath3, FileMode.OpenOrCreate, FileAccess.ReadWrite)) { using (var srcStrm = await ytc.Videos.Streams.GetAsync(best2)) { len = srcStrm.Length; pos = destStrm.Length; IProgress myProgress = DownloadP; if (pos >= len) { myProgress.Report(1); } /* This is why videos get corrupted */ srcStrm.Seek(destStrm.Length, SeekOrigin.Begin); destStrm.Seek(destStrm.Length, SeekOrigin.Begin); byte[] buffer = new byte[4096]; int read = 0; do { read = srcStrm.Read(buffer, 0, buffer.Length); destStrm.Write(buffer, 0, read); pos += read; double myP = (double)pos / (double)len; myProgress.Report(myP); } while (read > 0 && DownloadIt); } } if (DownloadIt) { File.Move(mypath3, mypath3Complete); } } break; } if (DownloadIt) { DownloadCompleteEventArgs evt2 = new DownloadCompleteEventArgs(); evt2.RegularFile = v.RegularFile; evt2.Video = v.Video; ApiLoader.DownloadComplete(this, evt2); ffmpeg.on_video_done(v.Video.Id, (int)v.Resolution); } } if (RedownloadIt && !DownloadIt) { DownloadIt = true; goto redownload; } DownloadIt = true; } catch (Exception ex) { Console.WriteLine(ex.Message); } } nope: System.Threading.Thread.Sleep(2000); } while (true); } internal void _DownloadThumbnail(int w, int h, string id, string tnail) { try { string p = GetPath(true, "Thumbnails", w.ToString() + 'x' + h.ToString(), id + ".jpg"); if (!File.Exists(p)) { ffmpeg.download_thumbnail(tnail, p); } } catch (Exception ex) { Console.WriteLine(ex.Message); _ = ex; } } internal void _DownloadThumbnail2(int w, int h, string id, string tnail) { try { string p = GetPath(true, "Thumbnails", w.ToString() + 'x' + h.ToString(), id + ".jpg"); if (!File.Exists(p)) { using (var f = File.Create(p)) { Http.GetStreamAsync(tnail); } } } catch (Exception ex) { Console.WriteLine(ex.Message); _ = ex; } } public static async Task> Search(string text, bool downloadThumbs = true) { List media = new List(); try { await DL.ytc.Search.GetVideosAsync(text).ForEachAsync((e) => { if (downloadThumbs) { foreach (var t in e.Thumbnails) { DL._DownloadThumbnail2(t.Resolution.Width, t.Resolution.Height, e.Id, t.Url); } } media.Add(new SavedMedia() { Title = e.Title, Id = e.Id, Kind = InfoType.Video }); }); await DL.ytc.Search.GetPlaylistsAsync(text).ForEachAsync((e) => { if (downloadThumbs) { foreach (var t in e.Thumbnails) { DL._DownloadThumbnail2(t.Resolution.Width, t.Resolution.Height, e.Id, t.Url); } } media.Add(new SavedMedia() { Title = e.Title, Id = e.Id, Kind = InfoType.Playlist }); }); await DL.ytc.Search.GetChannelsAsync(text).ForEachAsync((e) => { if (downloadThumbs) { foreach (var t in e.Thumbnails) { DL._DownloadThumbnail2(t.Resolution.Width, t.Resolution.Height, e.Id, t.Url); } } media.Add(new SavedMedia() { Title = e.Title, Id = e.Id, Kind = InfoType.Channel }); }); } catch (Exception ex) { _ = ex; } return media; } public static List GetQueueItems() { return DL.Queue; } public static void DownloadFile(string id) { InfomationQueueItem item = new InfomationQueueItem(new Uri(id)); lock (DL.infoQueue) { DL.infoQueue.Insert(0, item); } } public static void DownloadVideo(string id, Resolution res) { VideoId? v = VideoId.TryParse(id); if (v.HasValue) { InfomationQueueItem item = new InfomationQueueItem(v.Value, res, true); lock (DL.infoQueue) { DL.infoQueue.Insert(0, item); } } } public static void DownloadVideoInfo(string id, Resolution res) { VideoId? v = VideoId.TryParse(id); if (v.HasValue) { InfomationQueueItem item = new InfomationQueueItem(v.Value, res, false); lock (DL.infoQueue) { DL.infoQueue.Insert(0, item); } } } public static void DownloadVideo(string v) { DownloadVideo(v, Resolution.NoConvert); } public static void DownloadCaptions(string id) { VideoId? v = VideoId.TryParse(id); if(v.HasValue) { InfomationQueueItem item = new InfomationQueueItem(v.Value); lock (DL.infoQueue) { DL.infoQueue.Insert(0, item); } } } public static void DownloadPlaylist(string id,Resolution res) { PlaylistId? v = PlaylistId.TryParse(id); if (v.HasValue) { InfomationQueueItem item = new InfomationQueueItem(v.Value, res, true); lock (DL.infoQueue) { DL.infoQueue.Insert(0, item); } } } public static void DownloadPlaylistOnly(string id, Resolution res) { PlaylistId? v = PlaylistId.TryParse(id); if (v.HasValue) { InfomationQueueItem item = new InfomationQueueItem(v.Value, res, false); lock (DL.infoQueue) { DL.infoQueue.Insert(0, item); } } } public static void DownloadPlaylist(string id) { DownloadPlaylist(id, Resolution.NoConvert); } public static void DownloadChannel(string id,Resolution res) { ChannelId? v = ChannelId.TryParse(id); if (v.HasValue) { InfomationQueueItem item = new InfomationQueueItem(v.Value, res, true); lock (DL.infoQueue) { DL.infoQueue.Insert(0, item); } } } public static void DownloadChannel(string id) { DownloadChannel(id, Resolution.NoConvert); } public static void DownloadItem(string id) { DownloadItem(id, Resolution.NoConvert); } public static void DownloadItem(string id,Resolution res) { VideoId? vid = VideoId.TryParse(id); PlaylistId? pid = PlaylistId.TryParse(id); ChannelId? cid = ChannelId.TryParse(id); UserName? user = UserName.TryParse(id); if (id.Length == 11) { if (vid.HasValue) { DownloadVideo(vid.Value, res); //shall we download video } } else { if (pid.HasValue) { DownloadPlaylist(pid.Value, res); } else if (vid.HasValue) { DownloadVideo(vid.Value, res); } else if (cid.HasValue) { DownloadChannel(cid.Value, res); } else if (user.HasValue) { DownloadUser(user.Value, res); } } } public static void DownloadUser(string name, Resolution res) { UserName? v=UserName.TryParse(name); if (v.HasValue) { InfomationQueueItem item = new InfomationQueueItem(v.Value, res, true); lock (DL.infoQueue) { DL.infoQueue.Insert(0, item); } } } public static void DownloadUser(string name) { DownloadUser(name, Resolution.NoConvert); } public string GetPath(bool createParent,params string[] _path) { if (createParent) { string dir = GetPath(_path); try { Directory.CreateDirectory(Path.GetDirectoryName(dir)); }catch(Exception ex) { Console.WriteLine(ex.Message); _ = ex; } return dir; } else { return GetPath(_path); } } private string GetPath(params string[] _path) { string[] array2 = new string[_path.Length + 1]; array2[0] = StorageLocation; Array.Copy(_path, 0, array2, 1,_path.Length); return System.IO.Path.Combine(array2); } public static Downloader DL = new Downloader(); } }