tytd-server/TYTD.Api/Server/Functions/Downloader.cs

1027 lines
39 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using YoutubeExplode.Videos;
using YoutubeExplode;
using System.Net.Http;
using System.Net;
using System.IO;
using TYTD.Server.Models;
using Newtonsoft.Json;
using YoutubeExplode.Videos.Streams;
using YoutubeExplode.Channels;
using YoutubeExplode.Playlists;
using Dasync.Collections;
namespace TYTD.Server.Functions
{
public class Downloader
{
public Downloader()
{
// TessesYoutubeDownloader.Server.Functions.Downloader.DL.DownloadThread().GetAwaiter().GetResult();
}
public static bool DownloadIt = true;
public static bool RedownloadIt = false;
public List<InfomationQueueItem> infoQueue = new List<InfomationQueueItem>();
public static YoutubeClient CreateYoutubeClient()
{
ServicePointManager
.ServerCertificateValidationCallback +=
(sender, cert, chain, sslPolicyErrors) => true;
HttpClientHandler handler = new HttpClientHandler();
handler.ServerCertificateCustomValidationCallback = (sender, cert, chain, sslPolicyErrors) => true;
Directory.CreateDirectory("config");
string cookiesFile = Path.Combine("config", "cookies.txt");
if (File.Exists(cookiesFile))
{
var cookies = CookiesTxt.Parser.ParseFileAsCookieCollection(cookiesFile);
handler.CookieContainer.Add(cookies);
}
Http = new HttpClient(handler);
return new YoutubeClient(Http);
}
public static VideoDownloadProgress GetProgress()
{
return P;
}
static HttpClient Http;
internal YoutubeClient ytc = CreateYoutubeClient();
static VideoDownloadProgress P = new VideoDownloadProgress();
const int NUM_WAITS = 5;
static int WAITS = 0;
public static void SendProgress(double p)
{
WAITS++;
if (WAITS <= NUM_WAITS)
return;
WAITS = 0;
ApiLoader.SetProgress(p);
}
Progress<double> DownloadP = new Progress<double>((e) => { P.Progress = (int)(e * 100.0); P.ProgressRaw = e; SendProgress(e); });
public List<SavedVideoObject> Queue = new List<SavedVideoObject>();
public static string GetQueue()
{
string q;
lock (DL.Queue)
{
q = JsonConvert.SerializeObject(DL.Queue.Select<SavedVideoObject, SavedVideo>(o => o.Video));
}
return q;
}
public async Task ListenForQueueItem()
{
do
{
InfomationQueueItem item;
bool canAdd = false;
lock (infoQueue)
{
canAdd = infoQueue.Count > 0;
if (canAdd)
{
item = infoQueue[0];
infoQueue.RemoveAt(0);
}
else
{
item = null;
}
}
if (canAdd)
{
try
{
var items = await item.DownloadData();
if (item.DownloadActualDataAfterwards)
{
_DownloadVideos(items);
}
} catch (Exception ex)
{
Console.WriteLine(ex.Message);
_ = ex;
}
}
} while (true);
}
public static void ModQueue2(string mvto, string index)
{
try
{
//?mv=up|down|top|bottom|remove,int&i=videoId;URL
lock (DL.Queue)
{
int index2 = -1;
for(int i=0;i<DL.Queue.Count;i++)
{
if(DL.Queue[i].Video.Id == index)
{
index2 = i;
break;
}
}
if(index2 < 0)
{
return;
}
if (mvto == "top")
{
var v = DL.Queue[index2];
DL.Queue.Remove(v);
DL.Queue.Insert(0, v);
}
else if (mvto == "bottom")
{
var v = DL.Queue[index2];
DL.Queue.Remove(v);
DL.Queue.Add(v);
}
else if (mvto == "remove")
{
var v = DL.Queue[index2];
DL.Queue.Remove(v);
}
else if (mvto == "up")
{
if (index2 > 0)
{
var v = DL.Queue[index2];
DL.Queue.Remove(v);
DL.Queue.Insert(index2 - 1, v);
}
}
else if (mvto == "down")
{
if (index2 < DL.Queue.Count - 1)
{
var v = DL.Queue[index2];
DL.Queue.Remove(v);
DL.Queue.Insert(index2 + 1, v);
}
}
else
{
int n1;
if (int.TryParse(mvto, out n1))
{
var v = DL.Queue[index2];
DL.Queue.Remove(v);
if (n1 > index2)
{
DL.Queue.Insert(n1 - 1, v);
}
else
{
DL.Queue.Insert(n1, v);
}
}
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
_ = ex;
}
}
public static void ModQueue(string mvto, string index)
{
try
{
//?mv=up|down|top|bottom|remove,int&i=0,last
lock (DL.Queue)
{
int index2 = 0;
if (index == "last")
{
index2 = DL.Queue.Count - 1;
}
else
{
if (!int.TryParse(index, out index2))
{
index2 = 0;
}
}
if (index2 >= DL.Queue.Count)
{
index2 = DL.Queue.Count - 1;
}
if (mvto == "top")
{
var v = DL.Queue[index2];
DL.Queue.Remove(v);
DL.Queue.Insert(0, v);
}
else if (mvto == "bottom")
{
var v = DL.Queue[index2];
DL.Queue.Remove(v);
DL.Queue.Add(v);
}
else if (mvto == "remove")
{
var v = DL.Queue[index2];
DL.Queue.Remove(v);
}
else if (mvto == "up")
{
if (index2 > 0)
{
var v = DL.Queue[index2];
DL.Queue.Remove(v);
DL.Queue.Insert(index2 - 1, v);
}
}
else if (mvto == "down")
{
if (index2 < DL.Queue.Count - 1)
{
var v = DL.Queue[index2];
DL.Queue.Remove(v);
DL.Queue.Insert(index2 + 1, v);
}
}
else
{
int n1;
if (int.TryParse(mvto, out n1))
{
var v = DL.Queue[index2];
DL.Queue.Remove(v);
if (n1 > index2)
{
DL.Queue.Insert(n1 - 1, v);
}
else
{
DL.Queue.Insert(n1, v);
}
}
}
}
} catch (Exception ex)
{
Console.WriteLine(ex.Message);
_ = ex;
}
}
public static void DownloadChannelOnly(string id)
{
ChannelId? v = ChannelId.TryParse(id);
if (v.HasValue)
{
InfomationQueueItem item = new InfomationQueueItem(v.Value, Resolution.NoConvert, false);
lock (DL.infoQueue)
{
DL.infoQueue.Insert(0, item);
}
}
}
public static void DownloadItems(List<IDResolutionTypeTriplet> id)
{
List<InfomationQueueItem> items = new List<InfomationQueueItem>();
foreach (var item in id)
{
var iqi = item.ToInfomationQueueItem();
if (iqi != null)
{
items.Add(iqi);
}
}
if (items.Count > 0)
{
lock (DL.infoQueue)
{
DL.infoQueue.InsertRange(0, items);
}
}
}
public static void DownloadUserOnly(string id)
{
UserName? v = UserName.TryParse(id);
if (v.HasValue)
{
InfomationQueueItem item = new InfomationQueueItem(v.Value, Resolution.NoConvert, false);
lock (DL.infoQueue)
{
DL.infoQueue.Insert(0, item);
}
}
}
public bool Continue(string v)
{
if (File.Exists(v))
{
using (var f = File.OpenRead(v))
{
return f.Length == 0;
}
}
return true;
}
private string gStorageLocation()
{
if (File.Exists("loc.txt"))
{
string loc = File.ReadAllText("loc.txt");
try
{
Directory.CreateDirectory(loc);
if (Directory.Exists(loc))
{
return loc;
}
} catch (Exception ex)
{
Console.WriteLine(ex.Message);
_ = ex;
}
}
return Environment.CurrentDirectory;
}
public string StorageLocation { get { return gStorageLocation(); } }
private void _DownloadVideos(SavedVideoObject[] items)
{
lock (Queue)
{
foreach (var item in items)
{
Queue.Insert(0, item);
//new elements get added to begining
}
}
}
public bool FileExists(string nameSrc,ref string nameDest,ref int i)
{
if(i == 1)
{
i++;
return File.Exists(nameSrc);
}
i++;
nameDest= Path.Combine(Path.GetDirectoryName(nameSrc), $"{Path.GetFileNameWithoutExtension(nameSrc)} ({i}){Path.GetExtension(nameSrc)}");
return File.Exists(nameDest);
}
public static Func<YoutubeClient,VideoId, Task<StreamManifest>> GetManifest;
public async Task DownloadThread()
{
do
{
bool canDownload = false;
SavedVideoObject v;
lock (Queue)
{
canDownload = Queue.Count > 0;
if (canDownload)
{
v = Queue[0];
Queue.RemoveAt(0);
P.Saved = v.Video;
if (v.RegularFile)
{
Console.WriteLine($"Download: {v.Video.Id}");
}
else
{
Console.WriteLine($"Download: {v.Video.Title}");
}
}
else
{
v = null;
}
}
if (canDownload)
{
DownloadStartEventArgs evt = new DownloadStartEventArgs();
evt.Cancel = false;
evt.RegularFile = v.RegularFile;
evt.Video = v.Video;
ApiLoader.DownloadStarted(this, evt);
if(evt.Cancel)
{
DownloadIt = true;
goto nope;
}
redownload:
try
{
if (v.RegularFile) {
var req=await Http.GetAsync(v.Video.Id);
long? Len=req.Content.Headers.ContentLength;
Uri u = new Uri(v.Video.Id);
string abs = u.AbsolutePath;
string name = "file.bin";
if(abs.Contains("/"))
{
name=abs.Substring(abs.LastIndexOf('/') + 1);
}
if (req.Content.Headers.Contains("Content-Disposition"))
{
name=req.Content.Headers.ContentDisposition.FileName;
}
int fileI=1;
P.Saved.Title = name;
name=GetPath(true, "Download", name);
string filename=name;
while (FileExists(name, ref filename, ref fileI)) ;
long Len2 = long.MaxValue;
if(Len.HasValue)
{
if(Len.Value > 0)
{
Len2 = Len.Value;
}
}
P.Length = Len2;
long Pos = 0;
byte[] buffer = new byte[4096];
int Cycles = 0;
IProgress<double> p = DownloadP;
int CYCLES_BETWEEN_REPORT = 1;
using (var srcFile = await req.Content.ReadAsStreamAsync())
{
using (var destFile = File.Create(filename))
{
int read;
do
{
read = await srcFile.ReadAsync(buffer, 0, buffer.Length);
await destFile.WriteAsync(buffer, 0, read);
Pos += read;
Cycles++;
if (Cycles > CYCLES_BETWEEN_REPORT)
{
Cycles = 0;
p.Report(Pos / Len2);
}
} while (read > 0 && DownloadIt);
}
}
p.Report(1);
}
else
{
switch (v.Resolution)
{
case Resolution.Convert:
string mypath = GetPath(true, "Converted", v.Video.Id + "-vidonly.bkp");
string mypathaudio = GetPath(true, "Audio", v.Video.Id + "incomplete.mp4");
string mypathCompleteAudio = GetPath(true, "Audio", v.Video.Id + ".mp4");
string mypathComplete = GetPath(true, "Converted", v.Video.Id + ".mp4");
string mypathIncompleteConverting = GetPath(true, "Converted", "conv.mkv");
if (Continue(mypathComplete))
{
var s3 = await GetManifest(ytc,v.Video.Id);
var best2 = s3.GetAudioOnlyStreams().GetWithHighestBitrate();
var best = s3.GetVideoOnlyStreams().GetWithHighestVideoQuality();
P.Length = best.Size.Bytes + best2.Size.Bytes;
ProgressTwo p = new ProgressTwo(best.Size.Bytes, best2.Size.Bytes, DownloadP);
using (var destStrm = System.IO.File.Open(mypath, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
long pos = 0;
long len = 0;
using (var srcStrm = await ytc.Videos.Streams.GetAsync(best))
{
len = srcStrm.Length;
pos = destStrm.Length;
IProgress<double> myProgress = p.Video;
if (pos >= len)
{
myProgress.Report(1);
}
/* This is why videos get corrupted */
srcStrm.Seek(destStrm.Length, SeekOrigin.Begin);
destStrm.Seek(destStrm.Length, SeekOrigin.Begin);
byte[] buffer = new byte[4096];
int read = 0;
do
{
read = srcStrm.Read(buffer, 0, buffer.Length);
destStrm.Write(buffer, 0, read);
pos += read;
double myP = (double)pos / (double)len;
myProgress.Report(myP);
}
while (read > 0 && DownloadIt ) ;
}
}
IProgress<double> pv = p.Video;
pv.Report(1);
if (Continue(mypathCompleteAudio))
{
long pos = 0;
long len = 0;
using (var destStrm = File.Open(mypathaudio, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
using (var srcStrm = await ytc.Videos.Streams.GetAsync(best2))
{
len = srcStrm.Length;
pos = destStrm.Length;
IProgress<double> myProgress = p.Audio;
if (pos >= len)
{
myProgress.Report(1);
}
/* This is why videos get corrupted */
srcStrm.Seek(destStrm.Length, SeekOrigin.Begin);
destStrm.Seek(destStrm.Length, SeekOrigin.Begin);
byte[] buffer = new byte[4096];
int read = 0;
do
{
read = srcStrm.Read(buffer, 0, buffer.Length);
destStrm.Write(buffer, 0, read);
pos += read;
double myP = (double)pos / (double)len;
myProgress.Report(myP);
}
while (read > 0 && DownloadIt);
}
}
if(DownloadIt)
{
File.Move(mypathaudio, mypathCompleteAudio);
}
}
IProgress<double> pa = p.Video;
if (DownloadIt)
{
pa.Report(1);
ffmpeg.mux(mypath, mypathCompleteAudio, mypathIncompleteConverting);
File.Move(mypathIncompleteConverting, mypathComplete);
}
}
break;
case Resolution.NoConvert:
string mypath2 = GetPath(true, "NotConverted", v.Video.Id + "incomplete.mp4");
string mypath2Complete = GetPath(true, "NotConverted", v.Video.Id + ".mp4");
if (Continue(mypath2Complete))
{
var s = await GetManifest(ytc,v.Video.Id);
var best = s.GetMuxedStreams().GetWithHighestVideoQuality();
P.Length = best.Size.Bytes;
long pos = 0;
long len = 0;
using (var destStrm = System.IO.File.Open(mypath2, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
using (var srcStrm = await ytc.Videos.Streams.GetAsync(best))
{
len = srcStrm.Length;
pos = destStrm.Length;
IProgress<double> myProgress = DownloadP;
if (pos >= len)
{
myProgress.Report(1);
}
/* This is why videos get corrupted */
srcStrm.Seek(destStrm.Length, SeekOrigin.Begin);
destStrm.Seek(destStrm.Length, SeekOrigin.Begin);
byte[] buffer = new byte[4096];
int read = 0;
do
{
read = srcStrm.Read(buffer, 0, buffer.Length);
destStrm.Write(buffer, 0, read);
pos += read;
double myP = (double)pos / (double)len;
myProgress.Report(myP);
}
while (read > 0 && DownloadIt);
}
}
if (DownloadIt)
{
File.Move(mypath2, mypath2Complete);
}
}
break;
case Resolution.Audio:
string mypath3 = GetPath(true, "Audio", v.Video.Id + "incomplete.mp4");
string mypath3Complete = GetPath(true, "Audio", v.Video.Id + ".mp4");
if (Continue(mypath3Complete))
{
var s2 = await GetManifest(ytc,v.Video.Id);
var best2 = s2.GetAudioOnlyStreams().GetWithHighestBitrate();
P.Length = best2.Size.Bytes;
long pos = 0;
long len = 0;
using (var destStrm = System.IO.File.Open(mypath3, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
using (var srcStrm = await ytc.Videos.Streams.GetAsync(best2))
{
len = srcStrm.Length;
pos = destStrm.Length;
IProgress<double> myProgress = DownloadP;
if (pos >= len)
{
myProgress.Report(1);
}
/* This is why videos get corrupted */
srcStrm.Seek(destStrm.Length, SeekOrigin.Begin);
destStrm.Seek(destStrm.Length, SeekOrigin.Begin);
byte[] buffer = new byte[4096];
int read = 0;
do
{
read = srcStrm.Read(buffer, 0, buffer.Length);
destStrm.Write(buffer, 0, read);
pos += read;
double myP = (double)pos / (double)len;
myProgress.Report(myP);
}
while (read > 0 && DownloadIt);
}
}
if (DownloadIt)
{
File.Move(mypath3, mypath3Complete);
}
}
break;
}
if (DownloadIt)
{
DownloadCompleteEventArgs evt2 = new DownloadCompleteEventArgs();
evt2.RegularFile = v.RegularFile;
evt2.Video = v.Video;
ApiLoader.DownloadComplete(this, evt2);
ffmpeg.on_video_done(v.Video.Id, (int)v.Resolution);
}
}
if (RedownloadIt && !DownloadIt)
{
DownloadIt = true;
goto redownload;
}
DownloadIt = true;
} catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
nope: System.Threading.Thread.Sleep(2000);
}
while (true);
}
internal void _DownloadThumbnail(int w, int h, string id, string tnail)
{
try
{
string p = GetPath(true, "Thumbnails", w.ToString() + 'x' + h.ToString(), id + ".jpg");
if (!File.Exists(p))
{
ffmpeg.download_thumbnail(tnail, p);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
_ = ex;
}
}
internal void _DownloadThumbnail2(int w, int h, string id, string tnail)
{
try
{
string p = GetPath(true, "Thumbnails", w.ToString() + 'x' + h.ToString(), id + ".jpg");
if (!File.Exists(p))
{
using (var f = File.Create(p))
{
Http.GetStreamAsync(tnail);
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
_ = ex;
}
}
public static async Task<List<SavedMedia>> Search(string text, bool downloadThumbs = true)
{
List<SavedMedia> media = new List<SavedMedia>();
try
{
await DL.ytc.Search.GetVideosAsync(text).ForEachAsync((e) =>
{
if (downloadThumbs)
{
foreach (var t in e.Thumbnails)
{
DL._DownloadThumbnail2(t.Resolution.Width, t.Resolution.Height, e.Id, t.Url);
}
}
media.Add(new SavedMedia() { Title = e.Title, Id = e.Id, Kind = InfoType.Video });
});
await DL.ytc.Search.GetPlaylistsAsync(text).ForEachAsync((e) => {
if (downloadThumbs)
{
foreach (var t in e.Thumbnails)
{
DL._DownloadThumbnail2(t.Resolution.Width, t.Resolution.Height, e.Id, t.Url);
}
}
media.Add(new SavedMedia() { Title = e.Title, Id = e.Id, Kind = InfoType.Playlist });
});
await DL.ytc.Search.GetChannelsAsync(text).ForEachAsync((e) => {
if (downloadThumbs)
{
foreach (var t in e.Thumbnails)
{
DL._DownloadThumbnail2(t.Resolution.Width, t.Resolution.Height, e.Id, t.Url);
}
}
media.Add(new SavedMedia() { Title = e.Title, Id = e.Id, Kind = InfoType.Channel });
});
} catch (Exception ex)
{
_ = ex;
}
return media;
}
public static List<SavedVideoObject> GetQueueItems()
{
return DL.Queue;
}
public static void DownloadFile(string id)
{
InfomationQueueItem item = new InfomationQueueItem(new Uri(id));
lock (DL.infoQueue)
{
DL.infoQueue.Insert(0, item);
}
}
public static void DownloadVideo(string id, Resolution res)
{
VideoId? v = VideoId.TryParse(id);
if (v.HasValue)
{
InfomationQueueItem item = new InfomationQueueItem(v.Value, res, true);
lock (DL.infoQueue)
{
DL.infoQueue.Insert(0, item);
}
}
}
public static void DownloadVideoInfo(string id, Resolution res)
{
VideoId? v = VideoId.TryParse(id);
if (v.HasValue)
{
InfomationQueueItem item = new InfomationQueueItem(v.Value, res, false);
lock (DL.infoQueue)
{
DL.infoQueue.Insert(0, item);
}
}
}
public static void DownloadVideo(string v)
{
DownloadVideo(v, Resolution.NoConvert);
}
public static void DownloadCaptions(string id)
{
VideoId? v = VideoId.TryParse(id);
if(v.HasValue)
{
InfomationQueueItem item = new InfomationQueueItem(v.Value);
lock (DL.infoQueue)
{
DL.infoQueue.Insert(0, item);
}
}
}
public static void DownloadPlaylist(string id,Resolution res)
{
PlaylistId? v = PlaylistId.TryParse(id);
if (v.HasValue)
{
InfomationQueueItem item = new InfomationQueueItem(v.Value, res, true);
lock (DL.infoQueue)
{
DL.infoQueue.Insert(0, item);
}
}
}
public static void DownloadPlaylistOnly(string id, Resolution res)
{
PlaylistId? v = PlaylistId.TryParse(id);
if (v.HasValue)
{
InfomationQueueItem item = new InfomationQueueItem(v.Value, res, false);
lock (DL.infoQueue)
{
DL.infoQueue.Insert(0, item);
}
}
}
public static void DownloadPlaylist(string id)
{
DownloadPlaylist(id, Resolution.NoConvert);
}
public static void DownloadChannel(string id,Resolution res)
{
ChannelId? v = ChannelId.TryParse(id);
if (v.HasValue)
{
InfomationQueueItem item = new InfomationQueueItem(v.Value, res, true);
lock (DL.infoQueue)
{
DL.infoQueue.Insert(0, item);
}
}
}
public static void DownloadChannel(string id)
{
DownloadChannel(id, Resolution.NoConvert);
}
public static void DownloadItem(string id)
{
DownloadItem(id, Resolution.NoConvert);
}
public static void DownloadItem(string id,Resolution res)
{
VideoId? vid = VideoId.TryParse(id);
PlaylistId? pid = PlaylistId.TryParse(id);
ChannelId? cid = ChannelId.TryParse(id);
UserName? user = UserName.TryParse(id);
if (id.Length == 11)
{
if (vid.HasValue)
{
DownloadVideo(vid.Value, res); //shall we download video
}
}
else
{
if (pid.HasValue)
{
DownloadPlaylist(pid.Value, res);
}
else if (vid.HasValue)
{
DownloadVideo(vid.Value, res);
}
else if (cid.HasValue)
{
DownloadChannel(cid.Value, res);
}
else if (user.HasValue)
{
DownloadUser(user.Value, res);
}
}
}
public static void DownloadUser(string name, Resolution res)
{
UserName? v=UserName.TryParse(name);
if (v.HasValue)
{
InfomationQueueItem item = new InfomationQueueItem(v.Value, res, true);
lock (DL.infoQueue)
{
DL.infoQueue.Insert(0, item);
}
}
}
public static void DownloadUser(string name)
{
DownloadUser(name, Resolution.NoConvert);
}
public string GetPath(bool createParent,params string[] _path)
{
if (createParent)
{
string dir = GetPath(_path);
try
{
Directory.CreateDirectory(Path.GetDirectoryName(dir));
}catch(Exception ex)
{
Console.WriteLine(ex.Message);
_ = ex;
}
return dir;
}
else
{
return GetPath(_path);
}
}
private string GetPath(params string[] _path)
{
string[] array2 = new string[_path.Length + 1];
array2[0] = StorageLocation;
Array.Copy(_path, 0, array2, 1,_path.Length);
return System.IO.Path.Combine(array2);
}
public static Downloader DL = new Downloader();
}
}