tesses-backup/TessesDedup/Client.cs

414 lines
16 KiB
C#
Raw Normal View History

2024-07-23 03:49:40 +00:00
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using LiteDB;
using Newtonsoft.Json;
using ShellProgressBar;
using Tesses.VirtualFilesystem;
using Tesses.VirtualFilesystem.Extensions;
using Tesses.WebServer;
using System.Net;
using System.Security;
2024-07-23 03:49:40 +00:00
namespace TessesDedup
{
public class DedupClient : IDisposable
{
HttpClient clt;
string url;
bool ownClient;
public bool ShowProgress {get;set;}
public bool IgnoreErrors {get;set;}
public DedupClient(HttpClient client, string url,bool ownClient=true,bool showProgress=true,bool ignoreErrors=true)
2024-07-23 03:49:40 +00:00
{
ShowProgress = showProgress;
IgnoreErrors = ignoreErrors;
2024-07-23 03:49:40 +00:00
this.clt = client;
this.url = url.TrimEnd('/');
this.ownClient = ownClient;
2024-07-23 03:49:40 +00:00
}
public DedupClient(string url,bool showProgress=true,bool ignoreErrors=true) : this(new HttpClient(),url,true,showProgress,ignoreErrors)
2024-07-23 03:49:40 +00:00
{
}
public async Task LogoutAsync(string key)
{
Dictionary<string,string> dictionary= new Dictionary<string, string>
{
{ "key", key }
};
FormUrlEncodedContent formUrlEncodedContent=new FormUrlEncodedContent(dictionary);
using(var resp=await clt.PostAsync($"{url}/api/v1/Logout",formUrlEncodedContent))
{
}
}
public async Task<LoginResult> LoginAsync(string username,string password, string device_name)
{
Dictionary<string,string> dictionary= new Dictionary<string, string>
{
{ "username", username },
{ "password", password },
{ "device_name", device_name }
};
FormUrlEncodedContent formUrlEncodedContent=new FormUrlEncodedContent(dictionary);
using(var resp=await clt.PostAsync($"{url}/api/v1/Login",formUrlEncodedContent))
{
if(resp.IsSuccessStatusCode)
{
return JsonConvert.DeserializeObject<LoginResult>(await resp.Content.ReadAsStringAsync());
}
else
{
return new LoginResult();
}
}
}
private async Task<int> ReadAsync(Stream strm, byte[] buffer,CancellationToken token=default)
{
Array.Clear(buffer,0,buffer.Length);
int totalRead = 0;
while(totalRead < buffer.Length)
{
int read = Math.Min(buffer.Length-totalRead,DedupStorage.BlockReadSize);
if(read == 0)
{
break;
}
read=await strm.ReadAsync(buffer,totalRead,read,token);
if(token.IsCancellationRequested) return 0;
if(read == 0)
{
break;
}
totalRead+=read;
}
return totalRead;
}
private async Task<bool> HasBlockAsync(string authkey,string hash,CancellationToken token=default)
{
HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Head,$"{url}/api/v1/Block?hash={hash}");
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);
using(var req=await clt.SendAsync(request,token))
return req.StatusCode == System.Net.HttpStatusCode.OK;
}
private async Task WriteBlockAsync(string authkey,string hash,byte[] buffer,CancellationToken token=default)
{
HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Put,$"{url}/api/v1/Block");
request.Content = new ByteArrayContent(buffer);
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);
using(var req=await clt.SendAsync(request,token))
{
if(req.StatusCode == System.Net.HttpStatusCode.OK)
{
string server_hash=await req.Content.ReadAsStringAsync();
if(server_hash != hash)
throw new BackupFailedException(server_hash,hash);
}
}
}
public async Task<bool> BackupAsync(string authkey, IVirtualFilesystem fs,string tag,CancellationToken token=default)
{
var dt = DateTime.Now;
var fse=await BackupPathAsync(authkey,fs,Special.Root,token);
if(token.IsCancellationRequested) return false;
Backup bkp=new Backup();
bkp.Id = 0;
bkp.Tag = tag;
bkp.CreationDate = dt;
bkp.Root = fse;
await WriteBackupAsync(authkey,bkp,token);
if(token.IsCancellationRequested) return false;
return true;
}
private async Task WriteBackupAsync(string authkey, Backup bkp,CancellationToken token=default)
{
var str=JsonConvert.SerializeObject(bkp);
HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Put,$"{url}/api/v1/Backup");
request.Content = new StringContent(str,Encoding.UTF8,"application/json");
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);
using(var c=await clt.SendAsync(request,token))
{
c.EnsureSuccessStatusCode();
}
}
private async Task<FilesystemEntry> BackupPathAsync(string authkey,IVirtualFilesystem fs,UnixPath path,CancellationToken token=default)
{
if(await fs.SymlinkExistsAsync(path))
{
return new FilesystemEntry(){Name = path.Name, Type = FilesystemEntryType.Symlink, PointsTo = (await fs.ReadLinkAsync(path,token)).Path};
}
else if(await fs.DirectoryExistsAsync(path))
{
FilesystemEntry entry=new FilesystemEntry();
entry.Name = path.Name;
entry.Type = FilesystemEntryType.Dir;
await foreach(var ent in fs.EnumerateFileSystemEntriesAsync(path))
{
if(token.IsCancellationRequested) return entry;
try{
2024-07-23 03:49:40 +00:00
entry.Entries.Add(await BackupPathAsync(authkey,fs,ent,token));
}catch(Exception ex)
{
if(!IgnoreErrors)
throw;
}
2024-07-23 03:49:40 +00:00
}
return entry;
}
else if(await fs.FileExistsAsync(path))
{
FilesystemEntry entry = new FilesystemEntry();
entry.Name = path.Name;
entry.Type = FilesystemEntryType.File;
entry.Length = 0;
entry.Hashes=new List<string>();
using(var strm = await fs.OpenReadAsync(path))
{
if(strm.CanSeek) entry.Length = strm.Length;
await foreach(var hash in BackupFileAsync($"/{string.Join("/",path.Parts)}",authkey,strm,(len)=>entry.Length=len,token))
{
if(token.IsCancellationRequested) return entry;
entry.Hashes.Add(hash);
}
}
return entry;
}
else
{
return new FilesystemEntry(){Length=0, Type = FilesystemEntryType.File};
}
}
public async IAsyncEnumerable<Backup> ListBackupsAsync(string authkey)
{
HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Get,$"{url}/api/v1/Backup");
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);
var resp=await clt.SendAsync(request);
resp.EnsureSuccessStatusCode();
foreach(var item in JsonConvert.DeserializeObject<List<Backup>>(await resp.Content.ReadAsStringAsync()))
{
if(item != null) yield return item;
}
}
public async Task<Backup> GetBackup(string authkey,long id,bool noHashes=true)
{
HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Get,$"{url}/api/v1/Backup?id={id}&noHashes={noHashes}");
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);
var resp=await clt.SendAsync(request);
resp.EnsureSuccessStatusCode();
var bkp=JsonConvert.DeserializeObject<Backup>(await resp.Content.ReadAsStringAsync());
return bkp;
}
private async Task RestoreAsync(string key,long id,FilesystemEntry entry,UnixPath srcPath,IVirtualFilesystem fs,UnixPath destPath,CancellationToken token=default)
{
if(entry.Type == FilesystemEntryType.Dir)
{
await fs.CreateDirectoryAsync(destPath,token);
foreach(var ent in entry.Entries)
{
if(token.IsCancellationRequested) return;
await RestoreAsync(key,id,ent,srcPath/ent.Name,fs,destPath/ent.Name,token);
}
}
else if(entry.Type == FilesystemEntryType.File)
{
using(var file = await fs.OpenAsync(destPath,FileMode.OpenOrCreate,FileAccess.ReadWrite,FileShare.None))
{
ProgressBar bar=null;
IProgress<double> p=null;
if(ShowProgress && file.CanSeek)
{
var options = new ProgressBarOptions
{
ProgressCharacter = '─',
ProgressBarOnBottom = true
};
bar=new ProgressBar(10000,$"Restoring file {destPath.Path}");
p = bar.AsProgress<double>();
}else if(ShowProgress)
{
Console.WriteLine("Stream is not seekable");
}
await RestoreFileAsync(key,id,srcPath.Path,file,new Progress<long>(e=>{
if(ShowProgress && file.CanSeek)
{
try{p.Report((double)e/(double)file.Length);}
catch(Exception ex)
{
_=ex;
}
}
}),token);
if(ShowProgress && file.CanSeek)
{
bar.Dispose();
}
}
}
else if(entry.Type == FilesystemEntryType.Symlink) {
await fs.CreateSymlinkAsync(new UnixPath(entry.PointsTo),destPath,token);
}
}
public async Task RestoreAsync(string key,long id,UnixPath srcPath,IVirtualFilesystem fs,UnixPath destPath,CancellationToken token=default)
{
var bkp=await GetBackup(key,id);
await RestoreAsync(key,id,bkp.GetEntryFromPath(srcPath),srcPath,fs,destPath,token);
}
private async Task RestoreFileAsync(string key,long id, string path,Stream strm,IProgress<long> progress=null,CancellationToken token=default)
{
var _path = $"{url}/api/v1/Download?id={id}&access_key={WebUtility.UrlEncode(key)}&id={id}&path={WebUtility.UrlEncode(path)}";
byte[] buffer = new byte[1024];
long offset=0;
HttpRequestMessage requestMessage=new HttpRequestMessage(HttpMethod.Get,_path);
if(strm.CanSeek && strm.Length > 0)
{
strm.Position = strm.Length;
offset=strm.Position;
requestMessage.Headers.Range=new System.Net.Http.Headers.RangeHeaderValue(strm.Length,null);
}
if(token.IsCancellationRequested) return;
using(var req = await clt.SendAsync(requestMessage,token))
{
if(token.IsCancellationRequested) return;
if(req.StatusCode == System.Net.HttpStatusCode.RequestedRangeNotSatisfiable) return;
req.EnsureSuccessStatusCode();
int read = 0;
using(var srcStrm = await req.Content.ReadAsStreamAsync())
do {
read = await srcStrm.ReadAsync(buffer,0,buffer.Length,token);
if(token.IsCancellationRequested) return;
await strm.WriteAsync(buffer,0,read,token);
if(token.IsCancellationRequested) return;
offset += read;
progress?.Report(offset);
} while(read > 0);
}
}
private async IAsyncEnumerable<string> BackupFileAsync(string name,string authkey,Stream file,Action<long> length,[EnumeratorCancellation]CancellationToken token=default)
{
if(file.CanSeek && file.Length == 0) yield break;
byte[] buffer = new byte[DedupStorage.BlockLength];
int read=0;
long totalLength=0;
ProgressBar bar=null;
IProgress<double> p=null;
if(ShowProgress && file.CanSeek)
{
var options = new ProgressBarOptions
{
ProgressCharacter = '─',
ProgressBarOnBottom = true
};
bar=new ProgressBar(10000,$"Backing up file {name}");
p = bar.AsProgress<double>();
}
else if(ShowProgress)
{
Console.WriteLine("Stream is not seekable");
}
do{
read=await ReadAsync(file,buffer,token);
if(read == 0) yield break;
if(token.IsCancellationRequested) yield break;
string hash=DedupStorage.Sha512Hash(buffer);
yield return hash;
if(!await HasBlockAsync(authkey,hash,token))
{
await WriteBlockAsync(authkey,hash,buffer,token);
if(token.IsCancellationRequested) yield break;
}
totalLength += read;
if(ShowProgress)
{
if(file.CanSeek)
{
double progress = totalLength / (double)file.Length;
p?.Report(progress);
}
}
} while(read == buffer.Length);
if(ShowProgress)
bar.Dispose();
if(!file.CanSeek)
length(totalLength);
}
public void Dispose()
{
if(ownClient)
this.clt.Dispose();
}
}
[Serializable]
internal class BackupFailedException : Exception
{
public BackupFailedException()
{
}
public BackupFailedException(string message) : base(message)
{
}
public BackupFailedException(string server_hash, string hash) : base($"The server hash \"{server_hash}\" does not match client hash \"{hash}\"")
{
}
public BackupFailedException(string message, Exception innerException) : base(message, innerException)
{
}
protected BackupFailedException(SerializationInfo info, StreamingContext context) : base(info, context)
{
}
}
public class LoginResult
{
[JsonProperty("success")]
public bool Success {get;set;}=false;
[JsonProperty("key")]
public string Key {get;set;}="";
}
}