404 lines
15 KiB
C#
404 lines
15 KiB
C#
|
using System;
|
||
|
|
using System.Collections.Generic;
|
||
|
|
using System.IO;
|
||
|
|
using System.Linq;
|
||
|
|
using System.Net.Http;
|
||
|
|
using System.Runtime.CompilerServices;
|
||
|
|
using System.Runtime.Serialization;
|
||
|
|
using System.Security.Cryptography;
|
||
|
|
using System.Text;
|
||
|
|
using System.Threading;
|
||
|
|
using System.Threading.Tasks;
|
||
|
|
using LiteDB;
|
||
|
|
using Newtonsoft.Json;
|
||
|
|
using ShellProgressBar;
|
||
|
|
using Tesses.VirtualFilesystem;
|
||
|
|
using Tesses.VirtualFilesystem.Extensions;
|
||
|
|
using Tesses.WebServer;
|
||
|
|
using System.Net;
|
||
|
|
namespace TessesDedup
|
||
|
|
{
|
||
|
|
public class DedupClient : IDisposable
|
||
|
|
{
|
||
|
|
HttpClient clt;
|
||
|
|
string url;
|
||
|
|
bool ownClient;
|
||
|
|
public bool ShowProgress {get;set;}
|
||
|
|
public DedupClient(HttpClient client, string url,bool ownClient=true,bool showProgress=true)
|
||
|
|
{
|
||
|
|
ShowProgress = showProgress;
|
||
|
|
this.clt = client;
|
||
|
|
this.url = url.TrimEnd('/');
|
||
|
|
this.ownClient = ownClient;
|
||
|
|
}
|
||
|
|
public DedupClient(string url,bool showProgress=true) : this(new HttpClient(),url,true,showProgress)
|
||
|
|
{
|
||
|
|
|
||
|
|
}
|
||
|
|
public async Task LogoutAsync(string key)
|
||
|
|
{
|
||
|
|
Dictionary<string,string> dictionary= new Dictionary<string, string>
|
||
|
|
{
|
||
|
|
{ "key", key }
|
||
|
|
};
|
||
|
|
|
||
|
|
FormUrlEncodedContent formUrlEncodedContent=new FormUrlEncodedContent(dictionary);
|
||
|
|
using(var resp=await clt.PostAsync($"{url}/api/v1/Logout",formUrlEncodedContent))
|
||
|
|
{
|
||
|
|
|
||
|
|
}
|
||
|
|
}
|
||
|
|
public async Task<LoginResult> LoginAsync(string username,string password, string device_name)
|
||
|
|
{
|
||
|
|
Dictionary<string,string> dictionary= new Dictionary<string, string>
|
||
|
|
{
|
||
|
|
{ "username", username },
|
||
|
|
{ "password", password },
|
||
|
|
{ "device_name", device_name }
|
||
|
|
};
|
||
|
|
|
||
|
|
FormUrlEncodedContent formUrlEncodedContent=new FormUrlEncodedContent(dictionary);
|
||
|
|
using(var resp=await clt.PostAsync($"{url}/api/v1/Login",formUrlEncodedContent))
|
||
|
|
{
|
||
|
|
if(resp.IsSuccessStatusCode)
|
||
|
|
{
|
||
|
|
return JsonConvert.DeserializeObject<LoginResult>(await resp.Content.ReadAsStringAsync());
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
return new LoginResult();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
private async Task<int> ReadAsync(Stream strm, byte[] buffer,CancellationToken token=default)
|
||
|
|
{
|
||
|
|
Array.Clear(buffer,0,buffer.Length);
|
||
|
|
int totalRead = 0;
|
||
|
|
while(totalRead < buffer.Length)
|
||
|
|
{
|
||
|
|
int read = Math.Min(buffer.Length-totalRead,DedupStorage.BlockReadSize);
|
||
|
|
if(read == 0)
|
||
|
|
{
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
read=await strm.ReadAsync(buffer,totalRead,read,token);
|
||
|
|
if(token.IsCancellationRequested) return 0;
|
||
|
|
if(read == 0)
|
||
|
|
{
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
totalRead+=read;
|
||
|
|
}
|
||
|
|
return totalRead;
|
||
|
|
}
|
||
|
|
private async Task<bool> HasBlockAsync(string authkey,string hash,CancellationToken token=default)
|
||
|
|
{
|
||
|
|
HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Head,$"{url}/api/v1/Block?hash={hash}");
|
||
|
|
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);
|
||
|
|
using(var req=await clt.SendAsync(request,token))
|
||
|
|
return req.StatusCode == System.Net.HttpStatusCode.OK;
|
||
|
|
}
|
||
|
|
private async Task WriteBlockAsync(string authkey,string hash,byte[] buffer,CancellationToken token=default)
|
||
|
|
{
|
||
|
|
HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Put,$"{url}/api/v1/Block");
|
||
|
|
request.Content = new ByteArrayContent(buffer);
|
||
|
|
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);
|
||
|
|
using(var req=await clt.SendAsync(request,token))
|
||
|
|
{
|
||
|
|
if(req.StatusCode == System.Net.HttpStatusCode.OK)
|
||
|
|
{
|
||
|
|
string server_hash=await req.Content.ReadAsStringAsync();
|
||
|
|
if(server_hash != hash)
|
||
|
|
throw new BackupFailedException(server_hash,hash);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
public async Task<bool> BackupAsync(string authkey, IVirtualFilesystem fs,string tag,CancellationToken token=default)
|
||
|
|
{
|
||
|
|
var dt = DateTime.Now;
|
||
|
|
var fse=await BackupPathAsync(authkey,fs,Special.Root,token);
|
||
|
|
if(token.IsCancellationRequested) return false;
|
||
|
|
Backup bkp=new Backup();
|
||
|
|
bkp.Id = 0;
|
||
|
|
bkp.Tag = tag;
|
||
|
|
bkp.CreationDate = dt;
|
||
|
|
bkp.Root = fse;
|
||
|
|
|
||
|
|
await WriteBackupAsync(authkey,bkp,token);
|
||
|
|
if(token.IsCancellationRequested) return false;
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
private async Task WriteBackupAsync(string authkey, Backup bkp,CancellationToken token=default)
|
||
|
|
{
|
||
|
|
var str=JsonConvert.SerializeObject(bkp);
|
||
|
|
HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Put,$"{url}/api/v1/Backup");
|
||
|
|
request.Content = new StringContent(str,Encoding.UTF8,"application/json");
|
||
|
|
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);
|
||
|
|
|
||
|
|
using(var c=await clt.SendAsync(request,token))
|
||
|
|
{
|
||
|
|
c.EnsureSuccessStatusCode();
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
private async Task<FilesystemEntry> BackupPathAsync(string authkey,IVirtualFilesystem fs,UnixPath path,CancellationToken token=default)
|
||
|
|
{
|
||
|
|
if(await fs.SymlinkExistsAsync(path))
|
||
|
|
{
|
||
|
|
return new FilesystemEntry(){Name = path.Name, Type = FilesystemEntryType.Symlink, PointsTo = (await fs.ReadLinkAsync(path,token)).Path};
|
||
|
|
}
|
||
|
|
else if(await fs.DirectoryExistsAsync(path))
|
||
|
|
{
|
||
|
|
FilesystemEntry entry=new FilesystemEntry();
|
||
|
|
entry.Name = path.Name;
|
||
|
|
entry.Type = FilesystemEntryType.Dir;
|
||
|
|
await foreach(var ent in fs.EnumerateFileSystemEntriesAsync(path))
|
||
|
|
{
|
||
|
|
if(token.IsCancellationRequested) return entry;
|
||
|
|
entry.Entries.Add(await BackupPathAsync(authkey,fs,ent,token));
|
||
|
|
}
|
||
|
|
return entry;
|
||
|
|
}
|
||
|
|
else if(await fs.FileExistsAsync(path))
|
||
|
|
{
|
||
|
|
FilesystemEntry entry = new FilesystemEntry();
|
||
|
|
entry.Name = path.Name;
|
||
|
|
entry.Type = FilesystemEntryType.File;
|
||
|
|
entry.Length = 0;
|
||
|
|
entry.Hashes=new List<string>();
|
||
|
|
|
||
|
|
using(var strm = await fs.OpenReadAsync(path))
|
||
|
|
{
|
||
|
|
if(strm.CanSeek) entry.Length = strm.Length;
|
||
|
|
await foreach(var hash in BackupFileAsync($"/{string.Join("/",path.Parts)}",authkey,strm,(len)=>entry.Length=len,token))
|
||
|
|
{
|
||
|
|
if(token.IsCancellationRequested) return entry;
|
||
|
|
entry.Hashes.Add(hash);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return entry;
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
return new FilesystemEntry(){Length=0, Type = FilesystemEntryType.File};
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
public async IAsyncEnumerable<Backup> ListBackupsAsync(string authkey)
|
||
|
|
{
|
||
|
|
HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Get,$"{url}/api/v1/Backup");
|
||
|
|
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);
|
||
|
|
var resp=await clt.SendAsync(request);
|
||
|
|
resp.EnsureSuccessStatusCode();
|
||
|
|
foreach(var item in JsonConvert.DeserializeObject<List<Backup>>(await resp.Content.ReadAsStringAsync()))
|
||
|
|
{
|
||
|
|
if(item != null) yield return item;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
public async Task<Backup> GetBackup(string authkey,long id,bool noHashes=true)
|
||
|
|
{
|
||
|
|
HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Get,$"{url}/api/v1/Backup?id={id}&noHashes={noHashes}");
|
||
|
|
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);
|
||
|
|
var resp=await clt.SendAsync(request);
|
||
|
|
resp.EnsureSuccessStatusCode();
|
||
|
|
var bkp=JsonConvert.DeserializeObject<Backup>(await resp.Content.ReadAsStringAsync());
|
||
|
|
return bkp;
|
||
|
|
}
|
||
|
|
private async Task RestoreAsync(string key,long id,FilesystemEntry entry,UnixPath srcPath,IVirtualFilesystem fs,UnixPath destPath,CancellationToken token=default)
|
||
|
|
{
|
||
|
|
if(entry.Type == FilesystemEntryType.Dir)
|
||
|
|
{
|
||
|
|
await fs.CreateDirectoryAsync(destPath,token);
|
||
|
|
foreach(var ent in entry.Entries)
|
||
|
|
{
|
||
|
|
if(token.IsCancellationRequested) return;
|
||
|
|
await RestoreAsync(key,id,ent,srcPath/ent.Name,fs,destPath/ent.Name,token);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else if(entry.Type == FilesystemEntryType.File)
|
||
|
|
{
|
||
|
|
|
||
|
|
using(var file = await fs.OpenAsync(destPath,FileMode.OpenOrCreate,FileAccess.ReadWrite,FileShare.None))
|
||
|
|
{
|
||
|
|
ProgressBar bar=null;
|
||
|
|
IProgress<double> p=null;
|
||
|
|
if(ShowProgress && file.CanSeek)
|
||
|
|
{
|
||
|
|
var options = new ProgressBarOptions
|
||
|
|
{
|
||
|
|
ProgressCharacter = '─',
|
||
|
|
ProgressBarOnBottom = true
|
||
|
|
};
|
||
|
|
bar=new ProgressBar(10000,$"Restoring file {destPath.Path}");
|
||
|
|
p = bar.AsProgress<double>();
|
||
|
|
}else if(ShowProgress)
|
||
|
|
{
|
||
|
|
Console.WriteLine("Stream is not seekable");
|
||
|
|
}
|
||
|
|
|
||
|
|
await RestoreFileAsync(key,id,srcPath.Path,file,new Progress<long>(e=>{
|
||
|
|
if(ShowProgress && file.CanSeek)
|
||
|
|
{
|
||
|
|
try{p.Report((double)e/(double)file.Length);}
|
||
|
|
catch(Exception ex)
|
||
|
|
{
|
||
|
|
_=ex;
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
}),token);
|
||
|
|
|
||
|
|
if(ShowProgress && file.CanSeek)
|
||
|
|
{
|
||
|
|
bar.Dispose();
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
else if(entry.Type == FilesystemEntryType.Symlink) {
|
||
|
|
await fs.CreateSymlinkAsync(new UnixPath(entry.PointsTo),destPath,token);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
public async Task RestoreAsync(string key,long id,UnixPath srcPath,IVirtualFilesystem fs,UnixPath destPath,CancellationToken token=default)
|
||
|
|
{
|
||
|
|
var bkp=await GetBackup(key,id);
|
||
|
|
await RestoreAsync(key,id,bkp.GetEntryFromPath(srcPath),srcPath,fs,destPath,token);
|
||
|
|
}
|
||
|
|
private async Task RestoreFileAsync(string key,long id, string path,Stream strm,IProgress<long> progress=null,CancellationToken token=default)
|
||
|
|
{
|
||
|
|
var _path = $"{url}/api/v1/Download?id={id}&access_key={WebUtility.UrlEncode(key)}&id={id}&path={WebUtility.UrlEncode(path)}";
|
||
|
|
byte[] buffer = new byte[1024];
|
||
|
|
|
||
|
|
long offset=0;
|
||
|
|
|
||
|
|
HttpRequestMessage requestMessage=new HttpRequestMessage(HttpMethod.Get,_path);
|
||
|
|
|
||
|
|
if(strm.CanSeek && strm.Length > 0)
|
||
|
|
{
|
||
|
|
strm.Position = strm.Length;
|
||
|
|
offset=strm.Position;
|
||
|
|
|
||
|
|
requestMessage.Headers.Range=new System.Net.Http.Headers.RangeHeaderValue(strm.Length,null);
|
||
|
|
}
|
||
|
|
if(token.IsCancellationRequested) return;
|
||
|
|
using(var req = await clt.SendAsync(requestMessage,token))
|
||
|
|
{
|
||
|
|
if(token.IsCancellationRequested) return;
|
||
|
|
if(req.StatusCode == System.Net.HttpStatusCode.RequestedRangeNotSatisfiable) return;
|
||
|
|
req.EnsureSuccessStatusCode();
|
||
|
|
int read = 0;
|
||
|
|
using(var srcStrm = await req.Content.ReadAsStreamAsync())
|
||
|
|
do {
|
||
|
|
read = await srcStrm.ReadAsync(buffer,0,buffer.Length,token);
|
||
|
|
if(token.IsCancellationRequested) return;
|
||
|
|
await strm.WriteAsync(buffer,0,read,token);
|
||
|
|
if(token.IsCancellationRequested) return;
|
||
|
|
offset += read;
|
||
|
|
progress?.Report(offset);
|
||
|
|
} while(read > 0);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
private async IAsyncEnumerable<string> BackupFileAsync(string name,string authkey,Stream file,Action<long> length,[EnumeratorCancellation]CancellationToken token=default)
|
||
|
|
{
|
||
|
|
if(file.CanSeek && file.Length == 0) yield break;
|
||
|
|
byte[] buffer = new byte[DedupStorage.BlockLength];
|
||
|
|
int read=0;
|
||
|
|
long totalLength=0;
|
||
|
|
ProgressBar bar=null;
|
||
|
|
IProgress<double> p=null;
|
||
|
|
if(ShowProgress && file.CanSeek)
|
||
|
|
{
|
||
|
|
var options = new ProgressBarOptions
|
||
|
|
{
|
||
|
|
ProgressCharacter = '─',
|
||
|
|
ProgressBarOnBottom = true
|
||
|
|
};
|
||
|
|
bar=new ProgressBar(10000,$"Backing up file {name}");
|
||
|
|
p = bar.AsProgress<double>();
|
||
|
|
}
|
||
|
|
else if(ShowProgress)
|
||
|
|
{
|
||
|
|
Console.WriteLine("Stream is not seekable");
|
||
|
|
}
|
||
|
|
do{
|
||
|
|
read=await ReadAsync(file,buffer,token);
|
||
|
|
if(read == 0) yield break;
|
||
|
|
if(token.IsCancellationRequested) yield break;
|
||
|
|
string hash=DedupStorage.Sha512Hash(buffer);
|
||
|
|
yield return hash;
|
||
|
|
if(!await HasBlockAsync(authkey,hash,token))
|
||
|
|
{
|
||
|
|
await WriteBlockAsync(authkey,hash,buffer,token);
|
||
|
|
if(token.IsCancellationRequested) yield break;
|
||
|
|
}
|
||
|
|
|
||
|
|
totalLength += read;
|
||
|
|
|
||
|
|
if(ShowProgress)
|
||
|
|
{
|
||
|
|
if(file.CanSeek)
|
||
|
|
{
|
||
|
|
double progress = totalLength / (double)file.Length;
|
||
|
|
p?.Report(progress);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
} while(read == buffer.Length);
|
||
|
|
|
||
|
|
if(ShowProgress)
|
||
|
|
bar.Dispose();
|
||
|
|
|
||
|
|
if(!file.CanSeek)
|
||
|
|
length(totalLength);
|
||
|
|
}
|
||
|
|
|
||
|
|
public void Dispose()
|
||
|
|
{
|
||
|
|
if(ownClient)
|
||
|
|
this.clt.Dispose();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
[Serializable]
|
||
|
|
internal class BackupFailedException : Exception
|
||
|
|
{
|
||
|
|
|
||
|
|
public BackupFailedException()
|
||
|
|
{
|
||
|
|
}
|
||
|
|
|
||
|
|
public BackupFailedException(string message) : base(message)
|
||
|
|
{
|
||
|
|
}
|
||
|
|
|
||
|
|
public BackupFailedException(string server_hash, string hash) : base($"The server hash \"{server_hash}\" does not match client hash \"{hash}\"")
|
||
|
|
{
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
public BackupFailedException(string message, Exception innerException) : base(message, innerException)
|
||
|
|
{
|
||
|
|
}
|
||
|
|
|
||
|
|
protected BackupFailedException(SerializationInfo info, StreamingContext context) : base(info, context)
|
||
|
|
{
|
||
|
|
}
|
||
|
|
}
|
||
|
|
public class LoginResult
|
||
|
|
{
|
||
|
|
[JsonProperty("success")]
|
||
|
|
public bool Success {get;set;}=false;
|
||
|
|
|
||
|
|
[JsonProperty("key")]
|
||
|
|
public string Key {get;set;}="";
|
||
|
|
}
|
||
|
|
}
|