using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; using System.Runtime.CompilerServices; using System.Runtime.Serialization; using System.Security.Cryptography; using System.Text; using System.Threading; using System.Threading.Tasks; using LiteDB; using Newtonsoft.Json; using ShellProgressBar; using Tesses.VirtualFilesystem; using Tesses.VirtualFilesystem.Extensions; using Tesses.WebServer; using System.Net; using System.Security; namespace TessesDedup { public class DedupClient : IDisposable { HttpClient clt; string url; bool ownClient; public bool ShowProgress {get;set;} public bool IgnoreErrors {get;set;} public DedupClient(HttpClient client, string url,bool ownClient=true,bool showProgress=true,bool ignoreErrors=true) { ShowProgress = showProgress; IgnoreErrors = ignoreErrors; this.clt = client; this.url = url.TrimEnd('/'); this.ownClient = ownClient; } public DedupClient(string url,bool showProgress=true,bool ignoreErrors=true) : this(new HttpClient(),url,true,showProgress,ignoreErrors) { } public async Task LogoutAsync(string key) { Dictionary dictionary= new Dictionary { { "key", key } }; FormUrlEncodedContent formUrlEncodedContent=new FormUrlEncodedContent(dictionary); using(var resp=await clt.PostAsync($"{url}/api/v1/Logout",formUrlEncodedContent)) { } } public async Task LoginAsync(string username,string password, string device_name) { Dictionary dictionary= new Dictionary { { "username", username }, { "password", password }, { "device_name", device_name } }; FormUrlEncodedContent formUrlEncodedContent=new FormUrlEncodedContent(dictionary); using(var resp=await clt.PostAsync($"{url}/api/v1/Login",formUrlEncodedContent)) { if(resp.IsSuccessStatusCode) { return JsonConvert.DeserializeObject(await resp.Content.ReadAsStringAsync()); } else { return new LoginResult(); } } } private async Task ReadAsync(Stream strm, byte[] buffer,CancellationToken token=default) { Array.Clear(buffer,0,buffer.Length); int totalRead = 0; while(totalRead < buffer.Length) { int read = Math.Min(buffer.Length-totalRead,DedupStorage.BlockReadSize); if(read == 0) { break; } read=await strm.ReadAsync(buffer,totalRead,read,token); if(token.IsCancellationRequested) return 0; if(read == 0) { break; } totalRead+=read; } return totalRead; } private async Task HasBlockAsync(string authkey,string hash,CancellationToken token=default) { HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Head,$"{url}/api/v1/Block?hash={hash}"); request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey); using(var req=await clt.SendAsync(request,token)) return req.StatusCode == System.Net.HttpStatusCode.OK; } private async Task WriteBlockAsync(string authkey,string hash,byte[] buffer,CancellationToken token=default) { HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Put,$"{url}/api/v1/Block"); request.Content = new ByteArrayContent(buffer); request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey); using(var req=await clt.SendAsync(request,token)) { if(req.StatusCode == System.Net.HttpStatusCode.OK) { string server_hash=await req.Content.ReadAsStringAsync(); if(server_hash != hash) throw new BackupFailedException(server_hash,hash); } } } public async Task BackupAsync(string authkey, IVirtualFilesystem fs,string tag,CancellationToken token=default) { var dt = DateTime.Now; var fse=await BackupPathAsync(authkey,fs,Special.Root,token); if(token.IsCancellationRequested) return false; Backup bkp=new Backup(); bkp.Id = 0; bkp.Tag = tag; bkp.CreationDate = dt; bkp.Root = fse; await WriteBackupAsync(authkey,bkp,token); if(token.IsCancellationRequested) return false; return true; } private async Task WriteBackupAsync(string authkey, Backup bkp,CancellationToken token=default) { var str=JsonConvert.SerializeObject(bkp); HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Put,$"{url}/api/v1/Backup"); request.Content = new StringContent(str,Encoding.UTF8,"application/json"); request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey); using(var c=await clt.SendAsync(request,token)) { c.EnsureSuccessStatusCode(); } } private async Task BackupPathAsync(string authkey,IVirtualFilesystem fs,UnixPath path,CancellationToken token=default) { if(await fs.SymlinkExistsAsync(path)) { return new FilesystemEntry(){Name = path.Name, Type = FilesystemEntryType.Symlink, PointsTo = (await fs.ReadLinkAsync(path,token)).Path}; } else if(await fs.DirectoryExistsAsync(path)) { FilesystemEntry entry=new FilesystemEntry(); entry.Name = path.Name; entry.Type = FilesystemEntryType.Dir; await foreach(var ent in fs.EnumerateFileSystemEntriesAsync(path)) { if(token.IsCancellationRequested) return entry; try{ entry.Entries.Add(await BackupPathAsync(authkey,fs,ent,token)); }catch(Exception ex) { if(!IgnoreErrors) throw; } } return entry; } else if(await fs.FileExistsAsync(path)) { FilesystemEntry entry = new FilesystemEntry(); entry.Name = path.Name; entry.Type = FilesystemEntryType.File; entry.Length = 0; entry.Hashes=new List(); using(var strm = await fs.OpenReadAsync(path)) { if(strm.CanSeek) entry.Length = strm.Length; await foreach(var hash in BackupFileAsync($"/{string.Join("/",path.Parts)}",authkey,strm,(len)=>entry.Length=len,token)) { if(token.IsCancellationRequested) return entry; entry.Hashes.Add(hash); } } return entry; } else { return new FilesystemEntry(){Length=0, Type = FilesystemEntryType.File}; } } public async IAsyncEnumerable ListBackupsAsync(string authkey) { HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Get,$"{url}/api/v1/Backup"); request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey); var resp=await clt.SendAsync(request); resp.EnsureSuccessStatusCode(); foreach(var item in JsonConvert.DeserializeObject>(await resp.Content.ReadAsStringAsync())) { if(item != null) yield return item; } } public async Task GetBackup(string authkey,long id,bool noHashes=true) { HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Get,$"{url}/api/v1/Backup?id={id}&noHashes={noHashes}"); request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey); var resp=await clt.SendAsync(request); resp.EnsureSuccessStatusCode(); var bkp=JsonConvert.DeserializeObject(await resp.Content.ReadAsStringAsync()); return bkp; } private async Task RestoreAsync(string key,long id,FilesystemEntry entry,UnixPath srcPath,IVirtualFilesystem fs,UnixPath destPath,CancellationToken token=default) { if(entry.Type == FilesystemEntryType.Dir) { await fs.CreateDirectoryAsync(destPath,token); foreach(var ent in entry.Entries) { if(token.IsCancellationRequested) return; await RestoreAsync(key,id,ent,srcPath/ent.Name,fs,destPath/ent.Name,token); } } else if(entry.Type == FilesystemEntryType.File) { using(var file = await fs.OpenAsync(destPath,FileMode.OpenOrCreate,FileAccess.ReadWrite,FileShare.None)) { ProgressBar bar=null; IProgress p=null; if(ShowProgress && file.CanSeek) { var options = new ProgressBarOptions { ProgressCharacter = '─', ProgressBarOnBottom = true }; bar=new ProgressBar(10000,$"Restoring file {destPath.Path}"); p = bar.AsProgress(); }else if(ShowProgress) { Console.WriteLine("Stream is not seekable"); } await RestoreFileAsync(key,id,srcPath.Path,file,new Progress(e=>{ if(ShowProgress && file.CanSeek) { try{p.Report((double)e/(double)file.Length);} catch(Exception ex) { _=ex; } } }),token); if(ShowProgress && file.CanSeek) { bar.Dispose(); } } } else if(entry.Type == FilesystemEntryType.Symlink) { await fs.CreateSymlinkAsync(new UnixPath(entry.PointsTo),destPath,token); } } public async Task RestoreAsync(string key,long id,UnixPath srcPath,IVirtualFilesystem fs,UnixPath destPath,CancellationToken token=default) { var bkp=await GetBackup(key,id); await RestoreAsync(key,id,bkp.GetEntryFromPath(srcPath),srcPath,fs,destPath,token); } private async Task RestoreFileAsync(string key,long id, string path,Stream strm,IProgress progress=null,CancellationToken token=default) { var _path = $"{url}/api/v1/Download?id={id}&access_key={WebUtility.UrlEncode(key)}&id={id}&path={WebUtility.UrlEncode(path)}"; byte[] buffer = new byte[1024]; long offset=0; HttpRequestMessage requestMessage=new HttpRequestMessage(HttpMethod.Get,_path); if(strm.CanSeek && strm.Length > 0) { strm.Position = strm.Length; offset=strm.Position; requestMessage.Headers.Range=new System.Net.Http.Headers.RangeHeaderValue(strm.Length,null); } if(token.IsCancellationRequested) return; using(var req = await clt.SendAsync(requestMessage,token)) { if(token.IsCancellationRequested) return; if(req.StatusCode == System.Net.HttpStatusCode.RequestedRangeNotSatisfiable) return; req.EnsureSuccessStatusCode(); int read = 0; using(var srcStrm = await req.Content.ReadAsStreamAsync()) do { read = await srcStrm.ReadAsync(buffer,0,buffer.Length,token); if(token.IsCancellationRequested) return; await strm.WriteAsync(buffer,0,read,token); if(token.IsCancellationRequested) return; offset += read; progress?.Report(offset); } while(read > 0); } } private async IAsyncEnumerable BackupFileAsync(string name,string authkey,Stream file,Action length,[EnumeratorCancellation]CancellationToken token=default) { if(file.CanSeek && file.Length == 0) yield break; byte[] buffer = new byte[DedupStorage.BlockLength]; int read=0; long totalLength=0; ProgressBar bar=null; IProgress p=null; if(ShowProgress && file.CanSeek) { var options = new ProgressBarOptions { ProgressCharacter = '─', ProgressBarOnBottom = true }; bar=new ProgressBar(10000,$"Backing up file {name}"); p = bar.AsProgress(); } else if(ShowProgress) { Console.WriteLine("Stream is not seekable"); } do{ read=await ReadAsync(file,buffer,token); if(read == 0) yield break; if(token.IsCancellationRequested) yield break; string hash=DedupStorage.Sha512Hash(buffer); yield return hash; if(!await HasBlockAsync(authkey,hash,token)) { await WriteBlockAsync(authkey,hash,buffer,token); if(token.IsCancellationRequested) yield break; } totalLength += read; if(ShowProgress) { if(file.CanSeek) { double progress = totalLength / (double)file.Length; p?.Report(progress); } } } while(read == buffer.Length); if(ShowProgress) bar.Dispose(); if(!file.CanSeek) length(totalLength); } public void Dispose() { if(ownClient) this.clt.Dispose(); } } [Serializable] internal class BackupFailedException : Exception { public BackupFailedException() { } public BackupFailedException(string message) : base(message) { } public BackupFailedException(string server_hash, string hash) : base($"The server hash \"{server_hash}\" does not match client hash \"{hash}\"") { } public BackupFailedException(string message, Exception innerException) : base(message, innerException) { } protected BackupFailedException(SerializationInfo info, StreamingContext context) : base(info, context) { } } public class LoginResult { [JsonProperty("success")] public bool Success {get;set;}=false; [JsonProperty("key")] public string Key {get;set;}=""; } }