tesses-backup/TessesDedup/Client.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using LiteDB;
using Newtonsoft.Json;
using ShellProgressBar;
using Tesses.VirtualFilesystem;
using Tesses.VirtualFilesystem.Extensions;
using Tesses.WebServer;
using System.Net;
using System.Security;
namespace TessesDedup
{
    public class DedupClient : IDisposable
    {
        HttpClient clt;
        string url;
        bool ownClient;
        public bool ShowProgress {get;set;}
        public bool IgnoreErrors {get;set;}
        public DedupClient(HttpClient client, string url,bool ownClient=true,bool showProgress=true,bool ignoreErrors=true)
        {
            ShowProgress = showProgress;
            IgnoreErrors = ignoreErrors;
            this.clt = client;
            this.url = url.TrimEnd('/');
            this.ownClient = ownClient;
            
        }
        public DedupClient(string url,bool showProgress=true,bool ignoreErrors=true) : this(new HttpClient(),url,true,showProgress,ignoreErrors)
        {

        }
         public async Task LogoutAsync(string key)
    {
        Dictionary<string,string> dictionary= new Dictionary<string, string>
        {
            { "key", key }
        };
        
        FormUrlEncodedContent formUrlEncodedContent=new FormUrlEncodedContent(dictionary);
        using(var resp=await clt.PostAsync($"{url}/api/v1/Logout",formUrlEncodedContent))
        {

        }
    }
    public async Task<LoginResult> LoginAsync(string username,string password, string device_name)
    {
        Dictionary<string,string> dictionary= new Dictionary<string, string>
        {
            { "username", username },
            { "password", password },
            { "device_name", device_name }
        };
        
        FormUrlEncodedContent formUrlEncodedContent=new FormUrlEncodedContent(dictionary);
        using(var resp=await clt.PostAsync($"{url}/api/v1/Login",formUrlEncodedContent))
        {
            if(resp.IsSuccessStatusCode)
            {
                return JsonConvert.DeserializeObject<LoginResult>(await resp.Content.ReadAsStringAsync());
            }
            else
            {
                return new LoginResult();
            }
        }
    }


        private async Task<int> ReadAsync(Stream strm, byte[] buffer,CancellationToken token=default)
        {
            Array.Clear(buffer,0,buffer.Length);
            int totalRead = 0;
            while(totalRead < buffer.Length)
            {
                int read = Math.Min(buffer.Length-totalRead,DedupStorage.BlockReadSize);
                if(read == 0) 
                {
                    break;
                }
                read=await strm.ReadAsync(buffer,totalRead,read,token);
                if(token.IsCancellationRequested) return 0;
                if(read == 0) 
                {
                    break;
                }
                totalRead+=read;
            }
            return totalRead;
        }
        private async Task<bool> HasBlockAsync(string authkey,string hash,CancellationToken token=default)
        {
            HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Head,$"{url}/api/v1/Block?hash={hash}");
            request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);
            using(var req=await clt.SendAsync(request,token))
                return req.StatusCode == System.Net.HttpStatusCode.OK;
        }
        private async Task WriteBlockAsync(string authkey,string hash,byte[] buffer,CancellationToken token=default)
        {
            HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Put,$"{url}/api/v1/Block");
            request.Content = new ByteArrayContent(buffer);
            request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);
            using(var req=await clt.SendAsync(request,token))
            {
                if(req.StatusCode == System.Net.HttpStatusCode.OK)
                {
                    string server_hash=await req.Content.ReadAsStringAsync();
                    if(server_hash != hash)
                        throw new BackupFailedException(server_hash,hash);
                } 
            }
        }
        public async Task<bool> BackupAsync(string authkey, IVirtualFilesystem fs,string tag,CancellationToken token=default)
        {
            var dt = DateTime.Now;
            var fse=await BackupPathAsync(authkey,fs,Special.Root,token);
            if(token.IsCancellationRequested) return false;
            Backup bkp=new Backup();
            bkp.Id = 0;
            bkp.Tag = tag;
            bkp.CreationDate = dt;
            bkp.Root = fse;
            
            await WriteBackupAsync(authkey,bkp,token);
            if(token.IsCancellationRequested) return false;
            return true;
        }

        private async Task WriteBackupAsync(string authkey, Backup bkp,CancellationToken token=default)
        {
            var str=JsonConvert.SerializeObject(bkp);
            HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Put,$"{url}/api/v1/Backup");
            request.Content = new StringContent(str,Encoding.UTF8,"application/json");
            request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);

            using(var c=await clt.SendAsync(request,token))
            {
                c.EnsureSuccessStatusCode();
            }
            
        }

        private async Task<FilesystemEntry> BackupPathAsync(string authkey,IVirtualFilesystem fs,UnixPath path,CancellationToken token=default)
        {
            if(await fs.SymlinkExistsAsync(path))
            {
                return new FilesystemEntry(){Name = path.Name, Type = FilesystemEntryType.Symlink, PointsTo = (await fs.ReadLinkAsync(path,token)).Path};
            }
            else if(await fs.DirectoryExistsAsync(path))
            {
                FilesystemEntry entry=new FilesystemEntry();
                entry.Name = path.Name;
                entry.Type = FilesystemEntryType.Dir;
                await foreach(var ent in fs.EnumerateFileSystemEntriesAsync(path))
                {
                    if(token.IsCancellationRequested) return entry;
                    try{
                    entry.Entries.Add(await BackupPathAsync(authkey,fs,ent,token));
                    }catch(Exception ex)
                    {
                        if(!IgnoreErrors)
                        throw;
                    }
                }
                return entry;
            }
            else if(await fs.FileExistsAsync(path))
            {
                FilesystemEntry entry = new FilesystemEntry();
                entry.Name = path.Name;
                entry.Type = FilesystemEntryType.File;
                entry.Length = 0;
                entry.Hashes=new List<string>();
                
                using(var strm = await fs.OpenReadAsync(path))
                {
                    if(strm.CanSeek) entry.Length = strm.Length;
                    await foreach(var hash in BackupFileAsync($"/{string.Join("/",path.Parts)}",authkey,strm,(len)=>entry.Length=len,token))
                    {
                        if(token.IsCancellationRequested) return entry;
                        entry.Hashes.Add(hash);
                    }
                }

                return entry;
            }
            else
            {
                return new FilesystemEntry(){Length=0, Type = FilesystemEntryType.File};
            }
        }

        public async IAsyncEnumerable<Backup> ListBackupsAsync(string authkey)
        {
            HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Get,$"{url}/api/v1/Backup");
            request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);
            var resp=await clt.SendAsync(request);
            resp.EnsureSuccessStatusCode();
            foreach(var item in JsonConvert.DeserializeObject<List<Backup>>(await resp.Content.ReadAsStringAsync()))
            {
                if(item != null) yield return item;
            }
        }
        public async Task<Backup> GetBackup(string authkey,long id,bool noHashes=true)
        {
            HttpRequestMessage request=new HttpRequestMessage(HttpMethod.Get,$"{url}/api/v1/Backup?id={id}&noHashes={noHashes}");
            request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer",authkey);
            var resp=await clt.SendAsync(request);
            resp.EnsureSuccessStatusCode();
            var bkp=JsonConvert.DeserializeObject<Backup>(await resp.Content.ReadAsStringAsync());
            return bkp;
        }
        private async Task RestoreAsync(string key,long id,FilesystemEntry entry,UnixPath srcPath,IVirtualFilesystem fs,UnixPath destPath,CancellationToken token=default)
        {
            if(entry.Type == FilesystemEntryType.Dir)
            {
                await fs.CreateDirectoryAsync(destPath,token);
                foreach(var ent in entry.Entries)
                {
                    if(token.IsCancellationRequested) return;
                    await RestoreAsync(key,id,ent,srcPath/ent.Name,fs,destPath/ent.Name,token);
                }
            }
            else if(entry.Type == FilesystemEntryType.File)
            {
                
               using(var file = await fs.OpenAsync(destPath,FileMode.OpenOrCreate,FileAccess.ReadWrite,FileShare.None))
               {
            ProgressBar bar=null;
            IProgress<double> p=null;
            if(ShowProgress && file.CanSeek)
            {
            var options = new ProgressBarOptions
            {
	            ProgressCharacter = '─',
	            ProgressBarOnBottom = true
            };
            bar=new ProgressBar(10000,$"Restoring file {destPath.Path}");
            p = bar.AsProgress<double>();
            }else if(ShowProgress)
            {
                Console.WriteLine("Stream is not seekable");
            }

                await RestoreFileAsync(key,id,srcPath.Path,file,new Progress<long>(e=>{
                    if(ShowProgress && file.CanSeek)
                    {
                        try{p.Report((double)e/(double)file.Length);}
                        catch(Exception ex)
                        {
                            _=ex;
                        }
                        
                    }
                }),token);

                if(ShowProgress && file.CanSeek)
                    {
                        bar.Dispose();
                    }
                

               }

            }
            else if(entry.Type == FilesystemEntryType.Symlink) {
                await fs.CreateSymlinkAsync(new UnixPath(entry.PointsTo),destPath,token);
            }
        }   

        public async Task RestoreAsync(string key,long id,UnixPath srcPath,IVirtualFilesystem fs,UnixPath destPath,CancellationToken token=default)
        {
            var bkp=await GetBackup(key,id);
            await RestoreAsync(key,id,bkp.GetEntryFromPath(srcPath),srcPath,fs,destPath,token);
        }
        private async Task RestoreFileAsync(string key,long id, string path,Stream strm,IProgress<long> progress=null,CancellationToken token=default)
        {
            var _path = $"{url}/api/v1/Download?id={id}&access_key={WebUtility.UrlEncode(key)}&id={id}&path={WebUtility.UrlEncode(path)}";
            byte[] buffer = new byte[1024];

            long offset=0;

            HttpRequestMessage requestMessage=new HttpRequestMessage(HttpMethod.Get,_path);

            if(strm.CanSeek && strm.Length > 0)
            {
                strm.Position = strm.Length;
                offset=strm.Position;

                requestMessage.Headers.Range=new System.Net.Http.Headers.RangeHeaderValue(strm.Length,null);
            }
            if(token.IsCancellationRequested) return;
            using(var req = await clt.SendAsync(requestMessage,token))
            {
                if(token.IsCancellationRequested) return;
                if(req.StatusCode == System.Net.HttpStatusCode.RequestedRangeNotSatisfiable) return;
                req.EnsureSuccessStatusCode();
            int read = 0;
            using(var srcStrm = await req.Content.ReadAsStreamAsync())
            do {
                read = await srcStrm.ReadAsync(buffer,0,buffer.Length,token);
                if(token.IsCancellationRequested) return;
                await strm.WriteAsync(buffer,0,read,token);
                if(token.IsCancellationRequested) return;
                offset += read;
                progress?.Report(offset);
            } while(read > 0);
            }
        }
        private async IAsyncEnumerable<string> BackupFileAsync(string name,string authkey,Stream file,Action<long> length,[EnumeratorCancellation]CancellationToken token=default)
        {
            if(file.CanSeek && file.Length == 0) yield break; 
            byte[] buffer = new byte[DedupStorage.BlockLength];
            int read=0;
            long totalLength=0;
            ProgressBar bar=null;
            IProgress<double> p=null;
            if(ShowProgress && file.CanSeek)
            {
            var options = new ProgressBarOptions
            {
	            ProgressCharacter = '─',
	            ProgressBarOnBottom = true
            };
            bar=new ProgressBar(10000,$"Backing up file {name}");
            p = bar.AsProgress<double>();
            }
            else if(ShowProgress)
            {
                Console.WriteLine("Stream is not seekable");
            }
            do{
                read=await ReadAsync(file,buffer,token);
                if(read == 0) yield break;
                if(token.IsCancellationRequested) yield break;
                string hash=DedupStorage.Sha512Hash(buffer);    
                yield return hash;
                if(!await HasBlockAsync(authkey,hash,token))
                {
                    await WriteBlockAsync(authkey,hash,buffer,token);
                    if(token.IsCancellationRequested) yield break;
                }
                
                totalLength += read;

                if(ShowProgress)
                {
                    if(file.CanSeek)
                    {
                        double progress = totalLength / (double)file.Length;
                        p?.Report(progress);
                    }
                }

                
            } while(read == buffer.Length);
            
            if(ShowProgress)
                bar.Dispose();

             if(!file.CanSeek)
                length(totalLength);
        }

        public void Dispose()
        {
            if(ownClient)
                this.clt.Dispose();
        }
    }

    [Serializable]
    internal class BackupFailedException : Exception
    {

        public BackupFailedException()
        {
        }

        public BackupFailedException(string message) : base(message)
        {
        }

        public BackupFailedException(string server_hash, string hash) : base($"The server hash \"{server_hash}\" does not match client hash \"{hash}\"")
        {
            
        }

        public BackupFailedException(string message, Exception innerException) : base(message, innerException)
        {
        }

        protected BackupFailedException(SerializationInfo info, StreamingContext context) : base(info, context)
        {
        }
    }
    public class LoginResult
{
    [JsonProperty("success")]
    public bool Success {get;set;}=false;

    [JsonProperty("key")]
    public string Key {get;set;}="";
}
}