From 554ad66b7aecd38d8e31a73f7ccbf07d2bda345b Mon Sep 17 00:00:00 2001 From: Mike Nolan Date: Tue, 23 Jul 2024 13:48:04 -0500 Subject: [PATCH] Hopefully make /api/v1/Stats efficient by storing count in database --- TessesDedup/Account.cs | 3 ++ TessesDedup/Class1.cs | 73 ++++++++++++++++++++++++++++++++++++++---- TessesDedup/Client.cs | 14 ++++++-- 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/TessesDedup/Account.cs b/TessesDedup/Account.cs index db84c77..83a28be 100644 --- a/TessesDedup/Account.cs +++ b/TessesDedup/Account.cs @@ -22,6 +22,9 @@ namespace TessesDedup [JsonIgnore] public string PasswordSalt {get;set;} + [JsonIgnore] + public long BlockCount {get;set;}=0; + public void NewSalt() { byte[] bytes=new byte[64]; diff --git a/TessesDedup/Class1.cs b/TessesDedup/Class1.cs index e878f83..596ff5b 100644 --- a/TessesDedup/Class1.cs +++ b/TessesDedup/Class1.cs @@ -12,6 +12,7 @@ using Tesses.WebServer; using System.Xml; using System.Text; using System.Web; +using System.Data.Common; namespace TessesDedup { @@ -249,6 +250,34 @@ namespace TessesDedup routeServer.Add("/api/v1/Registered",RegisteredAsync,"GET"); routeServer.Add("/api/v1/Stats",StatsAsync,"GET"); + + + using(var _db = Database) + { + if(_db.Accounts.Count() == 1) + { + var acnt = _db.Accounts.FindAll().First(); + if(acnt.BlockCount == 0) + { + + long blocks=0; + foreach(var dir in fs.EnumerateDirectories(Special.Root)) + { + foreach(var dir2 in fs.EnumerateDirectories(dir)) + { + foreach(var file in fs.EnumerateFiles(dir2)) + { + blocks++; + } + + } + + } + acnt.BlockCount = blocks; + _db.Accounts.Update(acnt); + } + } + } } private async Task AccessKeyDeleteAsync(ServerContext ctx) @@ -305,7 +334,8 @@ namespace TessesDedup private async Task StatsAsync(ServerContext ctx) { if(await Unauthenticated(ctx)) return; - long blocks = 0; + + /* await foreach(var dir in fs.EnumerateDirectoriesAsync(Special.Root)) { await foreach(var dir2 in fs.EnumerateDirectoriesAsync(dir)) @@ -317,10 +347,25 @@ namespace TessesDedup } - } - long bytes = blocks*DedupStorage.BlockLength; + }*/ using(var db = Database) - await ctx.SendJsonAsync(new{blocks,bytes,backups=db.Backups.Count(),label=BytesToUnited(bytes)}); + { + var key = GetAuthorizationKey(ctx); + + var ak=db.AccessKeys.FindOne(e=>e.Key == key); + + var user = db.Accounts.FindById(ak.UserId); + + if(user != null) + { + long blocks = user.BlockCount; + long bytes = blocks*DedupStorage.BlockLength; + + await ctx.SendJsonAsync(new{blocks,bytes,backups=db.Backups.Count(),label=BytesToUnited(bytes)}); + + } + + } } public bool Registered() @@ -549,8 +594,14 @@ namespace TessesDedup private async Task PutBlockAsync(ServerContext ctx) { if(await Unauthenticated(ctx)) return; + + var vals=await ctx.ReadBytesAsync(); - await ctx.SendTextAsync(Storage.WriteBlock(vals),"text/plain"); + + long acntId=GetAccountId(ctx); + + string res = Storage.WriteBlock(vals,this,acntId); + await ctx.SendTextAsync(res,"text/plain"); } private async Task HasBlockAsync(ServerContext ctx) @@ -695,18 +746,28 @@ namespace TessesDedup return BitConverter.ToString(hash).ToLower().Replace("-",""); } } - public string WriteBlock(byte[] data) + + public string WriteBlock(byte[] data,Dedup dedup,long acntId) { if(data.Length != BlockLength) mtx.WaitOne(); string hash=Sha512Hash(data); + if(!HasBlock(hash)) { + string hashSlice1 = hash.Substring(0,2); string hashSlice2 = hash.Substring(2,2); vfs.CreateDirectory(Special.Root / hashSlice1); vfs.CreateDirectory(Special.Root / hashSlice1 / hashSlice2); vfs.WriteAllBytes(GetHashPath(hash),data); + + using(var db = dedup.Database) + { + var acnt=db.Accounts.FindById(acntId); + acnt.BlockCount++; + db.Accounts.Update(acnt); + } } mtx.ReleaseMutex(); return hash; diff --git a/TessesDedup/Client.cs b/TessesDedup/Client.cs index caa8c50..d471103 100644 --- a/TessesDedup/Client.cs +++ b/TessesDedup/Client.cs @@ -16,6 +16,7 @@ using Tesses.VirtualFilesystem; using Tesses.VirtualFilesystem.Extensions; using Tesses.WebServer; using System.Net; +using System.Security; namespace TessesDedup { public class DedupClient : IDisposable @@ -24,14 +25,17 @@ namespace TessesDedup string url; bool ownClient; public bool ShowProgress {get;set;} - public DedupClient(HttpClient client, string url,bool ownClient=true,bool showProgress=true) + public bool IgnoreErrors {get;set;} + public DedupClient(HttpClient client, string url,bool ownClient=true,bool showProgress=true,bool ignoreErrors=true) { ShowProgress = showProgress; + IgnoreErrors = ignoreErrors; this.clt = client; this.url = url.TrimEnd('/'); this.ownClient = ownClient; + } - public DedupClient(string url,bool showProgress=true) : this(new HttpClient(),url,true,showProgress) + public DedupClient(string url,bool showProgress=true,bool ignoreErrors=true) : this(new HttpClient(),url,true,showProgress,ignoreErrors) { } @@ -159,7 +163,13 @@ namespace TessesDedup await foreach(var ent in fs.EnumerateFileSystemEntriesAsync(path)) { if(token.IsCancellationRequested) return entry; + try{ entry.Entries.Add(await BackupPathAsync(authkey,fs,ent,token)); + }catch(Exception ex) + { + if(!IgnoreErrors) + throw; + } } return entry; }