tesses50
/
tesses-backup
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
tesses-backup/TessesDedup/Class1.cs

786 lines
29 KiB
C#
Raw Permalink Normal View History

First commit
2024-07-23 03:49:40 +00:00
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
using LiteDB;
using Tesses.VirtualFilesystem;
using Tesses.VirtualFilesystem.Extensions;
using Tesses.WebServer;
using System.Xml;
using System.Text;
using System.Web;
Hopefully make /api/v1/Stats efficient by storing count in database
2024-07-23 18:48:04 +00:00
using System.Data.Common;
First commit
2024-07-23 03:49:40 +00:00
namespace TessesDedup
{
public class Dedup
{
public class HttpDirectoryServer : Server
{
Dedup dedup;
public HttpDirectoryServer(Dedup dedup)
{
this.dedup=dedup;
}
public override async Task GetAsync(ServerContext ctx)
{
if (!ctx.RequestHeaders.TryGetFirst("Authorization", out var value)){
ctx.ResponseHeaders.Add("WWW-Authenticate", "Basic realm=\"Webdav realm\"");
ctx.StatusCode = 401;
await ctx.SendTextAsync("Unauthorized");
return;
}
string[] array = value.Split(' ');
string[] array2 = Encoding.UTF8.GetString(Convert.FromBase64String(array[1])).Split(new char[1] { ':' }, 2);
string usern = array2[0];
Account user=null;
bool accessKeyMode=false;
if(usern == "$access_key")
accessKeyMode=true;
else
using(var db = dedup.Database)
{
user = db.Accounts.FindOne(e=>e.Username == usern );
}
if(user == null && !accessKeyMode) {
ctx.ResponseHeaders.Add("WWW-Authenticate", "Basic realm=\"Webdav realm\"");
ctx.StatusCode = 401;
await ctx.SendTextAsync("Unauthorized");
return;
}
if(!accessKeyMode && !user.PasswordCorrect(array2[1])) {
ctx.ResponseHeaders.Add("WWW-Authenticate", "Basic realm=\"Webdav realm\"");
ctx.StatusCode = 401;
await ctx.SendTextAsync("Unauthorized");
return;
}
if(accessKeyMode)
{
using(var db = dedup.Database)
{
string key = array2[1];
var ak = db.AccessKeys.FindOne(e=>e.Key == key);
if(ak != null)
{
user = db.Accounts.FindOne(e=>e.Id == ak.UserId);
if(user == null)
{
ctx.ResponseHeaders.Add("WWW-Authenticate", "Basic realm=\"Webdav realm\"");
ctx.StatusCode = 401;
await ctx.SendTextAsync("Unauthorized");
return;
}
}
else
{
ctx.ResponseHeaders.Add("WWW-Authenticate", "Basic realm=\"Webdav realm\"");
ctx.StatusCode = 401;
await ctx.SendTextAsync("Unauthorized");
return;
}
}
}
StringBuilder builder=new StringBuilder($"<!DOCTYPE html><html><head><title>Index of {HttpUtility.HtmlEncode(HttpUtility.UrlDecode(ctx.UrlPath))}</title></head><body><h1>Index of {HttpUtility.HtmlEncode(HttpUtility.UrlDecode(ctx.UrlPath))}</h1><hr><pre><a href=\"../\">../</a>\n");
bool isDir=true;
void Insert(string path,bool isDir=true)
{
builder.Append($"<a href=\"{HttpUtility.UrlPathEncode(path)}\">{HttpUtility.HtmlEncode(Path.GetFileName(path.TrimEnd('/')))}{(isDir?"/":"")}</a>\n");
}
if(ctx.UrlPath == "/")
{
List<string> deviceNames = new List<string>();
using(var db = dedup.Database)
foreach(var item in db.Backups.Find(e=>e.AccountId == user.Id))
{
if(!deviceNames.Contains(item.DeviceName))
deviceNames.Add(item.DeviceName);
}
foreach(var dev in deviceNames)
{
Insert($"{dev}/");
}
}
else
{
string[] path = ctx.UrlPath.Split(new char[]{'/'},StringSplitOptions.RemoveEmptyEntries);
if(path.Length == 1)
{
List<string> backups = new List<string>();
using(var db = dedup.Database)
foreach(var item in db.Backups.Find(e=>e.AccountId == user.Id))
{
string backupName = $"{item.Tag} ({item.CreationDate.ToString("yyyyMMdd_HHmmss")})";
if(item.DeviceName == path[0] && !backups.Contains(backupName))
{
backups.Add(backupName);
}
}
foreach(var dev in backups)
{
Insert($"{dev}/");
}
}
else {
var name = HttpUtility.UrlDecode(path[1]);
using(var db = dedup.Database)
foreach(var item in db.Backups.Find(e=>e.AccountId == user.Id))
{
string backupName = $"{item.Tag} ({item.CreationDate.ToString("yyyyMMdd_HHmmss")})";
if(item.DeviceName == path[0] && name == backupName)
{
UnixPath upath = Special.Root;
for(int i = 2;i<path.Length;i++)
{
upath /= HttpUtility.UrlDecode(path[i]);
}
var p = item.GetEntryFromPath(upath);
if(p.Type == FilesystemEntryType.Dir)
{
foreach(var item2 in p.Entries)
{
var _p =item2.Type == FilesystemEntryType.Symlink ? item.GetEntryFromPath(upath / item2.Name) : item2;
Insert(_p.Type == FilesystemEntryType.Dir ? $"{item2.Name}/" : item2.Name,_p.Type == FilesystemEntryType.Dir);
}
}
else if(p.Type == FilesystemEntryType.File)
{
await ctx.SendStreamAsync(dedup.Storage.OpenRead(p),HeyRed.Mime.MimeTypesMap.GetMimeType(upath.Name));
return;
}
goto dir;
}
}
await ctx.SendNotFoundAsync();
return;
}
}
dir:
if(isDir)
{
builder.Append("</pre><hr></body></html>");
await ctx.SendTextAsync(builder.ToString());
}
}
}
public IServer Server {get;}
public DedupStorage Storage {get;}
public DatabaseHandler Database => new DatabaseHandler(_db());
private Func<ILiteDatabase> _db;
public class DatabaseHandler : IDisposable
{
public ILiteCollection<Backup> Backups => Database.GetCollection<Backup>("backups");
public ILiteCollection<Account> Accounts => Database.GetCollection<Account>("accounts");
public ILiteCollection<AccessKey> AccessKeys=>Database.GetCollection<AccessKey>("accesskeys");
public ILiteDatabase Database {get;}
public DatabaseHandler(ILiteDatabase db)
{
Database = db;
}
public void Dispose()
{
Database.Dispose();
}
}
RouteServer routeServer;
IVirtualFilesystem fs;
public Dedup(IServer www,IVirtualFilesystem storage,Func<ILiteDatabase> db)
{
_db = db;
fs = storage;
Storage = new DedupStorage(storage);
MountableServer mnt=new MountableServer(www);
HttpDirectoryServer webdav=new HttpDirectoryServer(this);
mnt.Mount("/data/",webdav);
routeServer=new RouteServer(mnt);
Server = routeServer;
routeServer.Add("/api/v1/Block",HasBlockAsync,"HEAD");
routeServer.Add("/api/v1/Block",PutBlockAsync,"PUT");
routeServer.Add("/api/v1/Backup",GetBackupAsync,"GET");
routeServer.Add("/api/v1/Backup",PutBackupAsync,"PUT");
routeServer.Add("/api/v1/Login",LoginAsync,"POST");
routeServer.Add("/api/v1/Logout",LogoutAsync,"POST");
routeServer.Add("/api/v1/Download",DownloadAsync,"GET");
routeServer.Add("/api/v1/Download",DownloadAsync,"HEAD");
routeServer.Add("/api/v1/AccessKey",AccessKeyAsync,"GET");
routeServer.Add("/api/v1/AccessKey",AccessKeyDeleteAsync,"DELETE");
routeServer.Add("/api/v1/Registered",RegisteredAsync,"GET");
routeServer.Add("/api/v1/Stats",StatsAsync,"GET");
Hopefully make /api/v1/Stats efficient by storing count in database
2024-07-23 18:48:04 +00:00
using(var _db = Database)
{
if(_db.Accounts.Count() == 1)
{
var acnt = _db.Accounts.FindAll().First();
if(acnt.BlockCount == 0)
{
long blocks=0;
foreach(var dir in fs.EnumerateDirectories(Special.Root))
{
foreach(var dir2 in fs.EnumerateDirectories(dir))
{
foreach(var file in fs.EnumerateFiles(dir2))
{
blocks++;
}
}
}
acnt.BlockCount = blocks;
_db.Accounts.Update(acnt);
}
}
}
First commit
2024-07-23 03:49:40 +00:00
}
private async Task AccessKeyDeleteAsync(ServerContext ctx)
{
if(await Unauthenticated(ctx)) return;
string key = GetAuthorizationKey(ctx);
using(var db = Database){
var ak=db.AccessKeys.FindOne(e=>e.Key == key);
if(ctx.QueryParams.TryGetFirstInt64("id",out var id))
{
var item=db.AccessKeys.FindById(id);
if(item != null && item.UserId == ak.UserId)
{
if(item.Id != ak.Id)
{
db.AccessKeys.Delete(id);
await ctx.SendJsonAsync(new{success=true,reason="Success, it got deleted"});
}
else {
await ctx.SendJsonAsync(new{success=false,reason="Cannot delete this accesskey, must use Logout"});
}
}
else
{
await ctx.SendJsonAsync(new{success=false,reason="Either accesskey does not exist or it is not yours"});
}
}
}
}
private async Task AccessKeyAsync(ServerContext ctx)
{
if(await Unauthenticated(ctx)) return;
string key = GetAuthorizationKey(ctx);
using(var db = Database)
{
var ak=db.AccessKeys.FindOne(e=>e.Key == key);
await ctx.SendJsonAsync(db.AccessKeys.Find(e=>e.UserId == ak.UserId && e.Id != ak.Id).ToList());
}
}
private string BytesToUnited(long sz)
{
if(sz == 1) return "1 byte";
if(sz < 1024) return $"{sz} bytes";
if(sz < Math.Pow(1024,2)) return $"{sz / 1024} kiB";
if(sz < Math.Pow(1024,3)) return $"{sz / (int)Math.Pow(1024,2)} MiB";
if(sz < Math.Pow(1024,4)) return $"{sz / (long)Math.Pow(1024,3)} GiB";
return $"{sz / (long)Math.Pow(1024,4)} TiB";
}
private async Task StatsAsync(ServerContext ctx)
{
if(await Unauthenticated(ctx)) return;
Hopefully make /api/v1/Stats efficient by storing count in database
2024-07-23 18:48:04 +00:00
/*
First commit
2024-07-23 03:49:40 +00:00
await foreach(var dir in fs.EnumerateDirectoriesAsync(Special.Root))
{
await foreach(var dir2 in fs.EnumerateDirectoriesAsync(dir))
{
await foreach(var file in fs.EnumerateFilesAsync(dir2))
{
blocks++;
}
}
Hopefully make /api/v1/Stats efficient by storing count in database
2024-07-23 18:48:04 +00:00
}*/
First commit
2024-07-23 03:49:40 +00:00
using(var db = Database)
Hopefully make /api/v1/Stats efficient by storing count in database
2024-07-23 18:48:04 +00:00
{
var key = GetAuthorizationKey(ctx);
var ak=db.AccessKeys.FindOne(e=>e.Key == key);
var user = db.Accounts.FindById(ak.UserId);
if(user != null)
{
long blocks = user.BlockCount;
long bytes = blocks*DedupStorage.BlockLength;
await ctx.SendJsonAsync(new{blocks,bytes,backups=db.Backups.Count(),label=BytesToUnited(bytes)});
}
}
First commit
2024-07-23 03:49:40 +00:00
}
public bool Registered()
{
using(var db = Database)
return db.Accounts.FindById(1) != null && !fs.FileExists(Special.Root/"ResetPassword.txt");
}
private async Task RegisteredAsync(ServerContext ctx)
{
await ctx.SendJsonAsync(Registered());
}
private async Task DownloadAsync(ServerContext ctx)
{
if(await Unauthenticated(ctx)) return;
if(ctx.QueryParams.TryGetFirstInt64("id",out var id) && ctx.QueryParams.TryGetFirst("path",out var path))
{
UnixPath upath = path;
Backup bkp;
using(var db = Database)
bkp=db.Backups.FindById(id);
if(bkp != null)
{
try
{
var ent=bkp.GetEntryFromPath(upath);
ctx.WithFileName(ent.Name,true);
await ctx.SendStreamAsync(Storage.OpenRead(ent),HeyRed.Mime.MimeTypesMap.GetMimeType(ent.Name));
}catch(DirectoryNotFoundException)
{
ctx.StatusCode = 404;
await ctx.SendTextAsync($"Could not find directory {upath.Parent.Path}\r\n","text/plain");
}
catch (FileNotFoundException)
{
ctx.StatusCode = 404;
await ctx.SendTextAsync($"Could not find file {upath.Path}\r\n","text/plain");
}
}
else
{
ctx.StatusCode=404;
await ctx.SendTextAsync("Backup not found\r\n");
}
}
}
private async Task LogoutAsync(ServerContext ctx)
{
if(!ctx.RequestHeaders.ContainsKey("Content-Type"))
ctx.RequestHeaders.Add("Content-Type","application/x-www-form-urlencoded");
ctx.ParseBody();
var key = GetAuthorizationKey(ctx);
if(!string.IsNullOrWhiteSpace(key))
{
using(var db = Database){
var item = db.AccessKeys.FindOne(e=>e.Key ==key);
if(item != null)
db.AccessKeys.Delete(item.Id);
}
}
ctx.StatusCode = 204;
await ctx.WriteHeadersAsync();
}
private async Task LoginAsync(ServerContext ctx)
{
if(!ctx.RequestHeaders.ContainsKey("Content-Type"))
ctx.RequestHeaders.Add("Content-Type","application/x-www-form-urlencoded");
ctx.ParseBody();
if(ctx.QueryParams.TryGetFirst("username",out var username) && ctx.QueryParams.TryGetFirst("password",out var password) && ctx.QueryParams.TryGetFirst("device_name",out var device_name))
{
if(Registered())
{
Account user;
using(var db = Database)
user=db.Accounts.FindOne(e=>e.Username == username);
if(user != null)
{
if(user.PasswordCorrect(password))
{
AccessKey ak = new AccessKey();
ak.NewKey();
ak.Id = 0;
ak.UserId = user.Id;
ak.DeviceName = device_name;
ak.CreationDate=DateTime.Now;
using(var db = Database)
db.AccessKeys.Insert(ak);
await ctx.SendJsonAsync(new{success=true, key=ak.Key});
}
else
{
await ctx.SendJsonAsync(new {success=false});
}
}
else
{
await ctx.SendJsonAsync(new {success=false});
}
}
else
{
if(await fs.FileExistsAsync(Special.Root/"ResetPassword.txt")) await fs.DeleteFileAsync(Special.Root/"ResetPassword.txt");
Account user;
using(var db = Database)
user = db.Accounts.FindById(1);
if(user != null)
{
user.Username = username;
user.NewSalt();
user.PasswordHash=user.GetPasswordHash(password);
user.Username = username;
using(var db = Database)
db.Accounts.Update(user);
}
else
{
Account acnt = new Account();
acnt.Id = 1;
acnt.NewSalt();
acnt.PasswordHash=acnt.GetPasswordHash(password);
acnt.Username = username;
using(var db = Database)
db.Accounts.Insert(acnt);
}
AccessKey ak = new AccessKey();
ak.NewKey();
ak.Id = 0;
ak.UserId = 1;
ak.DeviceName = device_name;
using(var db = Database)
db.AccessKeys.Insert(ak);
await ctx.SendJsonAsync(new{success=true, key=ak.Key});
}
}
}
private async Task PutBackupAsync(ServerContext ctx)
{
if(await Unauthenticated(ctx)) return;
var item = await ctx.ReadJsonAsync<Backup>();
item.Id = 0;
string key = GetAuthorizationKey(ctx);
AccessKey ak;
using(var db = Database)
ak=db.AccessKeys.FindOne(e=>e.Key == key);
item.AccountId = ak.UserId;
item.DeviceName = ak.DeviceName;
using(var db = Database)
db.Backups.Insert(item);
ctx.StatusCode=204;
await ctx.WriteHeadersAsync();
}
private string GetAuthorizationKey(ServerContext ctx)
{
string key="";
if(ctx.RequestHeaders.TryGetFirst("Authorization",out var auth))
{
if(auth.StartsWith("Bearer "))
{
key = auth.Substring(7);
}
}
if(ctx.RequestHeaders.TryGetFirst("X-Access-Key",out var ak))
{
key = ak;
}
if(ctx.QueryParams.TryGetFirst("access_key",out var access_key))
{
key=access_key;
}
return key;
}
private long GetAccountId(ServerContext ctx)
{
string key = GetAuthorizationKey(ctx);
using(var db = Database)
{
var ak=db.AccessKeys.FindOne(e=>e.Key == key);
if(ak != null) return ak.UserId;
}
return -1;
}
private async Task GetBackupAsync(ServerContext ctx)
{
if(await Unauthenticated(ctx)) return;
bool noHashes=ctx.QueryParams.GetFirstBoolean("noHashes");
if(ctx.QueryParams.TryGetFirstInt64("id",out var id))
{
Backup item;
using(var db = Database)
item = db.Backups.FindById(id);
if(item != null)
await ctx.SendJsonAsync(noHashes ? item.WithoutHashes() : item);
else
{
ctx.StatusCode=404;
await ctx.SendNotFoundAsync();
}
}
else
{
List<Backup> backups=new List<Backup>();
using(var db = Database)
foreach(var item in db.Backups.FindAll())
{
backups.Add(item.WithoutRoot());
}
await ctx.SendJsonAsync(backups);
}
}
private async Task PutBlockAsync(ServerContext ctx)
{
if(await Unauthenticated(ctx)) return;
Hopefully make /api/v1/Stats efficient by storing count in database
2024-07-23 18:48:04 +00:00
First commit
2024-07-23 03:49:40 +00:00
var vals=await ctx.ReadBytesAsync();
Hopefully make /api/v1/Stats efficient by storing count in database
2024-07-23 18:48:04 +00:00
long acntId=GetAccountId(ctx);
string res = Storage.WriteBlock(vals,this,acntId);
await ctx.SendTextAsync(res,"text/plain");
First commit
2024-07-23 03:49:40 +00:00
}
private async Task HasBlockAsync(ServerContext ctx)
{
if(await Unauthenticated(ctx)) return;
if(ctx.QueryParams.TryGetFirst("hash",out var hash))
{
ctx.StatusCode = Storage.HasBlock(hash) ? 200 : 404;
await ctx.WriteHeadersAsync();
}
}
private async Task<bool> Unauthenticated(ServerContext ctx)
{
if(GetAccountId(ctx) == -1)
{
ctx.StatusCode = 401;
await ctx.SendTextAsync("Unauthorized use /api/v1/Login to Authenticate\r\nand use access_key query param or Authorization: Bearer YOURKEY on this endpoint\r\n","text/plain");
return true;
}
return false;
}
}
public class DedupStorage
{
private class DedupStream : Stream
{
private DedupStorage dedupStorage;
private FilesystemEntry fsE;
public DedupStream(DedupStorage dedupStorage, FilesystemEntry fsE)
{
this.dedupStorage = dedupStorage;
this.fsE = fsE;
}
public override bool CanRead => true;
public override bool CanSeek => true;
public override bool CanWrite => false;
public override long Length => fsE.Length;
public override long Position { get;set; }
public override void Flush()
{
}
public override int Read(byte[] buffer, int offset, int count)
{
int total_read = Math.Min(count,buffer.Length-offset);
long end = Position + total_read;
if(end > Length) {end = Length; total_read = (int)(Length-Position);}
int read = 0;
while(Position < end)
{
//get current hash for Position
int currentFile = (int)(Position / BlockLength);
//get current offset (in file) for Position
int fileOffset = (int)(Position % BlockLength);
int toReadFromFile = BlockLength - fileOffset;
toReadFromFile = Math.Min(total_read-read,toReadFromFile);
var blk=dedupStorage.ReadBlock(fsE.Hashes[currentFile]);
Array.Copy(blk,fileOffset,buffer,offset+read,toReadFromFile);
read+=toReadFromFile;
Position+=toReadFromFile;
}
return read;
}
public override long Seek(long offset, SeekOrigin origin)
{
switch(origin)
{
case SeekOrigin.Begin:
Position = offset;
break;
case SeekOrigin.Current:
Position += offset;
break;
case SeekOrigin.End:
Position = Length - offset;
break;
}
return Position;
}
public override void SetLength(long value)
{
throw new NotImplementedException();
}
public override void Write(byte[] buffer, int offset, int count)
{
throw new NotImplementedException();
}
}
public Stream OpenRead(FilesystemEntry fsE)
{
return new DedupStream(this,fsE);
}
public DedupStorage(IVirtualFilesystem fs)
{
this.vfs = fs;
}
public const int BlockReadSize = 1024;
public const int BlockReads=4096;
public const int BlockLength = BlockReadSize*BlockReads;
IVirtualFilesystem vfs;
public UnixPath GetHashPath(string hash)
{
if(hash.Length != 128) return Special.Root;
string hashSlice1 = hash.Substring(0,2);
string hashSlice2 = hash.Substring(2,2);
string hashRest = hash.Substring(4,124);
return Special.Root / hashSlice1 / hashSlice2 / hashRest;
}
Mutex mtx=new Mutex();
public bool HasBlock(string hash)
{
if(hash.Length != 128) return false;
return vfs.FileExists(GetHashPath(hash));
}
internal static string Sha512Hash(byte[] data)
{
using(var sha512=SHA512.Create())
{
sha512.Initialize();
byte[] hash=sha512.ComputeHash(data);
return BitConverter.ToString(hash).ToLower().Replace("-","");
}
}
Hopefully make /api/v1/Stats efficient by storing count in database
2024-07-23 18:48:04 +00:00
public string WriteBlock(byte[] data,Dedup dedup,long acntId)
First commit
2024-07-23 03:49:40 +00:00
{
if(data.Length != BlockLength)
mtx.WaitOne();
string hash=Sha512Hash(data);
Hopefully make /api/v1/Stats efficient by storing count in database
2024-07-23 18:48:04 +00:00
First commit
2024-07-23 03:49:40 +00:00
if(!HasBlock(hash))
{
Hopefully make /api/v1/Stats efficient by storing count in database
2024-07-23 18:48:04 +00:00
First commit
2024-07-23 03:49:40 +00:00
string hashSlice1 = hash.Substring(0,2);
string hashSlice2 = hash.Substring(2,2);
vfs.CreateDirectory(Special.Root / hashSlice1);
vfs.CreateDirectory(Special.Root / hashSlice1 / hashSlice2);
vfs.WriteAllBytes(GetHashPath(hash),data);
Hopefully make /api/v1/Stats efficient by storing count in database
2024-07-23 18:48:04 +00:00
using(var db = dedup.Database)
{
var acnt=db.Accounts.FindById(acntId);
acnt.BlockCount++;
db.Accounts.Update(acnt);
}
First commit
2024-07-23 03:49:40 +00:00
}
mtx.ReleaseMutex();
return hash;
}
public byte[] ReadBlock(string hash)
{
if(HasBlock(hash))
{
return vfs.ReadAllBytes(GetHashPath(hash));
}
return new byte[0];
}
}
}