diff options
Diffstat (limited to 'Services')
| -rw-r--r-- | Services/ConfigService.cs | 72 | ||||
| -rw-r--r-- | Services/FeedService.cs | 212 | ||||
| -rw-r--r-- | Services/MediaService.cs | 400 | ||||
| -rw-r--r-- | Services/OcrService.cs | 128 | ||||
| -rw-r--r-- | Services/SourceService.cs | 20 | ||||
| -rw-r--r-- | Services/TagService.cs | 305 | ||||
| -rw-r--r-- | Services/UserService.cs | 76 |
7 files changed, 1213 insertions, 0 deletions
diff --git a/Services/ConfigService.cs b/Services/ConfigService.cs new file mode 100644 index 0000000..ac1f155 --- /dev/null +++ b/Services/ConfigService.cs @@ -0,0 +1,72 @@ +using HyperBooru.ApiModels; + +namespace HyperBooru.Services; + +public interface IConfigService { + public string DataPath { get; } + public string KeyPath { get; } + public string DbConnectionString { get; } + public string MediaBasePath { get; } + public string ThumbnailBasePath { get; } + public string ConvertedMediaBasePath { get; } + public bool EnableOcr { get; } +} + +public class ConfigService : IConfigService { + private IConfiguration config; + + private const string AppName = "HyperBooru"; + + public string DataPath { + get { + #if DEBUG + return "Data"; + #else + string? path = config["DataPath"]; + if(path is not null) + return path; + + switch(Environment.OSVersion.Platform) { + case PlatformID.Win32NT: + return Path.Join( + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), + AppName); + case PlatformID.Unix: + return $"/var/lib/{AppName.ToLower()}"; + default: + throw new NotImplementedException( + $"Unknown Operating System: {Environment.OSVersion.Platform}"); + } + #endif + } + } + + public string KeyPath => + Path.Join(DataPath, "keys"); + + public string DbConnectionString => + config.GetConnectionString("DefaultConnection") ?? + throw new HBException("Unable to get default connection string"); + + public string MediaBasePath => + Path.Join(DataPath, "media"); + + public string ThumbnailBasePath => + Path.Join(DataPath, "thumb"); + + public string ConvertedMediaBasePath => + Path.Join(DataPath, "converted"); + + public bool EnableOcr => + bool.TryParse(config["DisableOcr"], out bool x) ? !x : true; + + public ConfigService(IConfiguration config) { + this.config = config; + InitDirectoryStructure(); + } + + private void InitDirectoryStructure() { + Directory.CreateDirectory(DataPath); + Directory.CreateDirectory(MediaBasePath); + } +}
\ No newline at end of file diff --git a/Services/FeedService.cs b/Services/FeedService.cs new file mode 100644 index 0000000..3744e73 --- /dev/null +++ b/Services/FeedService.cs @@ -0,0 +1,212 @@ +using HyperBooru.ApiModels; +using Microsoft.EntityFrameworkCore; + +namespace HyperBooru.Services; + +public interface IFeedService { + public Media[] LoadChunk( + bool selectIngest, + bool includeNsfw, + Media? key = null, + int count = 50, + SortOrder sortOrder = SortOrder.ObjectId); + + public Media[] LoadChunk( + bool selectIngest, + bool includeNsfw, + string query, + Media? key = null, + int count = 50, + SortOrder sortOrder = SortOrder.ObjectId); + + public Media[] LoadChunk( + bool selectIngest, + bool includeNsfw, + Guid tagId, + Media? key = null, + int count = 50, + SortOrder sortOrder = SortOrder.ObjectId); + + public Media[] LoadChunk(FeedRequest feedRequest); +} + +public class FeedService : IFeedService { + private IDbContextFactory<HBContext> dbFactory; + + public FeedService(IDbContextFactory<HBContext> dbFactory) => + this.dbFactory = dbFactory; + + public Media[] LoadChunk( + bool selectIngest, + bool includeNsfw, + Media? continuationToken, + int count, + SortOrder sortOrder) => LoadChunkInternal( + selectIngest, includeNsfw, null, null, continuationToken?.Guid, count, sortOrder); + + public Media[] LoadChunk( + bool selectIngest, + bool includeNsfw, + string query, + Media? continuationToken, + int count, + SortOrder sortOrder) => LoadChunkInternal( + selectIngest, includeNsfw, query, null, continuationToken?.Guid, count, sortOrder); + + public Media[] LoadChunk( + bool selectIngest, + bool includeNsfw, + Guid tagId, + Media? continuationToken, + int count, + SortOrder sortOrder) => LoadChunkInternal( + selectIngest, includeNsfw, null, tagId, continuationToken?.Guid, count, sortOrder); + + public Media[] LoadChunk(FeedRequest feedRequest) { + switch(feedRequest) { + case FeedSearchRequest searchRequest: + return LoadChunkInternal( + selectIngest: searchRequest.SelectIngest, + includeNsfw: searchRequest.IncludeNsfw, + query: searchRequest.Query, + tagId: null, + continuationToken: searchRequest.ContinuationToken, + count: searchRequest.Count, + sortOrder: searchRequest.SortOrder); + case FeedTagRequest tagRequest: + return LoadChunkInternal( + selectIngest: tagRequest.SelectIngest, + includeNsfw: tagRequest.IncludeNsfw, + query: null, + tagId: tagRequest.TagId, + continuationToken: tagRequest.ContinuationToken, + count: tagRequest.Count, + sortOrder: tagRequest.SortOrder); + default: + return LoadChunkInternal( + selectIngest: feedRequest.SelectIngest, + includeNsfw: feedRequest.IncludeNsfw, + query: null, + tagId: null, + continuationToken: feedRequest.ContinuationToken, + count: feedRequest.Count, + sortOrder: feedRequest.SortOrder); + } + } + + private Media[] LoadChunkInternal( + bool selectIngest, + bool includeNsfw, + string? query, + Guid? tagId, + Guid? continuationToken, + int count, + SortOrder sortOrder) { + + if(selectIngest && !includeNsfw) + return Array.Empty<Media>(); + + using var db = dbFactory.CreateDbContext(); + + IQueryable<Media> media = db.Media + .AsSingleQuery() + .AsNoTracking() + .Include(m => m.Tags) + .Include(m => m.CurrentUploadedFile); + + if(!includeNsfw) + media = media + .Where(m => !TagsThatImply(db, HBContext.NsfwTag) + .Intersect(m.Tags.Select(t => t.TagDefinitionId)) + .Any()); + + if(selectIngest) { + media = media + .Where(m => m.Tags + .Select(t => t.TagDefinitionId) + .Contains((int) HBObjectId.IngestTag)); + } else { + media = media + .Where(m => !m.Tags + .Select(t => t.TagDefinitionId) + .Contains((int) HBObjectId.IngestTag)); + } + + if(query is not null) { + media = Search(media, query); + } else if(tagId is not null) { + media = media + .Where(m => TagsThatImply(db, (Guid) tagId) + .Intersect(m.Tags.Select(t => t.TagDefinitionId)) + .Any()); + } + + if(continuationToken is not null) + media = media + .Where(m => m.ObjectId > db.Media.First(m => m.Guid == continuationToken).ObjectId); + + switch(sortOrder) { + case SortOrder.ObjectId: + media = media.OrderBy(m => m.ObjectId); + break; + case SortOrder.LastWriteTime: + media = media.OrderBy(m => m.CurrentUploadedFile!.LastWriteTime); + break; + case SortOrder.Random: + media = media.OrderBy(m => EF.Functions.Random()); + break; + } + + return media + .Take(count) + .ToArray(); + } + + private static IQueryable<Media> Search(IQueryable<Media> media, string query) { + // TODO: search implicit tags as well + + query = query.ToLower().Trim(); + + return media + .Where(m => + (m.ShortDescription != null && m.ShortDescription.ToLower().Contains(query)) || + (m.LongDescription != null && m.LongDescription.ToLower().Contains(query)) || + (m.UploadedFiles.Any(uf => uf.Filename != null && uf.Filename.ToLower().Contains(query))) || + (m.OcrData != null && m.OcrData.SearchableText.ToLower().Contains(query)) || + (m.Tags.Any(t => t.TagDefinition.Name.ToLower().Contains(query)))); + } + + private static IQueryable<int> TagsThatImply(HBContext db, Guid tagId) => + db.Database.SqlQueryRaw<int>(""" + WITH RECURSIVE basetag AS ( + SELECT "ObjectId" FROM "Objects" WHERE "Guid" = {0} + ), + impliedtags AS ( + SELECT + "TagDefinitionObjectId" + FROM + "TagDefinitionTagDefinition" + INNER JOIN + basetag + ON + "ImplicitTagsObjectId" = basetag."ObjectId" + UNION + SELECT + "TagDefinitionTagDefinition"."TagDefinitionObjectId" + FROM + "TagDefinitionTagDefinition" + INNER JOIN + impliedtags + ON + impliedtags."TagDefinitionObjectId" = "TagDefinitionTagDefinition"."ImplicitTagsObjectId" + ) + SELECT DISTINCT + "TagDefinitionObjectId" AS "Value" + FROM impliedtags + UNION + SELECT + "ObjectId" AS "Value" + FROM + basetag + """, tagId); +} diff --git a/Services/MediaService.cs b/Services/MediaService.cs new file mode 100644 index 0000000..2d1533c --- /dev/null +++ b/Services/MediaService.cs @@ -0,0 +1,400 @@ +using HyperBooru.ApiModels; +using ImageMagick; +using Microsoft.EntityFrameworkCore; +using MimeDetective; +using MimeDetective.Definitions; +using System.Security.Cryptography; +using System.Text.RegularExpressions; + +namespace HyperBooru.Services; + +public interface IMediaService { + public void SetDescription( + Media media, + string? shortDescription, + string? longDescription); + + public void SetIngest(Media media, bool ingest); + + public Media Create( + Stream fileData, + string fileName, + string? checksum = null, + DateTime? lastAccessTime = null, + DateTime? lastWriteTime = null, + DateTime? createTime = null, + string? path = null, + PathType? pathType = null, + Guid[]? tagIds = null); + + public void Delete(Guid media); + public void Delete(Media media); + public void DeleteThumbnails(Guid media); + public void DeleteThumbnails(Media media); + public Stream GetThumbnail(Guid media, int? width, int? height); + public Stream GetThumbnail(Media media, int? width, int? height); + public Stream GetConverted(Guid mediaId, string mimeType = "image/png"); + public Stream GetConverted(Media media, string mimeType = "image/png"); + public string GetPath(Guid media); + public string GetPath(Media media); + +} + +public class MediaService : IMediaService { + private readonly Dictionary<string,MagickFormat> FormatMap = new() { + ["image/jpeg"] = MagickFormat.Jpeg, + ["image/jpg"] = MagickFormat.Jpg, + ["image/png"] = MagickFormat.Png, + ["image/webp"] = MagickFormat.WebP + }; + + private IDbContextFactory<HBContext> dbFactory; + private IConfigService config; + + private IContentInspector inspector; + + public MediaService(IDbContextFactory<HBContext> dbFactory, + IConfigService config) { + + this.dbFactory = dbFactory; + this.config = config; + + ContentInspectorBuilder inspectorBuilder = new() { + Definitions = + DefaultDefinitions.FileTypes.Images.All() + .Union(DefaultDefinitions.FileTypes.Video.All()) + .ToList() + }; + + inspector = inspectorBuilder.Build(); + } + + public void SetIngest(Media media, bool ingest) { + using var db = dbFactory.CreateDbContext(); + media = db.Media + .Include(m => m.Tags) + .ThenInclude(t => t.TagDefinition) + .First(m => m.Guid == media.Guid); + var ingestTag = db.TagDefinitions + .First(td => td.Guid == HBContext.IngestTag); + + if(ingest) { + if(!media.Tags.Select(t => t.TagDefinition.Guid).Contains(HBContext.IngestTag)) + media.Tags.Add(new(ingestTag)); + } else { + media.Tags.RemoveAll(t => t.TagDefinition.Guid == HBContext.IngestTag); + } + + db.SaveChanges(); + } + + public void SetDescription( + Media media, + string? shortDescription, + string? longDescription) { + + using var db = dbFactory.CreateDbContext(); + var m = db.Media.First(m => m.Guid == media.Guid); + + shortDescription = shortDescription?.Trim(); + longDescription = longDescription?.Trim(); + + if(string.IsNullOrEmpty(shortDescription)) + shortDescription = null; + if(string.IsNullOrEmpty(longDescription)) + longDescription = null; + + m.ShortDescription = shortDescription; + m.LongDescription = longDescription; + + db.SaveChanges(); + } + + public Media Create( + Stream fileData, + string fileName, + string? checksum = null, + DateTime? lastAccessTime = null, + DateTime? lastWriteTime = null, + DateTime? createTime = null, + string? path = null, + PathType? pathType = null, + Guid[]? tagIds = null) { + + using var db = dbFactory.CreateDbContext(); + using var transaction = db.Database.BeginTransaction(); + + if(fileData.Length == 0) + throw new MediaCreateException("File is empty"); + + // Calculate the checksum using the in-memory file contents + var hash = BitConverter + .ToString(MD5.Create().ComputeHash(fileData)) + .Replace("-", "") + .ToLower(); + + if(checksum is not null && hash != checksum.ToLower()) + throw new MediaCreateException("Checksum does not match"); + + // Determine the MIME type + fileData.Seek(0, SeekOrigin.Begin); + var defs = inspector.Inspect(fileData); + var mime = defs.ByMimeType().FirstOrDefault()?.MimeType; + if(mime is null) + throw new MediaCreateException("Unsupported file type"); + + // Read the image with ImageMagick to determine the width and height + fileData.Seek(0, SeekOrigin.Begin); + using var magickImage = new MagickImage(fileData); + + var media = db.Media + .Include(m => m.UploadedFiles) + .Include(m => m.Tags) + .FirstOrDefault(m => m.UploadedFiles.Any(uf => uf.Checksum == hash)); + + var fileRecord = new UploadedFile() { + Filename = fileName, + Length = fileData.Length, + Checksum = hash, + ChecksumVerified = checksum is not null, + MimeType = mime, + Width = (int) magickImage.Width, + Height = (int) magickImage.Height, + UploadTime = DateTime.UtcNow, + LastAccessTime = lastAccessTime, + LastWriteTime = lastWriteTime, + CreateTime = createTime, + Path = pathType is null ? null : path, + PathType = pathType + }; + + var tags = Array.Empty<TagDefinition>(); + if(tagIds is not null) { + tagIds = tagIds.Distinct().ToArray(); + + tags = db.TagDefinitions + .Where(td => tagIds.Contains(td.Guid)) + .ToArray(); + + if(tags.Count() < tagIds.Count()) { + var badIds = tagIds + .Where(x => !tags.Select(td => td.Guid).Contains(x)) + .Order(); + + throw new MediaCreateException( + $"Non-existent tags specified: {string.Join(", ", badIds)}"); + } + } + + if(media is null) { + var ingestTagDef = db.TagDefinitions + .First(td => td.Guid == HBContext.IngestTag); + + media = new() { + UploadedFiles = new() { + fileRecord + }, + Tags = tags is null ? [ new() { TagDefinition = ingestTagDef } ] : tags + .Select(td => new Tag() { TagDefinition = td }) + .ToList() + }; + + using var newFile = File.Create(GetPath(media)); + + fileData.Seek(0, SeekOrigin.Begin); + fileData.CopyTo(newFile); + newFile.Flush(); + + db.Media.Add(media); + db.SaveChanges(); + media.CurrentUploadedFile = fileRecord; + db.SaveChanges(); + } else { + var fileHashes = media.UploadedFiles + .Select(uf => GetUploadedFileHash(uf)); + // Only add the uploaded file record if it contains new information + if(!fileHashes.Contains(GetUploadedFileHash(fileRecord))) + media.UploadedFiles.Add(fileRecord); + // Add new tags if needed + var missingTags = tags + .Where(td => !media.Tags.Select(t => t.TagDefinition.Guid).Contains(td.Guid)); + media.Tags.AddRange(missingTags.Select(td => new Tag() { TagDefinition = td })); + db.Update(media); + db.SaveChanges(); + } + + transaction.Commit(); + + return media; + } + + public void Delete(Guid media) { + using var db = dbFactory.CreateDbContext(); + var m = db.Media.First(m => m.Guid == media); + + var path = Path.Join( + config.MediaBasePath, + m.Guid.ToString().Substring(0, 2), + m.Guid.ToString().Substring(2, 2), + m.Guid.ToString()); + + try { + var fileInfo = new FileInfo(path); + fileInfo.Delete(); + fileInfo.Directory?.Delete(); + fileInfo.Directory?.Parent?.Delete(); + } catch(IOException) {} + + try { + DeleteThumbnails(media); + } catch {} + + db.Media.Remove(m); + db.SaveChanges(); + } + + public void Delete(Media media) => + Delete(media.Guid); + + public void DeleteThumbnails(Guid media) { + var dir = new DirectoryInfo(Path.Join( + config.ThumbnailBasePath, + media.ToString().Substring(0, 2), + media.ToString().Substring(2, 2))); + + var pattern = new Regex($"^{media}-[0-9]+-[0-9]+$"); + var toDelete = dir.GetFiles() + .Where(f => pattern.IsMatch(f.Name)) + .ToList(); + + List<Exception> exceptions = new(); + + foreach(var file in toDelete) { + try { + file.Delete(); + } catch(Exception e) { + exceptions.Add(e); + } + } + + try { + dir.Delete(); + dir.Parent?.Delete(); + } catch(Exception e) { + exceptions.Add(e); + } + + // TODO: wrap the AggregateException in a ThumbnailException + if(exceptions.Count() > 1) + throw new AggregateException(exceptions); + } + + public void DeleteThumbnails(Media media) => + DeleteThumbnails(media.Guid); + + public Stream GetThumbnail(Guid mediaId, int? width, int? height) { + if(width is null && height is null) + throw new ThumbnailException( + "Both width and height cannot be null!", + mediaId); + + var thumbPath = GetThumbnailPath(mediaId, width, height); + + if(File.Exists(thumbPath)) + return System.IO.File.OpenRead(thumbPath); + + if(!File.Exists(GetPath(mediaId))) + throw new ObjectNotFoundException([ mediaId ]); + + using var image = new MagickImage(GetPath(mediaId)); + + if(width > image.Width || height > image.Height) { + width = (int) image.Width; + height = (int) image.Height; + } + + image.Thumbnail((uint) (width ?? -1), (uint) (height ?? -1)); + image.Write(thumbPath, MagickFormat.Jpeg); + + return System.IO.File.OpenRead(thumbPath); + } + + public Stream GetConverted(Guid mediaId, string mimeType) { + if(!FormatMap.TryGetValue(mimeType, out var format)) + throw new MediaException($"Cannot convert to unknown format ({mimeType})", mediaId); + + var convertedPath = GetConvertedPath(mediaId, mimeType); + + if(File.Exists(convertedPath)) + return System.IO.File.OpenRead(convertedPath); + + if(!File.Exists(GetPath(mediaId))) + throw new ObjectNotFoundException([ mediaId ]); + + using var image = new MagickImage(GetPath(mediaId)); + image.Write(convertedPath, format); + + return System.IO.File.OpenRead(convertedPath); + } + + public Stream GetThumbnail(Media media, int? width, int? height) => + GetThumbnail(media.Guid, width, height); + + public Stream GetConverted(Media media, string mimeType) => + GetConverted(media.Guid, mimeType); + + public string GetPath(Guid mediaId) { + var fileInfo = new FileInfo( + Path.Join( + config.MediaBasePath, + mediaId.ToString().Substring(0, 2), + mediaId.ToString().Substring(2, 2), + mediaId.ToString())); + + Directory.CreateDirectory(fileInfo.Directory!.FullName); + return fileInfo.FullName; + } + + public string GetThumbnailPath(Guid mediaId, int? width, int? height) { + if(width is null && height is null) + throw new ThumbnailException( + "Both width and height cannot be null!", + mediaId); + + var fileInfo = new FileInfo(Path.Join( + config.ThumbnailBasePath, + mediaId.ToString().Substring(0, 2), + mediaId.ToString().Substring(2, 2), + $"{mediaId.ToString()}-{(width ?? 0)}-{(height ?? 0)}")); + + Directory.CreateDirectory(fileInfo.Directory!.FullName); + return fileInfo.FullName; + } + + public string GetConvertedPath(Guid mediaId, string mimeType) { + var fileInfo = new FileInfo(Path.Join( + config.ConvertedMediaBasePath, + mediaId.ToString().Substring(0, 2), + mediaId.ToString().Substring(2, 2), + $"{mediaId.ToString()}-{mimeType.Split('/')[1]}")); + + Directory.CreateDirectory(fileInfo.Directory!.FullName); + return fileInfo.FullName; + } + + public string GetPath(Media media) => + GetPath(media.Guid); + + public string GetThumbnailPath(Media media, int? width, int? height) => + GetThumbnailPath(media.Guid, width, height); + + public string GetConvertedPath(Media media, string mimeType) => + GetConvertedPath(media.Guid, mimeType); + + private int GetUploadedFileHash(UploadedFile uf) => ( + uf.CreateTime, + uf.LastWriteTime, + uf.Filename, + uf.Length, + uf.Checksum).GetHashCode(); +} diff --git a/Services/OcrService.cs b/Services/OcrService.cs new file mode 100644 index 0000000..d43db2e --- /dev/null +++ b/Services/OcrService.cs @@ -0,0 +1,128 @@ +using HyperBooru.Util; +using Microsoft.EntityFrameworkCore; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; +using Tesseract; + +namespace HyperBooru.Services; + +public class OcrService : IHostedService { + private readonly string[] InvalidMimeTypes = [ "image/heic", "image/webp" ]; + + private readonly TimeSpan ProcessInterval = TimeSpan.FromMinutes(30); + private readonly TimeSpan StartupDelay = TimeSpan.FromSeconds(30); + + private readonly Regex SpaceRegex = new(@"[^0-9a-z]+", RegexOptions.Compiled); + + private Task? task; + private CancellationTokenSource cts = new(); + + private Timer timer; + + private IConfigService configService; + private IServiceScopeFactory scopeFactory; + private ILogger<OcrService> logger; + private IDbContextFactory<HBContext> dbFactory; + + public OcrService( + IConfigService configService, + IServiceScopeFactory scopeFactory, + ILogger<OcrService> logger, + IDbContextFactory<HBContext> dbFactory) { + + this.configService = configService; + this.scopeFactory = scopeFactory; + this.logger = logger; + this.dbFactory = dbFactory; + + timer = new((object? state) => { + if(task is not null && !task.IsCompleted) + return; + cts = new(); + task = ProcessAllAsync(cts.Token); + }); + } + + public Task StartAsync(CancellationToken ct) { + if(configService.EnableOcr) { + logger.LogInformation("Service starting..."); + timer.Change(StartupDelay, ProcessInterval); + } + + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken ct) { + logger.LogInformation("Service stopping..."); + timer.Change(Timeout.Infinite, Timeout.Infinite); + cts.Cancel(); + return Task.CompletedTask; + } + + async Task ProcessAllAsync(CancellationToken ct) { + using var scope = scopeFactory.CreateScope(); + var mediaService = scope.ServiceProvider + .GetRequiredService<IMediaService>(); + + using var db = dbFactory.CreateDbContext(); + Guid[] guids = db.Media + .AsNoTracking() + .Include(m => m.CurrentUploadedFile) + .Include(m => m.OcrData) + .Where(m => m.OcrData == null) + .Where(m => m.CurrentUploadedFile!.MimeType.Contains("image/")) + .Where(m => !InvalidMimeTypes.Contains(m.CurrentUploadedFile!.MimeType)) + .Select(m => m.Guid) + .ToArray(); + db.Dispose(); + + logger.LogInformation($"Performing OCR pass on {guids.Count()} media items"); + + var factory = new TaskFactory(new LimitedConcurrencyTaskScheduler()); + var tasks = new List<Task>(); + + var stopwatch = Stopwatch.StartNew(); + + foreach(var guid in guids) + tasks.Add(factory.StartNew(() => Process(guid, mediaService), ct)); + + await Task.WhenAll(tasks); + stopwatch.Stop(); + + var time = stopwatch.Elapsed.ToStringHumanReadable(); + logger.LogInformation( + $"Performed OCR pass on {guids.Count()} media items in {time}"); + } + + private void Process(Guid media, IMediaService mediaService) { + logger.LogDebug($"Performing OCR on media item {media}"); + + using var db = dbFactory.CreateDbContext(); + var m = db.Media + .Include(m => m.OcrData) + .First(m => m.Guid == media); + + OcrData o = m.OcrData ?? new(); + + using var engine = new TesseractEngine("tessdata", "eng", EngineMode.Default); + using var image = Pix.LoadFromFile(mediaService.GetPath(m)); + engine.SetVariable("debug_file", NullFile); + + o.Timestamp = DateTime.UtcNow; + o.Text = engine.Process(image).GetText().Trim(); + o.SearchableText = SpaceRegex.Replace(o.Text.ToLower(), " ").Trim(); + + m.OcrData = o; + db.SaveChanges(); + } + + private string NullFile { + get { + if(RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + return "NUL"; + else + return "/dev/null"; + } + } +} diff --git a/Services/SourceService.cs b/Services/SourceService.cs new file mode 100644 index 0000000..d145346 --- /dev/null +++ b/Services/SourceService.cs @@ -0,0 +1,20 @@ +using System.Text.RegularExpressions; + +namespace HyperBooru.Services; + +public interface ISourceService { + public string? GetUrlFromFilename(string filename); +} + +public class SourceService : ISourceService { + private Regex PixivRegex = + new(@"^([0-9]+)_p[0-9]+(_master1200)?\.[^.]+$", RegexOptions.Compiled); + + public string? GetUrlFromFilename(string filename) { + var pixivMatch = PixivRegex.Match(filename); + if(pixivMatch.Success) + return $"https://pixiv.net/en/artworks/{pixivMatch.Groups[1].Value}"; + + return null; + } +} diff --git a/Services/TagService.cs b/Services/TagService.cs new file mode 100644 index 0000000..f7b91dc --- /dev/null +++ b/Services/TagService.cs @@ -0,0 +1,305 @@ +using HyperBooru.ApiModels; +using Microsoft.EntityFrameworkCore; + +namespace HyperBooru.Services; + +public interface ITagService { + public void AddTag(Guid obj, Guid tagDef); + public void AddTag(HBObject obj, TagDefinition tagDef); + public void RemoveTag(Guid obj, Guid tagDef); + public void RemoveTag(HBObject obj, TagDefinition tagDef); + public void SetImplicitTags(TagDefinition tagDef, TagDefinition[] implicitTagDefs); + public void SetImplicitTags(Guid tagDef, Guid[] implicitTagDefs); + public void AddImplicitTag(Guid tagDef, Guid implicitTagDef); + public void AddImplicitTag(TagDefinition tagDef, TagDefinition implicitTagDef); + public void RemoveImplicitTag(Guid tagDef, Guid implicitTagDef); + public void RemoveImplicitTag(TagDefinition tagDef, TagDefinition implicitTagDef); + public void CreateTagDefinition(string name, string? @namespace = null, string? alias = null); + public void DeleteTagDefinition(Guid tagDef); + public void DeleteTagDefinition(TagDefinition tagDef); + public void UpdateTagDefinition(Guid tagDef, string name, string? @namespace = null, string? alias = null); + public void UpdateTagDefinition(TagDefinition tagDef, string name, string? @namespace = null, string? alias = null); + public (TagDefinition tagDefinition, bool isImplicit)[] GetAllTags(Guid obj); + public (TagDefinition tagDefinition, bool isImplicit)[] GetAllTags(HBObject obj); + public (TagDefinition tagDefinition, bool isImplicit)[] GetAllTags(TagDefinition tagDef); + public TagDefinition[] TagsThatImply(Guid tagDef); + public TagDefinition[] TagsThatImply(TagDefinition tagDef); +} + +public class TagService : ITagService { + private IDbContextFactory<HBContext> dbFactory; + + public TagService(IDbContextFactory<HBContext> dbFactory) => + this.dbFactory = dbFactory; + + public void AddTag(Guid obj, Guid tagDef) { + using var db = dbFactory.CreateDbContext(); + + var tag = db.TagDefinitions.First(td => td.Guid == tagDef); + + db.Objects + .Include(o => o.Tags) + .ThenInclude(t => t.TagDefinition) + .Where(o => !o.Tags.Select(t => t.TagDefinition.Guid).Contains(tagDef)) + .FirstOrDefault(o => o.Guid == obj)? + .Tags + .Add(new(tag)); + + db.SaveChanges(); + } + + public void AddTag(HBObject obj, TagDefinition tagDef) => + AddTag(obj.Guid, tagDef.Guid); + + public void RemoveTag(Guid obj, Guid tagDef) { + using var db = dbFactory.CreateDbContext(); + + db.Objects + .Include(o => o.Tags) + .ThenInclude(t => t.TagDefinition) + .First(o => o.Guid == obj) + .Tags + .RemoveAll(t => t.TagDefinition.Guid == tagDef); + + db.SaveChanges(); + } + + public void RemoveTag(HBObject obj, TagDefinition tagDef) => + RemoveTag(obj.Guid, tagDef.Guid); + + public void SetImplicitTags(Guid tagDef, Guid[] implicitTagDefs) { + using var db = dbFactory.CreateDbContext(); + using var transaction = db.Database.BeginTransaction(); + + var tag = db.TagDefinitions + .Include(td => td.ImplicitTags) + .First(td => td.Guid == tagDef); + + tag.ImplicitTags.RemoveAll(td => !implicitTagDefs.Contains(td.Guid)); + tag.ImplicitTags.AddRange( + db.TagDefinitions + .Where(td => implicitTagDefs.Contains(td.Guid)) + .Where(td => !tag.ImplicitTags + .Select(td => td.Guid) + .Contains(td.Guid))); + + db.SaveChanges(); + transaction.Commit(); + } + + public void SetImplicitTags(TagDefinition tagDef, TagDefinition[] implicitTagDefs) => + SetImplicitTags(tagDef.Guid, implicitTagDefs.Select(td => td.Guid).ToArray()); + + public void AddImplicitTag(Guid tagDef, Guid implicitTagDef) { + using var db = dbFactory.CreateDbContext(); + + var tag = db.TagDefinitions + .Include(td => td.ImplicitTags) + .First(td => td.Guid == tagDef); + var implicitTag = db.TagDefinitions.First(td => td.Guid == implicitTagDef); + + tag.ImplicitTags.Add(implicitTag); + db.SaveChanges(); + } + + public void AddImplicitTag(TagDefinition tagDef, TagDefinition implicitTagDef) => + AddImplicitTag(tagDef, implicitTagDef); + + public void RemoveImplicitTag(Guid tagDef, Guid implicitTagDef) { + using var db = dbFactory.CreateDbContext(); + + var tag = db.TagDefinitions + .Include(td => td.ImplicitTags) + .First(td => td.Guid == tagDef); + + tag.ImplicitTags.RemoveAll(td => td.Guid == implicitTagDef); + db.SaveChanges(); + } + + public void RemoveImplicitTag(TagDefinition tagDef, TagDefinition implicitTagDef) => + RemoveImplicitTag(tagDef, implicitTagDef); + + public void CreateTagDefinition(string name, string? @namespace = null, string? alias = null) { + using var db = dbFactory.CreateDbContext(); + + if(string.IsNullOrEmpty(@namespace)) + @namespace = null; + if(string.IsNullOrEmpty(alias)) + alias = null; + + // Remove leading and trailing whitespace + name = name.Trim(); + @namespace = @namespace?.Trim(); + alias = alias?.Trim(); + + TagDefinition tagDef = new() { + Source = TagSource.UserTag, + Namespace = @namespace, + Name = name, + Alias = alias + }; + + bool nameExists = db.TagDefinitions.Any(td => td.Name.ToLower() == name.ToLower()); + bool aliasExists = false; + if(alias is not null) + aliasExists = db.TagDefinitions + .Where(td => td.Alias != null) + .Any(td => td.Alias!.ToLower() == alias.ToLower()); + if(nameExists || aliasExists) + throw new TagDuplicateException(nameExists, aliasExists); + + if(!db.TagDefinitions.Contains(tagDef)) + db.TagDefinitions.Add(tagDef); + db.SaveChanges(); + } + + public void DeleteTagDefinition(Guid tagDef) { + using var db = dbFactory.CreateDbContext(); + + var tag = db.TagDefinitions.First(td => td.Guid == tagDef); + + using var transaction = db.Database.BeginTransaction(); + + db.Tags.RemoveRange( + db.Tags + .Include(t => t.TagDefinition) + .Where(t => t.TagDefinition.Guid == tagDef)); + db.TagDefinitions.Remove(tag); + db.SaveChanges(); + + transaction.Commit(); + } + + public void DeleteTagDefinition(TagDefinition tagDef) => + DeleteTagDefinition(tagDef.Guid); + + public void UpdateTagDefinition(Guid tagDef, string name, string? @namespace = null, string? alias = null) { + using var db = dbFactory.CreateDbContext(); + + if(string.IsNullOrEmpty(@namespace)) + @namespace = null; + if(string.IsNullOrEmpty(alias)) + alias = null; + + // Remove leading and trailing whitespace + name = name.Trim(); + @namespace = @namespace?.Trim(); + alias = alias?.Trim(); + + var tag = db.TagDefinitions.First(td => td.Guid == tagDef); + + TagDefinition? nameExisting = db.TagDefinitions.FirstOrDefault(td => td.Name.ToLower() == name.ToLower()); + TagDefinition? aliasExisting = null; + if(alias is not null) + aliasExisting = db.TagDefinitions + .Where(td => td.Alias != null) + .FirstOrDefault(td => td.Alias!.ToLower() == alias.ToLower()); + bool nameExists = nameExisting is not null && nameExisting != tag; + bool aliasExists = aliasExisting is not null && aliasExisting != tag; + if(nameExists || aliasExists) + throw new TagDuplicateException(nameExists, aliasExists); + + tag.Name = name; + tag.Namespace = @namespace; + tag.Alias = alias; + + db.SaveChanges(); + } + + public void UpdateTagDefinition(TagDefinition tagDef, string name, string? @namespace = null, string? alias = null) => + UpdateTagDefinition(tagDef.Guid, name, @namespace, alias); + + private (TagDefinition tagDefinition, bool isImplicit)[] GetAllTags(IEnumerable<TagDefinition> tagDefs) { + using var db = dbFactory.CreateDbContext(); + + var tagGuids = tagDefs + .Select(td => td.Guid) + .ToArray(); + + // Query all tag definitions + var allTags = db.TagDefinitions + .Include(td => td.ImplicitTags) + .ToArray(); + + var tags = new List<TagDefinition>( + allTags.IntersectBy( + tagGuids, + td => td.Guid)); + + while(true) { + var toAdd = tags + .SelectMany(td => td.ImplicitTags) + .ExceptBy(tags.Select(td => td.Guid), td => td.Guid) + .ToArray(); + + if(toAdd.Count() == 0) + break; + + tags.AddRange(toAdd); + } + + return tags + .Select(td => new ValueTuple<TagDefinition, bool>(td, !tagGuids.Contains(td.Guid))) + .ToArray(); + } + + public (TagDefinition tagDefinition, bool isImplicit)[] GetAllTags(Guid obj) { + using var db = dbFactory.CreateDbContext(); + + // Query a list of tag GUIDs for this object + var tags = db.Objects + .Include(o => o.Tags) + .ThenInclude(t => t.TagDefinition) + .First(o => o.Guid == obj) + .Tags + .Select(t => t.TagDefinition) + .ToArray(); + + return GetAllTags(tags); + } + + + public (TagDefinition tagDefinition, bool isImplicit)[] GetAllTags(HBObject obj) => + GetAllTags(obj.Guid); + + public (TagDefinition tagDefinition, bool isImplicit)[] GetAllTags(TagDefinition tagDef) { + using var db = dbFactory.CreateDbContext(); + + var tags = db.TagDefinitions + .Include(td => td.ImplicitTags) + .First(td => td.Guid == tagDef.Guid) + .ImplicitTags + .ToArray(); + + return GetAllTags(tags); + } + + public TagDefinition[] TagsThatImply(Guid tagDef) { + using var db = dbFactory.CreateDbContext(); + + var tagDefs = db.TagDefinitions + .Include(td => td.ImplicitTags) + .ToArray(); + + var tags = new List<TagDefinition>() { + db.TagDefinitions.First(td => td.Guid == tagDef) + }; + + while(true) { + var toAdd = tagDefs + .Where(td => td.ImplicitTags.Select(it => it.Guid).Intersect(tags.Select(td => td.Guid)).Any()) + .ExceptBy(tags.Select(td => td.Guid), td => td.Guid) + .ToArray(); + + if(toAdd.Count() == 0) + break; + + tags.AddRange(toAdd); + } + + return tags.ToArray(); + } + + public TagDefinition[] TagsThatImply(TagDefinition tagDef) => + TagsThatImply(tagDef.Guid); +} diff --git a/Services/UserService.cs b/Services/UserService.cs new file mode 100644 index 0000000..9e79dc6 --- /dev/null +++ b/Services/UserService.cs @@ -0,0 +1,76 @@ +using Microsoft.AspNetCore.Cryptography.KeyDerivation; + +namespace HyperBooru.Services; + +public interface IUserService { + public UserSessionState UserSessionState { get; } +} + +public class UserService : IUserService { + public UserSessionState UserSessionState => + globalUserService.GetSessionState(httpContext.Session.Id); + + private IHttpContextAccessor httpContextAccessor; + private IGlobalUserService globalUserService; + + private HttpContext httpContext => + httpContextAccessor.HttpContext!; + + public UserService( + IHttpContextAccessor httpContextAccessor, + IGlobalUserService globalUserService) { + + this.httpContextAccessor = httpContextAccessor; + this.globalUserService = globalUserService; + + // HTTP context session states are discarded if no values + // are set. Set a dummy value so that the session state + // will not be discarded later when we actually need it. + httpContext.Session.SetInt32("Persist", 1); + } + + public static string HashPassword(string password) => + Convert.ToBase64String( + KeyDerivation.Pbkdf2( + password, + Array.Empty<byte>(), + KeyDerivationPrf.HMACSHA512, + 100_000, + 512 / 8)); +} + +public interface IGlobalUserService { + public UserSessionState GetSessionState(string id); +} + +public class GlobalUserService : IGlobalUserService { + // TODO: prune this list periodically + private Dictionary<string, UserSessionState> sessionStates = new(); + + public UserSessionState GetSessionState(string id) { + sessionStates.TryGetValue(id, out var state); + + if(state is null) { + state = new(); + sessionStates[id] = state; + } + + return state; + } +} + +public record UserSessionState { + public event UserSessionStateChange OnStateChange; + + public bool ShowNsfw { + get => showNsfw; + set { + showNsfw = value; + OnStateChange.Invoke(this); + } + } + + private bool showNsfw = false; +} + +public delegate void UserSessionStateChange(UserSessionState sessionState); |