initial commit

2026-06-24 14:25:32 +01:00
commit 9788479ae7
109 changed files with 8666 additions and 0 deletions
@@ -0,0 +1,54 @@
+using System.Diagnostics.CodeAnalysis;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
+using Quartz;
+
+namespace Solicitors.CacheBuild;
+
+[SuppressMessage("ReSharper", "InconsistentNaming")]
+public static class DIExtensions
+{
+    public static IServiceCollection AddCacheBuild(this IServiceCollection services)
+    {
+        return services
+            .AddScoped<ISolicitorParser, SolicitorsDotCom.SolicitorParser>()
+            .AddScoped<ISolicitorImporter, SolicitorImporter>()
+            .AddQuartz()
+            .AddQuartzHostedService(opt => opt.WaitForJobsToComplete = true);
+    }
+
+    public static async Task UseCacheBuildAsync(this IServiceScope scope, CancellationToken cancellationToken)
+    {
+        var factory = scope.ServiceProvider.GetRequiredService<ISchedulerFactory>();
+        var options = scope.ServiceProvider.GetRequiredService<IOptions<ImportConfiguration>>().Value;
+        var scheduler = await factory.GetScheduler(cancellationToken);
+
+        var importJob = JobBuilder.Create<ImportRunner>()
+            .WithIdentity("ImportJob", "ImportGroup")
+            .Build();
+        
+        var importTrigger = TriggerBuilder.Create()
+            .WithIdentity("ImportRunner", "ImportGroup")
+            .StartNow()
+            .WithSimpleSchedule(builder => builder
+                .WithIntervalInMinutes(options.ImportMinutes)
+                .RepeatForever())
+            .Build();
+        
+        await scheduler.ScheduleJob(importJob, importTrigger, cancellationToken);
+
+        var staleJob = JobBuilder.Create<StaleDataRemover>()
+            .WithIdentity("StaleDataRemover", "RemoveGroup")
+            .Build();
+
+        var staleTrigger = TriggerBuilder.Create()
+            .WithIdentity("StaleDataTrigger", "RemoveGroup")
+            .StartNow()
+            .WithSimpleSchedule(builder => builder
+                .WithIntervalInMinutes(options.RemoveMinutes)
+                .RepeatForever())
+            .Build();
+        
+        await scheduler.ScheduleJob(staleJob, staleTrigger, cancellationToken);
+    }
+}
@@ -0,0 +1,6 @@
+namespace Solicitors.CacheBuild;
+
+public interface ISolicitorImporter
+{
+    Task RunFullImport(CancellationToken cancellationToken);
+}
@@ -0,0 +1,9 @@
+using Solicitors.Core.Models.Imports;
+
+namespace Solicitors.CacheBuild;
+
+internal interface ISolicitorParser
+{
+    Task<SolicitorData[]> GetSolicitorsAsync(
+        CancellationToken cancellationToken = default);
+}
@@ -0,0 +1,8 @@
+namespace Solicitors.CacheBuild;
+
+public class ImportConfiguration
+{
+    public int StaleMinutes { get; set; }
+    public int RemoveMinutes { get; set; }
+    public int ImportMinutes { get; set; }
+}
@@ -0,0 +1,15 @@
+using Quartz;
+using Solicitors.Core.Data;
+
+namespace Solicitors.CacheBuild;
+
+internal class ImportRunner(
+    ISolicitorImporter importer,
+    ISolicitorRepository repository) : IJob
+{
+    public async Task Execute(IJobExecutionContext context)
+    {
+        await repository.EnsureCreatedAsync(context.CancellationToken);
+        await importer.RunFullImport(context.CancellationToken);
+    }
+}
@@ -0,0 +1,20 @@
+using Solicitors.Core.Data;
+using Solicitors.Core.Models.Imports;
+
+namespace Solicitors.CacheBuild;
+
+internal class SolicitorImporter(
+    ISolicitorParser parser,
+    ISolicitorRepository repository) : ISolicitorImporter
+{
+    public async Task RunFullImport(CancellationToken cancellationToken)
+    {
+        foreach (var solicitor in await parser.GetSolicitorsAsync(cancellationToken))
+        {
+           await RunImport(solicitor, cancellationToken); 
+        }
+    }
+
+    private Task RunImport(SolicitorData solicitor, CancellationToken cancellationToken) 
+        => repository.AddOrUpdateSolicitorAsync(solicitor, cancellationToken);
+}
@@ -0,0 +1,27 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net10.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Solicitors.Core\Solicitors.Core.csproj" />
+      <ProjectReference Include="..\Solicitors.HtmlParsing\Solicitors.HtmlParsing.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.9" />
+      <PackageReference Include="Quartz" Version="3.18.1" />
+      <PackageReference Include="Quartz.Extensions.DependencyInjection" Version="3.18.1" />
+      <PackageReference Include="Quartz.Extensions.Hosting" Version="3.18.1" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <Reference Include="Microsoft.Extensions.Hosting.Abstractions">
+        <HintPath>..\..\..\..\..\..\..\.dotnet\shared\Microsoft.AspNetCore.App\10.0.7\Microsoft.Extensions.Hosting.Abstractions.dll</HintPath>
+      </Reference>
+    </ItemGroup>
+
+</Project>
@@ -0,0 +1,14 @@
+namespace Solicitors.CacheBuild.SolicitorsDotCom;
+
+internal static class Consts
+{
+    public const string BaseUrl = "https://www.solicitors.com";
+    public static readonly string[] Locations =
+    [
+        "london", "birmingham", "leeds", "manchester", "sheffield", "bradford", "liverpool", "bristol"
+    ];
+
+    public const decimal MaxRating = 5.0m;
+    public const string Name = "Solicitors.com";
+    public const string IconPath = "/images/logo.svg";
+}
@@ -0,0 +1,31 @@
+namespace Solicitors.CacheBuild.SolicitorsDotCom;
+
+internal class SolicitorBuilder
+{
+    private readonly HashSet<string> _baseLocations = [];
+
+    public SolicitorBuilder(
+        string name,
+        string path,
+        string? shortDesc,
+        string baseLocation)
+    {
+        Name = name;
+        ShortDescription = shortDesc;
+        _baseLocations.Add(baseLocation);
+        
+        Path = path;
+    }
+
+    public void AddBaseLocation(string baseLocation)
+    {
+        _baseLocations.Add(baseLocation);
+    }
+    
+    public string Path { get; }
+    public string Name { get; }
+
+    public string? ShortDescription { get; }
+
+    public string[] BaseLocations => _baseLocations.ToArray();
+}
@@ -0,0 +1,315 @@
+using System.Diagnostics.CodeAnalysis;
+using Solicitors.Core.Misc;
+using Solicitors.Core.Models.Imports;
+using Solicitors.HtmlParsing;
+using Solicitors.HtmlParsing.Models;
+
+namespace Solicitors.CacheBuild.SolicitorsDotCom;
+
+internal class SolicitorParser : ISolicitorParser
+{
+    private readonly HttpClient _client;
+    private readonly IHtmlParser _htmlParser;
+
+    public SolicitorParser(
+        HttpClient client,
+        IHtmlParser htmlParser)
+    {
+        _client = client;
+        _client.BaseAddress = new Uri(Consts.BaseUrl);
+        
+        _htmlParser = htmlParser;
+    }
+
+    public async Task<SolicitorData[]> GetSolicitorsAsync(
+        CancellationToken cancellationToken = default)
+    {
+        var builders = new Dictionary<string, SolicitorBuilder>();
+        
+        foreach (var location in Consts.Locations)
+        {
+            var httpRequest = new HttpRequestMessage(HttpMethod.Get, $"/conveyancing+{location}.html");
+            httpRequest.Headers.Add("User-Agent", "Solicitors API Cache");
+            var httpResponse = await _client.SendAsync(httpRequest, cancellationToken);
+            var html = await httpResponse.Content.ReadAsStringAsync(cancellationToken);
+            var nodes = _htmlParser.ParseHtml(html);
+            IHtmlNode? mainNode = null;
+            foreach (var node in nodes)
+                if (node.TryGetByTagName("main", out mainNode))
+                    break;
+
+            if (TryGetResultSection(mainNode, out var results))
+                ParseResultsSection(results, location, builders);
+        }
+
+        var batches = builders.Values
+            .Select(builder => GetFullSolicitorDataAsync(builder, cancellationToken))
+            .Batch(5);
+
+        var solicitors = new List<SolicitorData?>();
+        foreach (var batch in batches)
+            solicitors.AddRange(await Task.WhenAll(batch));
+        
+        return solicitors
+            .Where(x => x is not null)
+            .Cast<SolicitorData>()
+            .ToArray();
+    }
+
+    private bool TryGetResultSection(
+        IHtmlNode? mainNode,
+        [NotNullWhen(true)] out IEnumerable<IHtmlNode>? results)
+    {
+        results = null;
+        
+        // going one level at a time helps a little with performance here, at the cost of messier code
+        return mainNode is not null
+               && mainNode.TryGetByTagNameAndClass("div", "content-holder", out var contentHolderNode)
+               && contentHolderNode.TryGetByTagNameAndClass("div", "container", out var containerNode)
+               && containerNode.TryGetByTagNameAndClass("div", "content", out var contentNode)
+               && contentNode.TryGetByTagNameAndClass("div", "result-section", out var resultSectionNode)
+               && resultSectionNode.TryGetChildren(out results);
+    }
+
+    private void ParseResultsSection(
+        IEnumerable<IHtmlNode> results,
+        string location,
+        Dictionary<string, SolicitorBuilder> builders)
+    {
+        foreach (var result in results)
+        {
+            if (result.TryGetByClass("result-item", out var resultItem, true)
+                && TryParseToNewBuilder(resultItem, location, out var builder))
+            {
+                if (!builders.TryAdd(builder.Path, builder))
+                    builders[builder.Path].AddBaseLocation(location);
+            }
+        }
+    }
+
+    private bool TryParseToNewBuilder(
+        IHtmlNode resultItemNode,
+        string baseLocation,
+        [NotNullWhen(true)] out SolicitorBuilder? builder)
+    {
+        builder = null;
+        if (!resultItemNode.TryGetByTagNameAndClass("a", "link-map", out var linkMapNode)
+            || !linkMapNode.TryGetAttributeValue("href", out var uniqueUrl)
+            || !resultItemNode.TryGetByTagNameAndClass("span", "h2", out var titleNode)
+            || !titleNode.TryGetChildren(out var titleChildren))
+            return false;
+
+        string? name = null;
+        foreach (var child in titleChildren)
+            if (child.TryGetText(out name))
+                break;
+
+        if (name is null)
+            return false;
+        
+        string? shortDesc = null;
+        if (resultItemNode.TryGetByTagName("p", out var pNode)
+            && pNode.TryGetChildren(out var pChildren))
+            foreach (var child in pChildren)
+                if (child.TryGetText(out shortDesc))
+                    break;
+
+        builder = new SolicitorBuilder(name, uniqueUrl, shortDesc, baseLocation);
+        return true;
+    }
+
+    private async Task<SolicitorData?> GetFullSolicitorDataAsync(
+        SolicitorBuilder builder,
+        CancellationToken cancellationToken)
+    {
+        
+        var httpRequest = new HttpRequestMessage(HttpMethod.Get, builder.Path);
+        httpRequest.Headers.Add("User-Agent", "Solicitors API Cache");
+        var httpResponse = await _client.SendAsync(httpRequest, cancellationToken);
+        var html = await httpResponse.Content.ReadAsStringAsync(cancellationToken);
+        var nodes = _htmlParser.ParseHtml(html);
+        IHtmlNode? mainNode = null;
+        foreach (var node in nodes)
+            if (node.TryGetByTagName("main", out mainNode))
+                break;
+
+        if (mainNode is null || !mainNode.TryGetByTagNameAndClass("div", "content-block", out var contentBlockNode))
+            return null;
+
+        Location[] offices = [];
+        if (contentBlockNode.TryGetByTagNameAndClass("div", "office-item", out var officeItemNode)
+            && officeItemNode.TryGetChildren(out var officeItems))
+        {
+            offices = ParseOffices(officeItems);
+        }
+
+        Rating[] ratings = [];
+        string? phone = null;
+        string? email = null;
+        string? website = null;
+        if (mainNode.TryGetByTagNameAndClass("div", "side-holder", out var sidebarNode)
+            && sidebarNode.TryGetChildren(out var sidebarChildren))
+        {
+            ParseSidebar(sidebarChildren, out phone, out email, out website, out ratings);
+        }
+
+        return new SolicitorData
+        {
+            Name = builder.Name,
+            UrlPath = builder.Path,
+            ShortDescription = builder.ShortDescription,
+            Phone = phone,
+            Email = email,
+            Website = website,
+            Ratings = ratings,
+            Offices = offices,
+            Cities = builder.BaseLocations
+        };
+    }
+
+    private Location[] ParseOffices(IEnumerable<IHtmlNode> officeNodes)
+        => officeNodes
+            .Select(ParseOffice)
+            .Where(x => x is not null)
+            .Cast<Location>()
+            .ToArray();
+
+    private Location? ParseOffice(IHtmlNode officeNode)
+    {
+        if (!officeNode.TryGetChildren(out var officeChildren)
+            || !officeNode.TryGetByTagName("address", out var addressNode)
+            || !addressNode.TryGetChildren(out var addressLines))
+            return null;
+
+        Location? location = null;
+        var address = "";
+        foreach (var line in addressLines)
+        {
+            if (line.TryGetText(out var lineText))
+                address += lineText;
+            else if (line.TryGetByTagName("br", out _, true))
+                address += "\n";
+        }
+
+        if (string.IsNullOrWhiteSpace(address))
+            address = null;
+
+        string? phone = null;
+        foreach (var child in officeChildren)
+        {
+            if (child.TryGetByTagName("a", out var anchor)
+                && anchor.TryGetAttributeValue("href", out var telString)
+                && telString.StartsWith("\"tel:")
+                && telString.Length > 5)
+            {
+                phone = telString.Substring(5, telString.Length - 6);
+                break;
+            }
+        }
+
+        if (address is not null && phone is not null)
+            location = new Location(address, phone, []);
+
+        return location;
+    }
+
+    private void ParseSidebar(
+        IEnumerable<IHtmlNode> sidebarChildren,
+        out string? phone,
+        out string? email,
+        out string? website,
+        out Rating[] ratings)
+    {
+        List<Rating> ratingsList = [];
+        phone = null;
+        website = null;
+        email = null;
+
+        foreach (var node in sidebarChildren)
+        {
+            var isRevCount = node.TryGetByTagNameAndClass("div", "rev-count", out _, true);
+            var isLinksHolder = node.TryGetByTagNameAndClass("div", "links-holder", out _, true);
+            if ((!isRevCount && !isLinksHolder)
+                || !node.TryGetChildren(out var children))
+                continue;
+
+            if (isLinksHolder)
+            {
+                if (node.TryGetByTagNameAndClass("a", "website", out var websiteNode)
+                    && websiteNode.TryGetAttributeValue("href", out website))
+                    website = website.Trim('"');
+
+                if (node.TryGetByTagNameAndClass("a", "phone", out var phoneNode)
+                    && phoneNode.TryGetAttributeValue("href", out var phoneStr)
+                    && phoneStr.StartsWith("\"tel:")
+                    && phoneStr.Length > 5)
+                {
+                    phone = phoneStr.Substring(5, phoneStr.Length - 6);
+                }
+            }
+
+            var childrenArray = children as IHtmlNode[] ?? children.ToArray();
+            foreach (var child in childrenArray.Where(x => x.TryGetByClass("rev-box", out _, true)))
+            {
+                if (child.TryGetAttributeValue("title", out var ratingString)
+                    && TryParseRatingString(ratingString, out var rating, out var maxRating)
+                    && child.TryGetByTagNameAndClass("img", "rev-logo", out var imgNode)
+                    && imgNode.TryGetAttributeValue("src", out var imgSrc)
+                    && TryParseRatingSrc(imgSrc, out var ratingProvider))
+                {
+                    ratingsList.Add(new Rating(rating, maxRating, ratingProvider, imgSrc));
+                }
+            }
+
+            if (ratingsList.Count == 0)
+            {
+                foreach (var child in childrenArray.Where(x => !x.HasAttribute("class")))
+                {
+                    if (child.TryGetChildren(out var grandChildren))
+                        foreach (var grandChild in grandChildren)
+                            if (grandChild.TryGetText(out var textContent)
+                                && TryParseOwnRatingString(textContent, out var rating))
+                                ratingsList.Add(new Rating(rating, Consts.MaxRating, Consts.Name, Consts.IconPath));
+                }
+            }
+        }
+
+        ratings = ratingsList.ToArray();
+    }
+    
+
+    bool TryParseRatingString(string ratingString, out decimal rating, out decimal maxRating)
+    {
+        rating = 0;
+        maxRating = 0;
+        var parts = ratingString.Trim('"').Split(" / ");
+        if (parts.Length != 2)
+            return false;
+
+        return decimal.TryParse(parts[0], out rating) && decimal.TryParse(parts[1], out maxRating);
+    }
+
+    private bool TryParseRatingSrc(string imgSrc, out string provider)
+    {
+        provider = "";
+        imgSrc = imgSrc.Trim('"');
+        const string prefix = "/images/logo-";
+        if (!imgSrc.StartsWith(prefix))
+            return false;
+        
+        var providerAndExtension = imgSrc[prefix.Length..];
+        provider = providerAndExtension.Split('.').First();
+        return true;
+    }
+
+    bool TryParseOwnRatingString(string ratingString, out decimal rating)
+    {
+        rating = 0;
+        const string prefix = "Average review score : ";
+        if (!ratingString.StartsWith(prefix))
+            return false;
+
+        var ratingPart = ratingString[prefix.Length..];
+        return decimal.TryParse(ratingPart, out rating);
+    }
+}
@@ -0,0 +1,19 @@
+using Microsoft.Extensions.Options;
+using Quartz;
+using Solicitors.Core.Data;
+
+namespace Solicitors.CacheBuild;
+
+internal class StaleDataRemover(
+    IOptions<ImportConfiguration> config,
+    ISolicitorRepository repository) : IJob
+{
+    private readonly ImportConfiguration _config = config.Value;
+
+    public Task Execute(IJobExecutionContext context)
+    {
+        return repository.RemoveStaleEntriesAsync(
+            TimeSpan.FromMinutes(_config.StaleMinutes),
+            context.CancellationToken);
+    }
+}