initial commit
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
namespace Solicitors.CacheBuild.SolicitorsDotCom;
|
||||
|
||||
internal static class Consts
|
||||
{
|
||||
public const string BaseUrl = "https://www.solicitors.com";
|
||||
public static readonly string[] Locations =
|
||||
[
|
||||
"london", "birmingham", "leeds", "manchester", "sheffield", "bradford", "liverpool", "bristol"
|
||||
];
|
||||
|
||||
public const decimal MaxRating = 5.0m;
|
||||
public const string Name = "Solicitors.com";
|
||||
public const string IconPath = "/images/logo.svg";
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
namespace Solicitors.CacheBuild.SolicitorsDotCom;
|
||||
|
||||
internal class SolicitorBuilder
|
||||
{
|
||||
private readonly HashSet<string> _baseLocations = [];
|
||||
|
||||
public SolicitorBuilder(
|
||||
string name,
|
||||
string path,
|
||||
string? shortDesc,
|
||||
string baseLocation)
|
||||
{
|
||||
Name = name;
|
||||
ShortDescription = shortDesc;
|
||||
_baseLocations.Add(baseLocation);
|
||||
|
||||
Path = path;
|
||||
}
|
||||
|
||||
public void AddBaseLocation(string baseLocation)
|
||||
{
|
||||
_baseLocations.Add(baseLocation);
|
||||
}
|
||||
|
||||
public string Path { get; }
|
||||
public string Name { get; }
|
||||
|
||||
public string? ShortDescription { get; }
|
||||
|
||||
public string[] BaseLocations => _baseLocations.ToArray();
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Net.Http.Headers;
|
||||
using Solicitors.Core.Misc;
|
||||
using Solicitors.Core.Models.Imports;
|
||||
using Solicitors.HtmlParsing;
|
||||
using Solicitors.HtmlParsing.Models;
|
||||
|
||||
namespace Solicitors.CacheBuild.SolicitorsDotCom;
|
||||
|
||||
internal class SolicitorParser : ISolicitorParser
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
private readonly IHtmlParser _htmlParser;
|
||||
|
||||
public SolicitorParser(
|
||||
HttpClient client,
|
||||
IHtmlParser htmlParser)
|
||||
{
|
||||
_client = client;
|
||||
_client.BaseAddress = new Uri(Consts.BaseUrl);
|
||||
|
||||
_htmlParser = htmlParser;
|
||||
}
|
||||
|
||||
public async Task<SolicitorData[]> GetSolicitorsAsync(
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var builders = new Dictionary<string, SolicitorBuilder>();
|
||||
|
||||
foreach (var location in Consts.Locations)
|
||||
{
|
||||
var httpRequest = new HttpRequestMessage(HttpMethod.Get, $"/conveyancing+{location}.html");
|
||||
httpRequest.Headers.Add("User-Agent", "Solicitors API Cache");
|
||||
var httpResponse = await _client.SendAsync(httpRequest, cancellationToken);
|
||||
var html = await httpResponse.Content.ReadAsStringAsync(cancellationToken);
|
||||
var nodes = _htmlParser.ParseHtml(html);
|
||||
IHtmlNode? mainNode = null;
|
||||
foreach (var node in nodes)
|
||||
if (node.TryGetByTagName("main", out mainNode))
|
||||
break;
|
||||
|
||||
if (TryGetResultSection(mainNode, out var results))
|
||||
ParseResultsSection(results, location, builders);
|
||||
}
|
||||
|
||||
var batches = builders.Values
|
||||
.Select(builder => GetFullSolicitorDataAsync(builder, cancellationToken))
|
||||
.Batch(5);
|
||||
|
||||
var solicitors = new List<SolicitorData?>();
|
||||
foreach (var batch in batches)
|
||||
solicitors.AddRange(await Task.WhenAll(batch));
|
||||
|
||||
return solicitors
|
||||
.Where(x => x is not null)
|
||||
.Cast<SolicitorData>()
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private bool TryGetResultSection(
|
||||
IHtmlNode? mainNode,
|
||||
[NotNullWhen(true)] out IEnumerable<IHtmlNode>? results)
|
||||
{
|
||||
results = null;
|
||||
|
||||
// going one level at a time helps a little with performance here, at the cost of messier code
|
||||
return mainNode is not null
|
||||
&& mainNode.TryGetByTagNameAndClass("div", "content-holder", out var contentHolderNode)
|
||||
&& contentHolderNode.TryGetByTagNameAndClass("div", "container", out var containerNode)
|
||||
&& containerNode.TryGetByTagNameAndClass("div", "content", out var contentNode)
|
||||
&& contentNode.TryGetByTagNameAndClass("div", "result-section", out var resultSectionNode)
|
||||
&& resultSectionNode.TryGetChildren(out results);
|
||||
}
|
||||
|
||||
private void ParseResultsSection(
|
||||
IEnumerable<IHtmlNode> results,
|
||||
string location,
|
||||
Dictionary<string, SolicitorBuilder> builders)
|
||||
{
|
||||
foreach (var result in results)
|
||||
{
|
||||
if (result.TryGetByClass("result-item", out var resultItem, true)
|
||||
&& TryParseToNewBuilder(resultItem, location, out var builder))
|
||||
{
|
||||
if (!builders.TryAdd(builder.Path, builder))
|
||||
builders[builder.Path].AddBaseLocation(location);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool TryParseToNewBuilder(
|
||||
IHtmlNode resultItemNode,
|
||||
string baseLocation,
|
||||
[NotNullWhen(true)] out SolicitorBuilder? builder)
|
||||
{
|
||||
builder = null;
|
||||
if (!resultItemNode.TryGetByTagNameAndClass("a", "link-map", out var linkMapNode)
|
||||
|| !linkMapNode.TryGetAttributeValue("href", out var uniqueUrl)
|
||||
|| !resultItemNode.TryGetByTagNameAndClass("span", "h2", out var titleNode)
|
||||
|| !titleNode.TryGetChildren(out var titleChildren))
|
||||
return false;
|
||||
|
||||
string? name = null;
|
||||
foreach (var child in titleChildren)
|
||||
if (child.TryGetText(out name))
|
||||
break;
|
||||
|
||||
if (name is null)
|
||||
return false;
|
||||
|
||||
string? shortDesc = null;
|
||||
if (resultItemNode.TryGetByTagName("p", out var pNode)
|
||||
&& pNode.TryGetChildren(out var pChildren))
|
||||
foreach (var child in pChildren)
|
||||
if (child.TryGetText(out shortDesc))
|
||||
break;
|
||||
|
||||
builder = new SolicitorBuilder(name, uniqueUrl, shortDesc, baseLocation);
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task<SolicitorData?> GetFullSolicitorDataAsync(
|
||||
SolicitorBuilder builder,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
|
||||
var httpRequest = new HttpRequestMessage(HttpMethod.Get, builder.Path);
|
||||
httpRequest.Headers.Add("User-Agent", "Solicitors API Cache");
|
||||
var httpResponse = await _client.SendAsync(httpRequest, cancellationToken);
|
||||
var html = await httpResponse.Content.ReadAsStringAsync(cancellationToken);
|
||||
var nodes = _htmlParser.ParseHtml(html);
|
||||
IHtmlNode? mainNode = null;
|
||||
foreach (var node in nodes)
|
||||
if (node.TryGetByTagName("main", out mainNode))
|
||||
break;
|
||||
|
||||
if (mainNode is null || !mainNode.TryGetByTagNameAndClass("div", "content-block", out var contentBlockNode))
|
||||
return null;
|
||||
|
||||
Location[] offices = [];
|
||||
if (contentBlockNode.TryGetByTagNameAndClass("div", "office-item", out var officeItemNode)
|
||||
&& officeItemNode.TryGetChildren(out var officeItems))
|
||||
{
|
||||
offices = ParseOffices(officeItems);
|
||||
}
|
||||
|
||||
Rating[] ratings = [];
|
||||
string? phone = null;
|
||||
string? email = null;
|
||||
string? website = null;
|
||||
if (mainNode.TryGetByTagNameAndClass("div", "side-holder", out var sidebarNode)
|
||||
&& sidebarNode.TryGetChildren(out var sidebarChildren))
|
||||
{
|
||||
ParseSidebar(sidebarChildren, out phone, out email, out website, out ratings);
|
||||
}
|
||||
|
||||
return new SolicitorData()
|
||||
{
|
||||
Name = builder.Name,
|
||||
UrlPath = builder.Path,
|
||||
ShortDescription = builder.ShortDescription,
|
||||
Phone = phone,
|
||||
Email = email,
|
||||
Website = website,
|
||||
Ratings = ratings,
|
||||
Offices = offices,
|
||||
Cities = builder.BaseLocations
|
||||
};
|
||||
}
|
||||
|
||||
Location[] ParseOffices(IEnumerable<IHtmlNode> officeNodes)
|
||||
=> officeNodes
|
||||
.Select(ParseOffice)
|
||||
.Where(x => x is not null)
|
||||
.Cast<Location>()
|
||||
.ToArray();
|
||||
|
||||
Location? ParseOffice(IHtmlNode officeNode)
|
||||
{
|
||||
if (!officeNode.TryGetChildren(out var officeChildren)
|
||||
|| !officeNode.TryGetByTagName("address", out var addressNode)
|
||||
|| !addressNode.TryGetChildren(out var addressLines))
|
||||
return null;
|
||||
|
||||
Location? location = null;
|
||||
string? address = "";
|
||||
foreach (var line in addressLines)
|
||||
{
|
||||
if (line.TryGetText(out var lineText))
|
||||
address += lineText;
|
||||
else if (line.TryGetByTagName("br", out _, true))
|
||||
address += "\n";
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(address))
|
||||
address = null;
|
||||
|
||||
string? phone = null;
|
||||
foreach (var child in officeChildren)
|
||||
{
|
||||
if (child.TryGetByTagName("a", out var anchor)
|
||||
&& anchor.TryGetAttributeValue("href", out var telString)
|
||||
&& telString.StartsWith("\"tel:")
|
||||
&& telString.Length > 5)
|
||||
{
|
||||
phone = telString.Substring(5, telString.Length - 6);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (address is not null && phone is not null)
|
||||
location = new Location(address, phone, []);
|
||||
|
||||
return location;
|
||||
}
|
||||
|
||||
void ParseSidebar(
|
||||
IEnumerable<IHtmlNode> sidebarChildren,
|
||||
out string? phone,
|
||||
out string? email,
|
||||
out string? website,
|
||||
out Rating[] ratings)
|
||||
{
|
||||
List<Rating> ratingsList = [];
|
||||
phone = null;
|
||||
website = null;
|
||||
email = null;
|
||||
|
||||
foreach (var node in sidebarChildren)
|
||||
{
|
||||
var isRevCount = node.TryGetByTagNameAndClass("div", "rev-count", out _, true);
|
||||
var isLinksHolder = node.TryGetByTagNameAndClass("div", "links-holder", out _, true);
|
||||
if ((!isRevCount && !isLinksHolder)
|
||||
|| !node.TryGetChildren(out var children))
|
||||
continue;
|
||||
|
||||
if (isLinksHolder)
|
||||
{
|
||||
if (node.TryGetByTagNameAndClass("a", "website", out var websiteNode)
|
||||
&& websiteNode.TryGetAttributeValue("href", out website))
|
||||
website = website.Trim('"');
|
||||
|
||||
if (node.TryGetByTagNameAndClass("a", "phone", out var phoneNode)
|
||||
&& phoneNode.TryGetAttributeValue("href", out var phoneStr)
|
||||
&& phoneStr.StartsWith("\"tel:")
|
||||
&& phoneStr.Length > 5)
|
||||
{
|
||||
phone = phoneStr.Substring(5, phoneStr.Length - 6);
|
||||
}
|
||||
}
|
||||
|
||||
var childrenArray = children as IHtmlNode[] ?? children.ToArray();
|
||||
foreach (var child in childrenArray.Where(x => x.TryGetByClass("rev-box", out _, true)))
|
||||
{
|
||||
if (child.TryGetAttributeValue("title", out var ratingString)
|
||||
&& TryParseRatingString(ratingString, out decimal rating, out decimal maxRating)
|
||||
&& child.TryGetByTagNameAndClass("img", "rev-logo", out var imgNode)
|
||||
&& imgNode.TryGetAttributeValue("src", out var imgSrc)
|
||||
&& TryParseRatingSrc(imgSrc, out var ratingProvider))
|
||||
{
|
||||
ratingsList.Add(new Rating(rating, maxRating, ratingProvider, imgSrc));
|
||||
}
|
||||
}
|
||||
|
||||
if (ratingsList.Count == 0)
|
||||
{
|
||||
foreach (var child in childrenArray.Where(x => !x.HasAttribute("class")))
|
||||
{
|
||||
if (child.TryGetChildren(out var grandChildren))
|
||||
foreach (var grandChild in grandChildren)
|
||||
if (grandChild.TryGetText(out var textContent)
|
||||
&& TryParseOwnRatingString(textContent, out var rating))
|
||||
ratingsList.Add(new Rating(rating, Consts.MaxRating, Consts.Name, Consts.IconPath));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ratings = ratingsList.ToArray();
|
||||
}
|
||||
|
||||
|
||||
bool TryParseRatingString(string ratingString, out decimal rating, out decimal maxRating)
|
||||
{
|
||||
rating = 0;
|
||||
maxRating = 0;
|
||||
var parts = ratingString.Trim('"').Split(" / ");
|
||||
if (parts.Length != 2)
|
||||
return false;
|
||||
|
||||
return decimal.TryParse(parts[0], out rating) && decimal.TryParse(parts[1], out maxRating);
|
||||
}
|
||||
|
||||
bool TryParseRatingSrc(string imgSrc, out string provider)
|
||||
{
|
||||
provider = "";
|
||||
imgSrc = imgSrc.Trim('"');
|
||||
var prefix = "/images/logo-";
|
||||
if (!imgSrc.StartsWith(prefix))
|
||||
return false;
|
||||
|
||||
var providerAndExtension = imgSrc.Substring(prefix.Length);
|
||||
provider = providerAndExtension.Split('.').First();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool TryParseOwnRatingString(string ratingString, out decimal rating)
|
||||
{
|
||||
rating = 0;
|
||||
var prefix = "Average review score : ";
|
||||
if (!ratingString.StartsWith(prefix))
|
||||
return false;
|
||||
|
||||
var ratingPart = ratingString.Substring(prefix.Length);
|
||||
return decimal.TryParse(ratingPart, out rating);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user