Fix Indeed salary data not being stored

jamerst · Apr 15, 2023 · 7c12ec3 · 7c12ec3
1 parent 998ae56
commit 7c12ec3
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 0 deletions.
diff --git a/jobhunt/Controllers/RefreshController.cs b/jobhunt/Controllers/RefreshController.cs
@@ -4,6 +4,8 @@
 using JobHunt.PageWatcher;
 using JobHunt.Searching.Indeed;
 using JobHunt.Workers;
+using JobHunt.Searching.Indeed.GraphQL;
+using JobHunt.Searching;
 
 namespace JobHunt.Controllers;
 [ApiController]
@@ -16,6 +18,7 @@ public class RefreshController : ControllerBase
     private readonly IPageScreenshotWorker _screenshotWorker;
     private readonly IJobService _jobService;
     private readonly IWatchedPageService _wpService;
+
     public RefreshController(
         IIndeedApiSearchProvider indeed,
         IPageWatcher pageWatcher,
@@ -38,6 +41,46 @@ public async Task Indeed(CancellationToken token)
         await _indeed.SearchAllAsync(token);
     }
 
+    [HttpGet]
+    public async Task<IActionResult> GetMissingSalaries(string country, [FromServices] IIndeedGraphQLService graphQL)
+    {
+        int updated = 0;
+
+        DateTimeOffset start = new DateTime(2023, 02, 01, 0, 0, 0, DateTimeKind.Utc);
+        List<Job> missing = await _jobService.Set.Where(j => j.Provider == SearchProviderName.Indeed && j.Posted >= start && !j.AvgYearlySalary.HasValue).ToListAsync();
+        List<string> seen = new List<string>();
+
+        int skip = 0;
+        var batch = missing.Take(500);
+        while (batch.Any())
+        {
+            var results = await graphQL.GetJobDataAsync(batch.Select(b => b.ProviderId!), country);
+            if (results != null)
+            {
+                foreach (var result in results.JobData.Results)
+                {
+                    var job = missing.First(j => j.ProviderId == result.Job.Key);
+                    job.Salary = result.Job.Compensation?.GetFormattedText();
+                    job.AvgYearlySalary = result.Job.Compensation?.GetAvgYearlySalary();
+
+                    updated++;
+                    seen.Add(result.Job.Key);
+                }
+            }
+
+            skip += 500;
+            batch = missing.Skip(skip).Take(500);
+        }
+
+        await _jobService.SaveChangesAsync();
+
+        return new JsonResult(new
+        {
+            Updated = updated,
+            Missing = missing.Select(j => j.ProviderId!).Except(seen)
+        });
+    }
+
     [HttpGet]
     public async Task PageWatcher(CancellationToken token)
     {

diff --git a/jobhunt/Extensions/ServiceCollectionExtensions.cs b/jobhunt/Extensions/ServiceCollectionExtensions.cs
@@ -75,6 +75,7 @@ public static IServiceCollection AddIndeedApiSearchProvider(this IServiceCollect
     public static IServiceCollection AddIndeedGraphQLApi(this IServiceCollection services)
     {
         services.AddScoped<IIndeedJobFetcher, IndeedGraphQLService>();
+        services.AddScoped<IIndeedGraphQLService, IndeedGraphQLService>();
 
         return services;
     }

diff --git a/jobhunt/Searching/Indeed/IndeedApiSearchProvider.cs b/jobhunt/Searching/Indeed/IndeedApiSearchProvider.cs
@@ -170,6 +170,8 @@ async Task<bool> processResults(IEnumerable<JobResult> results)
             {
                 Title = result.Title,
                 Description = markdown,
+                Salary = result.FormattedSalary,
+                AvgYearlySalary = result.AvgYearlySalary,
                 Location = result.Location,
                 Latitude = result.Latitude,
                 Longitude = result.Longitude,

diff --git a/jobhunt/Services/CompanyService.cs b/jobhunt/Services/CompanyService.cs
@@ -79,6 +79,11 @@ await _context.WatchedPages
             await transaction.CommitAsync();
         }
 
+        if (src.Name != dest.Name)
+        {
+            dest.AlternateNames.Add(new CompanyName { Name = src.Name });
+        }
+
         if (src.Recruiter && !dest.Recruiter)
         {
             dest.Recruiter = true;