Commit 86a693dc authored by Andreas Müller's avatar Andreas Müller
Browse files

Eigenes Basis-Paket geschnürt.

parent 989d8511
......@@ -17,6 +17,8 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AMWD.Common" Version="0.0.2" />
<PackageReference Include="AMWD.Common.AspNetCore" Version="0.0.2" />
<PackageReference Include="LumenWorksCsvReader" Version="4.0.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.9" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation" Version="3.1.9" />
......
using System.Net;
using CovidTable.Util;
using Microsoft.AspNetCore.Antiforgery;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.HttpOverrides;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
namespace CovidTable.Extensions
{
public static class AspNetCoreExtensions
{
public static void UseProxyHosting(this IApplicationBuilder app, IConfiguration configuration)
{
string path = configuration.GetValue<string>("ASPNETCORE_APPL_PATH");
if (!string.IsNullOrWhiteSpace(path))
app.UsePathBase(new PathString(path));
var options = new ForwardedHeadersOptions { ForwardedHeaders = ForwardedHeaders.All };
options.KnownNetworks.Clear();
options.KnownNetworks.Add(new IPNetwork(IPAddress.Parse("10.0.0.0"), 8));
options.KnownNetworks.Add(new IPNetwork(IPAddress.Parse("172.16.0.0"), 12));
options.KnownNetworks.Add(new IPNetwork(IPAddress.Parse("192.168.0.0"), 16));
app.UseForwardedHeaders(options);
}
public static (string Name, string Value) GetAntiforgeryToken(this HttpContext httpContext)
{
var af = httpContext.RequestServices.GetService<IAntiforgery>();
var set = af?.GetAndStoreTokens(httpContext);
return (Name: set?.FormFieldName, Value: set?.RequestToken);
}
public static IServiceCollection AddSingletonHostedService<TService>(this IServiceCollection services)
where TService : class, IHostedService
{
services.AddSingleton<TService>();
services.AddSingleton<IHostedService, BackgroundServiceStarter<TService>>();
return services;
}
}
}
using System;
using System.Text;
namespace CovidTable.Extensions
{
public static class DateTimeExtensions
{
public static DateTime AsUtc(this DateTime dt)
{
return dt.Kind switch
{
DateTimeKind.Local => dt.ToUniversalTime(),
DateTimeKind.Utc => dt,
_ => DateTime.SpecifyKind(dt, DateTimeKind.Utc),
};
}
public static DateTime AsLocal(this DateTime dt)
{
return dt.Kind switch
{
DateTimeKind.Local => dt,
DateTimeKind.Utc => dt.ToLocalTime(),
_ => DateTime.SpecifyKind(dt, DateTimeKind.Local),
};
}
public static TimeSpan GetAlignedInterval(this TimeSpan timeSpan, TimeSpan offset = default)
{
var now = DateTime.UtcNow;
var nextTime = new DateTime(now.Ticks / timeSpan.Ticks * timeSpan.Ticks) + offset;
if (nextTime <= now)
nextTime += timeSpan;
return nextTime - now;
}
public static string ToShortString(this TimeSpan timeSpan, bool withMilliseconds = false)
{
var sb = new StringBuilder();
if (timeSpan.TotalDays >= 1)
sb.Append(timeSpan.Days).Append("d ");
if (timeSpan.TotalHours >= 1)
sb.Append(timeSpan.Hours).Append("h ");
if (timeSpan.TotalMinutes >= 1)
sb.Append(timeSpan.Minutes).Append("m ");
sb.Append(timeSpan.Seconds).Append("s ");
if (withMilliseconds)
sb.Append(timeSpan.Milliseconds).Append("ms");
return sb.ToString().Trim();
}
}
}
......@@ -2,8 +2,8 @@
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AMWD.Common.Extensions;
using CovidTable.Database;
using CovidTable.Extensions;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
......
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using CovidTable.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
......
......@@ -3,8 +3,8 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using AMWD.Common.Extensions;
using CovidTable.Database;
using CovidTable.Extensions;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
......
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using AMWD.Common.Extensions;
using CovidTable.Database;
using CovidTable.Extensions;
using LumenWorks.Framework.IO.Csv;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
......@@ -74,7 +73,7 @@ namespace CovidTable.Services
var startOffset = TimeSpan.FromHours(4);
var offset = startOffset.Subtract(DateTimeOffset.Now.Offset);
var sleepInterval = TimeSpan.FromDays(1).GetAlignedInterval(offset);
logger.LogInformation($"C: Data up to date, sleeping for {sleepInterval.ToShortString()} ({DateTime.Now.Add(sleepInterval):yyyy-MM-dd HH:mm})");
await Task.Delay(sleepInterval, cancellationToken);
continue;
......@@ -83,142 +82,140 @@ namespace CovidTable.Services
logger.LogInformation("C: Starting data update");
try
{
using (var client = new HttpClient())
using var client = new HttpClient();
using var response = await client.GetAsync(CasesUrl, cancellationToken);
if (response.IsSuccessStatusCode)
{
using var response = await client.GetAsync(CasesUrl, cancellationToken);
if (response.IsSuccessStatusCode)
{
string json = await response.Content.ReadAsStringAsync();
var dataset = JsonConvert.DeserializeObject<DataSet>(json);
string json = await response.Content.ReadAsStringAsync();
var dataset = JsonConvert.DeserializeObject<DataSet>(json);
using var scope = serviceScopeFactory.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService<ServerDbContext>();
using var scope = serviceScopeFactory.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService<ServerDbContext>();
using var transaction = await dbContext.Database.BeginTransactionAsync(cancellationToken);
try
using var transaction = await dbContext.Database.BeginTransactionAsync(cancellationToken);
try
{
DateTime newDate = DateTime.MinValue;
foreach (var entry in dataset.Features.Select(f => f.Attributes))
{
DateTime newDate = DateTime.MinValue;
foreach (var entry in dataset.Features.Select(f => f.Attributes))
{
#region Base data
#region Base data
int bundeslandId = int.Parse(entry.BundeslandId);
var bundesland = await dbContext.Bundesland
.Where(b => b.Id == bundeslandId)
.FirstOrDefaultAsync(cancellationToken);
if (bundesland == null)
int bundeslandId = int.Parse(entry.BundeslandId);
var bundesland = await dbContext.Bundesland
.Where(b => b.Id == bundeslandId)
.FirstOrDefaultAsync(cancellationToken);
if (bundesland == null)
{
bundesland = new Bundesland
{
bundesland = new Bundesland
{
Id = bundeslandId,
Name = entry.BundeslandName
};
await dbContext.Bundesland.AddAsync(bundesland, cancellationToken);
}
bundesland.Einwohner = entry.BundeslandEinwohner;
Id = bundeslandId,
Name = entry.BundeslandName
};
await dbContext.Bundesland.AddAsync(bundesland, cancellationToken);
}
bundesland.Einwohner = entry.BundeslandEinwohner;
var kommune = await dbContext.Kommune
.Where(k => k.Id == entry.KommuneId)
.FirstOrDefaultAsync(cancellationToken);
if (kommune == null)
var kommune = await dbContext.Kommune
.Where(k => k.Id == entry.KommuneId)
.FirstOrDefaultAsync(cancellationToken);
if (kommune == null)
{
kommune = new Kommune
{
kommune = new Kommune
{
Id = entry.KommuneId,
Name = entry.KommuneName,
Type = entry.KommuneType
};
await dbContext.Kommune.AddAsync(kommune, cancellationToken);
}
kommune.Bundesland = bundesland;
kommune.Einwohner = entry.KommuneEinwohner;
await dbContext.SaveChangesAsync(cancellationToken);
Id = entry.KommuneId,
Name = entry.KommuneName,
Type = entry.KommuneType
};
await dbContext.Kommune.AddAsync(kommune, cancellationToken);
}
kommune.Bundesland = bundesland;
kommune.Einwohner = entry.KommuneEinwohner;
await dbContext.SaveChangesAsync(cancellationToken);
#endregion Base data
#endregion Base data
#region History
#region History
var dt = DateTime.ParseExact(entry.Timestamp, "dd.MM.yyyy, HH:mm 'Uhr'", CultureInfo.InvariantCulture).AsLocal();
var bundeslandHistory = await dbContext.BundeslandHistory
.Where(h => h.BundeslandId == bundesland.Id)
.Where(h => h.Timestamp == dt)
.FirstOrDefaultAsync(cancellationToken);
if (bundeslandHistory == null)
{
bundeslandHistory = new BundeslandHistorie
{
Bundesland = bundesland,
Timestamp = dt
};
await dbContext.BundeslandHistory.AddAsync(bundeslandHistory, cancellationToken);
}
bundeslandHistory.Inzidenz = entry.BundeslandInzidenz;
var kommuneHistory = await dbContext.KommuneHistory
.Where(h => h.KommuneId == kommune.Id)
.Where(h => h.Timestamp == dt)
.FirstOrDefaultAsync(cancellationToken);
if (kommuneHistory == null)
var dt = DateTime.ParseExact(entry.Timestamp, "dd.MM.yyyy, HH:mm 'Uhr'", CultureInfo.InvariantCulture).AsLocal();
var bundeslandHistory = await dbContext.BundeslandHistory
.Where(h => h.BundeslandId == bundesland.Id)
.Where(h => h.Timestamp == dt)
.FirstOrDefaultAsync(cancellationToken);
if (bundeslandHistory == null)
{
bundeslandHistory = new BundeslandHistorie
{
kommuneHistory = new KommuneHistorie
{
Kommune = kommune,
Timestamp = dt
};
await dbContext.KommuneHistory.AddAsync(kommuneHistory, cancellationToken);
newDate = dt;
}
kommuneHistory.Erkrankte = entry.KommuneErkrankte;
kommuneHistory.Tote = entry.KommuneTote;
kommuneHistory.Inzidenz = entry.KommuneInzidenz;
await dbContext.SaveChangesAsync(cancellationToken);
#endregion History
Bundesland = bundesland,
Timestamp = dt
};
await dbContext.BundeslandHistory.AddAsync(bundeslandHistory, cancellationToken);
}
bundeslandHistory.Inzidenz = entry.BundeslandInzidenz;
// Calculate denormalized data
var latest = dbContext.BundeslandHistory
.OrderByDescending(h => h.Timestamp)
.FirstOrDefault()?.Timestamp ?? DateTime.MinValue;
if (latest != DateTime.MinValue)
var kommuneHistory = await dbContext.KommuneHistory
.Where(h => h.KommuneId == kommune.Id)
.Where(h => h.Timestamp == dt)
.FirstOrDefaultAsync(cancellationToken);
if (kommuneHistory == null)
{
var blHistory = await dbContext.BundeslandHistory
.Where(h => h.Timestamp == latest)
.ToListAsync();
foreach (var bh in blHistory)
kommuneHistory = new KommuneHistorie
{
bh.Erkrankte = await dbContext.KommuneHistory
.Where(kh => kh.Timestamp == bh.Timestamp)
.Where(kh => kh.Kommune.BundeslandId == bh.BundeslandId)
.Select(kh => kh.Erkrankte)
.SumAsync();
bh.Tote = await dbContext.KommuneHistory
.Where(kh => kh.Timestamp == bh.Timestamp)
.Where(kh => kh.Kommune.BundeslandId == bh.BundeslandId)
.Select(kh => kh.Tote)
.SumAsync();
}
await dbContext.SaveChangesAsync();
Kommune = kommune,
Timestamp = dt
};
await dbContext.KommuneHistory.AddAsync(kommuneHistory, cancellationToken);
newDate = dt;
}
kommuneHistory.Erkrankte = entry.KommuneErkrankte;
kommuneHistory.Tote = entry.KommuneTote;
kommuneHistory.Inzidenz = entry.KommuneInzidenz;
await dbContext.SaveChangesAsync(cancellationToken);
await transaction.CommitAsync(cancellationToken);
logger.LogInformation("C: Data update finished");
#endregion History
}
if (newDate != DateTime.MinValue)
// Calculate denormalized data
var latest = dbContext.BundeslandHistory
.OrderByDescending(h => h.Timestamp)
.FirstOrDefault()?.Timestamp ?? DateTime.MinValue;
if (latest != DateTime.MinValue)
{
var blHistory = await dbContext.BundeslandHistory
.Where(h => h.Timestamp == latest)
.ToListAsync();
foreach (var bh in blHistory)
{
logger.LogDebug($"C: Latest cases from {newDate:yyyy-MM-dd'T'HH:mm:ssK}");
var push = scope.ServiceProvider.GetService<PushService>();
push?.Enqueue($"Daten wurden aktualisiert.\nStand: {newDate:dd.MM.yyyy HH:mm}");
continue;
bh.Erkrankte = await dbContext.KommuneHistory
.Where(kh => kh.Timestamp == bh.Timestamp)
.Where(kh => kh.Kommune.BundeslandId == bh.BundeslandId)
.Select(kh => kh.Erkrankte)
.SumAsync();
bh.Tote = await dbContext.KommuneHistory
.Where(kh => kh.Timestamp == bh.Timestamp)
.Where(kh => kh.Kommune.BundeslandId == bh.BundeslandId)
.Select(kh => kh.Tote)
.SumAsync();
}
await dbContext.SaveChangesAsync();
}
catch
await transaction.CommitAsync(cancellationToken);
logger.LogInformation("C: Data update finished");
if (newDate != DateTime.MinValue)
{
await transaction.RollbackAsync(cancellationToken);
throw;
logger.LogDebug($"C: Latest cases from {newDate:yyyy-MM-dd'T'HH:mm:ssK}");
var push = scope.ServiceProvider.GetService<PushService>();
push?.Enqueue($"Daten wurden aktualisiert.\nStand: {newDate:dd.MM.yyyy HH:mm}");
continue;
}
}
catch
{
await transaction.RollbackAsync(cancellationToken);
throw;
}
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
......@@ -339,7 +336,6 @@ namespace CovidTable.Services
logger.LogDebug($"R: Latest data from {newDate:yyyy-MM-dd'T'HH:mm:ssK}");
continue;
}
}
catch
{
......
......@@ -2,8 +2,8 @@ using System;
using System.IO;
using System.Linq;
using System.Reflection;
using AMWD.Common.AspNetCore.Extensions;
using CovidTable.Database;
using CovidTable.Extensions;
using CovidTable.Services;
using Microsoft.AspNetCore.Antiforgery;
using Microsoft.AspNetCore.Builder;
......@@ -104,7 +104,7 @@ namespace CovidTable
{
bool isDev = configuration.GetValue<string>("ASPNETCORE_ENVIRONMENT") == "Development";
app.UseProxyHosting(configuration);
app.UseProxyHosting();
app.UseResponseCompression();
if (!initService.IsSuccess)
......
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
namespace CovidTable.Util
{
public class BackgroundServiceStarter<TService> : IHostedService
where TService : class, IHostedService
{
private readonly TService service;
/// <summary>
/// Initializes an new instance of the <see cref="BackgroundServiceStarter{TService}"/> class.
/// </summary>
/// <param name="backgroundService">The service to work in background.</param>
public BackgroundServiceStarter(TService backgroundService)
{
service = backgroundService;
}
/// <summary>
/// Starts the service.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public Task StartAsync(CancellationToken cancellationToken)
{
return service.StartAsync(cancellationToken);
}
/// <summary>
/// Stops the service.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public Task StopAsync(CancellationToken cancellationToken)
{
return service.StopAsync(cancellationToken);
}
}
}
@using CovidTable
@using CovidTable.Extensions
@using AMWD.Common.AspNetCore.Extensions
@using CovidTable
@using CovidTable.Models
@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
<clear /> <!-- ensure only the sources defined below are used -->
<add key="NuGet.org" value="https://api.nuget.org/v3/index.json" />
<add key="AM.WD" value="https://nuget.am-wd.de/v3/index.json" />
</packageSources>
</configuration>
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment