Commit 989d8511 authored by Andreas Müller's avatar Andreas Müller
Browse files

Implementierung des geschätzten R-Werts

parent 901df21e
......@@ -30,19 +30,19 @@ namespace CovidTable.Controllers
ViewData["Title"] = "Covid19 Tabelle";
ViewData["PushKey"] = pushKey;
var latest = dbContext.BundeslandHistory
var latestDate = dbContext.BundeslandHistory
.OrderByDescending(h => h.Timestamp)
.FirstOrDefault()?.Timestamp ?? DateTime.MinValue;
if (latest == DateTime.MinValue)
if (latestDate == DateTime.MinValue)
{
ViewData["LastUpdate"] = "Keine Daten";
ViewData["History"] = new JArray();
}
else
{
var oldestDate = latest.AddMonths(-1);
ViewData["LastUpdate"] = latest.ToString("yyyy-MM-dd'T'HH:mm:ss");
var oldestDate = latestDate.AddMonths(-1);
ViewData["LastUpdate"] = latestDate.ToString("yyyy-MM-dd'T'HH:mm:ss");
var blHistory = dbContext.BundeslandHistory
.Where(h => h.Timestamp >= oldestDate)
......@@ -63,6 +63,47 @@ namespace CovidTable.Controllers
ViewData["History"] = history;
}
var latestItem = dbContext.ReproductionRate
.OrderByDescending(r => r.Timestamp)
.FirstOrDefault();
if (latestItem == null)
{
ViewData["Reproduction"] = new JArray();
}
else
{
var reprod = new JArray();
reprod.Add(new JObject
{
["title"] = "4-Tages-R-Wert",
["timestamp"] = latestItem.Timestamp,
["value"] = latestItem.ReprodRaw,
["min"] = latestItem.MinReprodRaw,
["max"] = latestItem.MaxReprodRaw
});
latestItem = dbContext.ReproductionRate
.Where(r => r.Reprod7d != null)
.OrderByDescending(r => r.Timestamp)
.FirstOrDefault();
if (latestItem != null)
{
reprod.Add(new JObject
{
["title"] = "7-Tages-R-Wert",
["timestamp"] = latestItem.Timestamp,
["value"] = latestItem.Reprod7d,
["min"] = latestItem.MinReprod7d,
["max"] = latestItem.MaxReprod7d
});
}
ViewData["Reproduction"] = reprod;
}
return View();
}
......@@ -294,6 +335,33 @@ namespace CovidTable.Controllers
return Json(result);
}
[HttpPost]
[ValidateAntiForgeryToken]
public IActionResult GetReproduction()
{
var latest = dbContext.ReproductionRate
.OrderByDescending(r => r.Timestamp)
.FirstOrDefault()?.Timestamp ?? DateTime.Today;
var oldest = latest.AddMonths(-1);
var list = dbContext.ReproductionRate
.Where(r => r.Timestamp >= oldest)
.OrderBy(r => r.Timestamp)
.Select(r => new ReproductionRate
{
Timestamp = r.Timestamp,
ReprodRaw = r.ReprodRaw,
MinReprodRaw = r.MinReprodRaw,
MaxReprodRaw = r.MaxReprodRaw,
Reprod7d = r.Reprod7d,
MinReprod7d = r.MinReprod7d,
MaxReprod7d = r.MaxReprod7d
})
.ToList();
return Json(list);
}
[HttpPost]
[ValidateAntiForgeryToken]
public IActionResult SaveNotification(string endpoint, string p256dh, string auth)
......
......@@ -17,6 +17,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="LumenWorksCsvReader" Version="4.0.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.9" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation" Version="3.1.9" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="3.1.9" />
......
using System;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
namespace CovidTable.Database
{
[Table("Reproduction")]
public class ReproductionRate
{
[Key]
public DateTime Timestamp { get; set; }
public int NeuerkrankungRaw { get; set; }
public int MinNeuRaw { get; set; }
public int MaxNeuRaw { get; set; }
public int Neuerkrankungen { get; set; }
public int MinNeu { get; set; }
public int MaxNeu { get; set; }
public float ReprodRaw { get; set; }
public float MinReprodRaw { get; set; }
public float MaxReprodRaw { get; set; }
public float? Reprod7d { get; set; }
public float? MinReprod7d { get; set; }
public float? MaxReprod7d { get; set; }
}
}
......@@ -22,6 +22,8 @@ namespace CovidTable.Database
public DbSet<PushEndpoint> PushEndpoints { get; set; }
public DbSet<ReproductionRate> ReproductionRate { get; set; }
protected override void OnModelCreating(ModelBuilder builder)
{
base.OnModelCreating(builder);
......
// <auto-generated />
using System;
using CovidTable.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
namespace CovidTable.Migrations
{
[DbContext(typeof(ServerDbContext))]
[Migration("20201105213626_Add Reproduction")]
partial class AddReproduction
{
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn)
.HasAnnotation("ProductVersion", "3.1.9")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
modelBuilder.Entity("CovidTable.Database.Bundesland", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn);
b.Property<int>("Einwohner")
.HasColumnType("integer");
b.Property<string>("Name")
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("Bundeslaender");
});
modelBuilder.Entity("CovidTable.Database.BundeslandHistorie", b =>
{
b.Property<int>("BundeslandId")
.HasColumnType("integer");
b.Property<DateTime>("Timestamp")
.HasColumnType("timestamp without time zone");
b.Property<int>("Erkrankte")
.HasColumnType("integer");
b.Property<decimal>("Inzidenz")
.HasColumnType("numeric");
b.Property<int>("Tote")
.HasColumnType("integer");
b.HasKey("BundeslandId", "Timestamp");
b.ToTable("BundeslandHistorie");
});
modelBuilder.Entity("CovidTable.Database.Kommune", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("integer")
.HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn);
b.Property<int>("BundeslandId")
.HasColumnType("integer");
b.Property<int>("Einwohner")
.HasColumnType("integer");
b.Property<string>("Name")
.HasColumnType("text");
b.Property<string>("Type")
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("BundeslandId");
b.ToTable("Kommunen");
});
modelBuilder.Entity("CovidTable.Database.KommuneHistorie", b =>
{
b.Property<int>("KommuneId")
.HasColumnType("integer");
b.Property<DateTime>("Timestamp")
.HasColumnType("timestamp without time zone");
b.Property<int>("Erkrankte")
.HasColumnType("integer");
b.Property<decimal>("Inzidenz")
.HasColumnType("numeric");
b.Property<int>("Tote")
.HasColumnType("integer");
b.HasKey("KommuneId", "Timestamp");
b.ToTable("KommuneHistorie");
});
modelBuilder.Entity("CovidTable.Database.PushEndpoint", b =>
{
b.Property<string>("Url")
.HasColumnType("text");
b.Property<string>("Auth")
.HasColumnType("text");
b.Property<bool>("IsFaulted")
.HasColumnType("boolean");
b.Property<string>("P256DH")
.HasColumnType("text");
b.HasKey("Url");
b.ToTable("PushEndpoints");
});
modelBuilder.Entity("CovidTable.Database.ReproductionRate", b =>
{
b.Property<DateTime>("Timestamp")
.HasColumnType("timestamp without time zone");
b.Property<int>("MaxNeu")
.HasColumnType("integer");
b.Property<int>("MaxNeuRaw")
.HasColumnType("integer");
b.Property<float?>("MaxReprod7d")
.HasColumnType("real");
b.Property<float>("MaxReprodRaw")
.HasColumnType("real");
b.Property<int>("MinNeu")
.HasColumnType("integer");
b.Property<int>("MinNeuRaw")
.HasColumnType("integer");
b.Property<float?>("MinReprod7d")
.HasColumnType("real");
b.Property<float>("MinReprodRaw")
.HasColumnType("real");
b.Property<int>("NeuerkrankungRaw")
.HasColumnType("integer");
b.Property<int>("Neuerkrankungen")
.HasColumnType("integer");
b.Property<float?>("Reprod7d")
.HasColumnType("real");
b.Property<float>("ReprodRaw")
.HasColumnType("real");
b.HasKey("Timestamp");
b.ToTable("Reproduction");
});
modelBuilder.Entity("CovidTable.Database.BundeslandHistorie", b =>
{
b.HasOne("CovidTable.Database.Bundesland", "Bundesland")
.WithMany("History")
.HasForeignKey("BundeslandId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("CovidTable.Database.Kommune", b =>
{
b.HasOne("CovidTable.Database.Bundesland", "Bundesland")
.WithMany("Kommunen")
.HasForeignKey("BundeslandId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("CovidTable.Database.KommuneHistorie", b =>
{
b.HasOne("CovidTable.Database.Kommune", "Kommune")
.WithMany("History")
.HasForeignKey("KommuneId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
#pragma warning restore 612, 618
}
}
}
using System;
using Microsoft.EntityFrameworkCore.Migrations;
namespace CovidTable.Migrations
{
public partial class AddReproduction : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "Reproduction",
columns: table => new
{
Timestamp = table.Column<DateTime>(nullable: false),
NeuerkrankungRaw = table.Column<int>(nullable: false),
MinNeuRaw = table.Column<int>(nullable: false),
MaxNeuRaw = table.Column<int>(nullable: false),
Neuerkrankungen = table.Column<int>(nullable: false),
MinNeu = table.Column<int>(nullable: false),
MaxNeu = table.Column<int>(nullable: false),
ReprodRaw = table.Column<float>(nullable: false),
MinReprodRaw = table.Column<float>(nullable: false),
MaxReprodRaw = table.Column<float>(nullable: false),
Reprod7d = table.Column<float>(nullable: true),
MinReprod7d = table.Column<float>(nullable: true),
MaxReprod7d = table.Column<float>(nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_Reproduction", x => x.Timestamp);
});
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "Reproduction");
}
}
}
......@@ -126,6 +126,52 @@ namespace CovidTable.Migrations
b.ToTable("PushEndpoints");
});
modelBuilder.Entity("CovidTable.Database.ReproductionRate", b =>
{
b.Property<DateTime>("Timestamp")
.HasColumnType("timestamp without time zone");
b.Property<int>("MaxNeu")
.HasColumnType("integer");
b.Property<int>("MaxNeuRaw")
.HasColumnType("integer");
b.Property<float?>("MaxReprod7d")
.HasColumnType("real");
b.Property<float>("MaxReprodRaw")
.HasColumnType("real");
b.Property<int>("MinNeu")
.HasColumnType("integer");
b.Property<int>("MinNeuRaw")
.HasColumnType("integer");
b.Property<float?>("MinReprod7d")
.HasColumnType("real");
b.Property<float>("MinReprodRaw")
.HasColumnType("real");
b.Property<int>("NeuerkrankungRaw")
.HasColumnType("integer");
b.Property<int>("Neuerkrankungen")
.HasColumnType("integer");
b.Property<float?>("Reprod7d")
.HasColumnType("real");
b.Property<float>("ReprodRaw")
.HasColumnType("real");
b.HasKey("Timestamp");
b.ToTable("Reproduction");
});
modelBuilder.Entity("CovidTable.Database.BundeslandHistorie", b =>
{
b.HasOne("CovidTable.Database.Bundesland", "Bundesland")
......
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using CovidTable.Database;
using CovidTable.Extensions;
using LumenWorks.Framework.IO.Csv;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
......@@ -18,13 +21,16 @@ namespace CovidTable.Services
{
public class UpdateService : IHostedService
{
private const string Url = "https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_Landkreisdaten/FeatureServer/0/query?where=1%3D1&outFields=OBJECTID,GEN,BEZ,EWZ,cases,deaths,BL,BL_ID,last_update,cases7_per_100k,EWZ_BL,cases7_bl_per_100k&outSR=4326&f=json";
private const string CasesUrl = "https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_Landkreisdaten/FeatureServer/0/query?where=1%3D1&outFields=OBJECTID,GEN,BEZ,EWZ,cases,deaths,BL,BL_ID,last_update,cases7_per_100k,EWZ_BL,cases7_bl_per_100k&outSR=4326&f=json";
private const string RUrl = "https://www.rki.de/DE/Content/InfAZ/N/Neuartiges_Coronavirus/Projekte_RKI/Nowcasting_Zahlen_csv.csv?__blob=publicationFile";
private readonly ILogger logger;
private readonly IServiceScopeFactory serviceScopeFactory;
private readonly CultureInfo de = new CultureInfo("de-DE");
private CancellationTokenSource cts;
private Task updateTask = Task.CompletedTask;
private Task updateCasesTask = Task.CompletedTask;
private Task updateRTask = Task.CompletedTask;
public UpdateService(ILogger<UpdateService> logger, IServiceScopeFactory serviceScopeFactory)
{
......@@ -35,7 +41,8 @@ namespace CovidTable.Services
public Task StartAsync(CancellationToken cancellationToken)
{
cts = new CancellationTokenSource();
updateTask = Task.Run(() => RunUpdate(cts.Token));
updateCasesTask = Task.Run(() => RunCases(cts.Token));
updateRTask = Task.Run(() => RunR(cts.Token));
return Task.CompletedTask;
}
......@@ -43,10 +50,10 @@ namespace CovidTable.Services
public async Task StopAsync(CancellationToken cancellationToken)
{
cts.Cancel();
await Task.WhenAny(updateTask, Task.Delay(Timeout.Infinite, cancellationToken));
await Task.WhenAny(Task.WhenAll(updateCasesTask, updateRTask), Task.Delay(Timeout.Infinite, cancellationToken));
}
private async Task RunUpdate(CancellationToken cancellationToken)
private async Task RunCases(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
......@@ -63,22 +70,22 @@ namespace CovidTable.Services
if (dbDate == DateTime.Today)
{
// first try every night at 03:00
var startOffset = TimeSpan.FromHours(3);
// first try every day at 04:00
var startOffset = TimeSpan.FromHours(4);
var offset = startOffset.Subtract(DateTimeOffset.Now.Offset);
var sleepInterval = TimeSpan.FromDays(1).GetAlignedInterval(offset);
logger.LogInformation($"Data up to date, sleeping for {sleepInterval.ToShortString()} ({DateTime.Now.Add(sleepInterval):yyyy-MM-dd HH:mm})");
logger.LogInformation($"C: Data up to date, sleeping for {sleepInterval.ToShortString()} ({DateTime.Now.Add(sleepInterval):yyyy-MM-dd HH:mm})");
await Task.Delay(sleepInterval, cancellationToken);
continue;
}
logger.LogInformation("Starting data update");
logger.LogInformation("C: Starting data update");
try
{
using (var client = new HttpClient())
{
using var response = await client.GetAsync(Url, cancellationToken);
using var response = await client.GetAsync(CasesUrl, cancellationToken);
if (response.IsSuccessStatusCode)
{
string json = await response.Content.ReadAsStringAsync();
......@@ -87,7 +94,7 @@ namespace CovidTable.Services
using var scope = serviceScopeFactory.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService<ServerDbContext>();
using var transaction = dbContext.Database.BeginTransaction();
using var transaction = await dbContext.Database.BeginTransactionAsync(cancellationToken);
try
{
DateTime newDate = DateTime.MinValue;
......@@ -195,11 +202,11 @@ namespace CovidTable.Services
}
await transaction.CommitAsync(cancellationToken);
logger.LogInformation("Data update finished");
logger.LogInformation("C: Data update finished");
if (newDate != DateTime.MinValue)
{
logger.LogDebug($"Latest data from {newDate:yyyy-MM-dd'T'HH:mm:ssK}");
logger.LogDebug($"C: Latest cases from {newDate:yyyy-MM-dd'T'HH:mm:ssK}");
var push = scope.ServiceProvider.GetService<PushService>();
push?.Enqueue($"Daten wurden aktualisiert.\nStand: {newDate:dd.MM.yyyy HH:mm}");
......@@ -220,11 +227,11 @@ namespace CovidTable.Services
}
catch (Exception ex)
{
logger.LogError(ex, $"Data update failed: {ex.InnerException?.Message ?? ex.Message}");
logger.LogError(ex, $"C: Data update failed: {ex.InnerException?.Message ?? ex.Message}");
}
var waitInterval = TimeSpan.FromMinutes(30).GetAlignedInterval();
logger.LogInformation($"Waiting for {waitInterval.ToShortString()} before next try ({DateTime.Now.Add(waitInterval):yyyy-MM-dd HH:mm})");
logger.LogInformation($"C: Waiting for {waitInterval.ToShortString()} before next try ({DateTime.Now.Add(waitInterval):yyyy-MM-dd HH:mm})");
await Task.Delay(waitInterval, cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
......@@ -233,7 +240,135 @@ namespace CovidTable.Services
}
catch (Exception ex)
{
logger.LogCritical(ex, $"Update crashed unexpected: {ex.InnerException?.Message ?? ex.Message}");
logger.LogCritical(ex, $"C: Update crashed unexpected: {ex.InnerException?.Message ?? ex.Message}");
}
}
}
private async Task RunR(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
DateTime? dbDate = null;
using (var scope = serviceScopeFactory.CreateScope())
{
var dbContext = scope.ServiceProvider.GetRequiredService<ServerDbContext>();
dbDate = dbContext.ReproductionRate
.OrderByDescending(h => h.Timestamp)
.FirstOrDefault()?.Timestamp.AsLocal();