问题
I have a problem concerning a complex JSON deserialization in c# using newtonsoft.json
1- my feed provider send me a gz file push through POST request each 3 seconds (size between 200ko and 3000 ko)
2- I get it back, unzip it and download the JSON file inside on my server
3- I need to treat this JSON file directly after receiving it (so in realtime). What I do is using a "File System watcher" class included into a timed background task (using "TimedHostedService" class)
When a json file is created into "tmp" folder, the file system watcher detect the change and I can apply a deserialization of this json file just after.
The result until now is a deserialization which works for a few moments, stopped for few moments, and working for few moments, etc. ... Sometimes it put the data in database, sometimes not => there's a problem of latency (after some debug, I isolated the problem and it seems to be a memory issue or maybe needs more time to treat than 3 seconds push interval. there's a latency when apply this line of code : Goalserve objects = serializer.Deserialize(reader);)
My app is a .net core 2.0 app. this is all parts of the code:
Startup.cs
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.DependencyInjection;
using System;
using System.IO;
using System.Net;
using static HelloWorld.Controllers.ValuesController;
namespace HelloWorld
{
public class Startup
{
// This method gets called by the runtime. Use this method to add services to the container.
// For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940
public void ConfigureServices(IServiceCollection services)
{
services.AddMvc();
services.AddHostedService<TimedHostedService>();
/*
services.Configure<IISServerOptions>(options =>
{
options.AutomaticAuthentication = false;
});
*/
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IHostingEnvironment env)
{
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
app.UseMvc();
app.Run(async (context) =>
{
await context.Response.WriteAsync("DEV Testing environment");
});
}
}
}
Goalserve class model
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading.Tasks;
namespace HelloWorld.Models
{
public partial class Goalserve
{
[JsonProperty("updated")]
public string Updated { get; set; }
[JsonProperty("updated_ts")]
public long UpdatedTs { get; set; }
[JsonProperty("events")]
public Dictionary<string, MatchIdDetails> Events { get; set; }
}
public partial class MatchIdDetails
{
[JsonProperty("core")]
public Core Core { get; set; }
[JsonProperty("info")]
public InfoClass Info { get; set; }
[JsonProperty("stats")]
public Dictionary<string, Stat> Stats { get; set; }
[JsonProperty("odds")]
public Dictionary<string, Odd> Odds { get; set; }
}
public partial class Core
{
[JsonProperty("safe")]
public long Safe { get; set; }
[JsonProperty("stopped")]
public long Stopped { get; set; }
[JsonProperty("blocked")]
public long Blocked { get; set; }
[JsonProperty("finished")]
public long Finished { get; set; }
[JsonProperty("updated")]
public DateTimeOffset Updated { get; set; }
[JsonProperty("updated_ts")]
public long UpdatedTs { get; set; }
}
public partial class InfoClass
{
[JsonProperty("id")]
public long Id { get; set; }
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("sport")]
public string Sport { get; set; }
[JsonProperty("league")]
public string League { get; set; }
[JsonProperty("start_time")]
public string StartTime { get; set; }
[JsonProperty("start_date")]
public string StartDate { get; set; }
[JsonProperty("start_ts")]
public long StartTs { get; set; }
[JsonProperty("period")]
public string Period { get; set; }
[JsonConverter(typeof(ParseStringConverter))]
[JsonProperty("minute")]
public long Minute { get; set; }
[JsonProperty("secunds")]
public string Secunds { get; set; }
[JsonProperty("score")]
public string Score { get; set; }
[JsonProperty("points")]
public string Points { get; set; }
[JsonProperty("pitch")]
public string Pitch { get; set; }
[JsonProperty("ball_pos")]
public string BallPos { get; set; }
[JsonProperty("add_time")]
public string AddTime { get; set; }
[JsonProperty("player")]
public string Player { get; set; }
[JsonConverter(typeof(ParseStringConverter))]
[JsonProperty("state")]
public long State { get; set; }
}
public partial class Odd
{
[JsonProperty("id")]
public long Id { get; set; }
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("short_name")]
public string ShortName { get; set; }
[JsonProperty("suspend")]
public long Suspend { get; set; }
[JsonProperty("order")]
public long Order { get; set; }
[JsonProperty("info")]
[JsonConverter(typeof(InfoEnumConverter))]
public InfoEnum Info { get; set; }
[JsonProperty("participants")]
public Dictionary<string, Participant> Participants { get; set; }
}
public partial class Participant
{
[JsonProperty("id")]
public long Id { get; set; }
[JsonProperty("order")]
public long Order { get; set; }
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("short_name")]
public string ShortName { get; set; }
[JsonProperty("value_eu")]
public string ValueEu { get; set; }
[JsonProperty("value_na")]
public string ValueNa { get; set; }
[JsonProperty("value_us")]
public string ValueUs { get; set; }
[JsonProperty("handicap")]
public string Handicap { get; set; }
[JsonProperty("suspend")]
public long Suspend { get; set; }
}
public partial class Stat
{
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("home")]
public string Home { get; set; }
[JsonProperty("away")]
public string Away { get; set; }
}
public enum InfoEnum { Count070007959, CurrentCorners11, Empty, OtherType };
internal static class Converter
{
public static readonly JsonSerializerSettings Settings = new JsonSerializerSettings
{
MetadataPropertyHandling = MetadataPropertyHandling.Ignore,
DateParseHandling = DateParseHandling.None,
ContractResolver = new CamelCasePropertyNamesContractResolver(),
Converters =
{
InfoEnumConverter.Singleton,
ParseStringConverter.Singleton,
new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal }
},
};
}
public class ParseStringConverter : JsonConverter
{
public override bool CanConvert(Type t) => t == typeof(long) || t == typeof(long?);
public override object ReadJson(JsonReader reader, Type t, object existingValue, JsonSerializer serializer)
{
if (reader.TokenType == JsonToken.Null) return null;
var value = serializer.Deserialize<string>(reader);
long l;
if (Int64.TryParse(value, out l))
{
return l;
}
throw new Exception("Cannot unmarshal type long");
}
public override void WriteJson(JsonWriter writer, object untypedValue, JsonSerializer serializer)
{
if (untypedValue == null)
{
serializer.Serialize(writer, null);
return;
}
var value = (long)untypedValue;
serializer.Serialize(writer, value.ToString());
return;
}
public static readonly ParseStringConverter Singleton = new ParseStringConverter();
}
public class InfoEnumConverter : JsonConverter
{
public override bool CanConvert(Type t) => t == typeof(InfoEnum) || t == typeof(InfoEnum?);
public override object ReadJson(JsonReader reader, Type t, object existingValue, JsonSerializer serializer)
{
if (reader.TokenType == JsonToken.Null) return null;
var value = serializer.Deserialize<string>(reader);
switch (value)
{
case "":
return InfoEnum.Empty;
case "Count : 0 (70:00 - 79:59)":
return InfoEnum.Count070007959;
case "Current Corners : 10":
return InfoEnum.CurrentCorners11;
default:
return InfoEnum.OtherType;
}
//throw new Exception("Cannot unmarshal type InfoEnum");
}
public override void WriteJson(JsonWriter writer, object untypedValue, JsonSerializer serializer)
{
if (untypedValue == null)
{
serializer.Serialize(writer, null);
return;
}
var value = (InfoEnum)untypedValue;
switch (value)
{
case InfoEnum.Empty:
serializer.Serialize(writer, "");
return;
case InfoEnum.Count070007959:
serializer.Serialize(writer, "Count : 0 (70:00 - 79:59)");
return;
case InfoEnum.CurrentCorners11:
serializer.Serialize(writer, "Current Corners : 11");
return;
}
throw new Exception("Cannot marshal type InfoEnum");
}
public static readonly InfoEnumConverter Singleton = new InfoEnumConverter();
}
}
This is an example of JSON feed received from my feed provider : https://filebin.net/k5enw2wn4f5bc89m/inplay-soccer.json?t=84y0df94
ValuesController
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.SqlClient;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using HelloWorld.Models;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
// For more information on enabling Web API for empty projects, visit https://go.microsoft.com/fwlink/?LinkID=397860
namespace HelloWorld.Controllers
{
[Route("")]
public class ValuesController : Controller
{
// POST api/<controller>
[HttpPost]
[Consumes("application/gzip")]
public async Task PostAsync()
{
var timestamp = new DateTimeOffset(DateTime.UtcNow).ToUnixTimeSeconds();
WebClient Client = new WebClient();
Client.DownloadFile("http://inplay.goalserve.com/inplay-soccer.gz", "C:\\temp\\inplay-soccer_" + timestamp + ".gz");
using (var inputFileStream = new FileStream("c:\\temp\\inplay-soccer_" + timestamp + ".gz", FileMode.Open))
using (var gzipStream = new GZipStream(inputFileStream, CompressionMode.Decompress))
using (var outputFileStream = new FileStream("c:\\temp_json\\inplay-soccer_" + timestamp + ".json", FileMode.Create))
{
await gzipStream.CopyToAsync(outputFileStream);
}
}
internal class TimedHostedService : IHostedService, IDisposable
{
private readonly ILogger _logger;
private Timer _timer;
FileProcessor fileProcessor = new FileProcessor();
FileSystemWatcher watcher = new FileSystemWatcher();
public TimedHostedService(ILogger<TimedHostedService> logger)
{
_logger = logger;
}
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Timed Background Service is starting.");
var startTimeSpan = TimeSpan.Zero;
var periodTimeSpan = TimeSpan.FromSeconds(1);
watcher.Path = "c:\\temp_json\\";
watcher.Filter = "*.json";
watcher.Created += OnChanged;
watcher.Changed += OnChanged;
watcher.EnableRaisingEvents = true;
return Task.CompletedTask;
}
private void OnChanged(object sender, FileSystemEventArgs e)
{
//WebClient Client = new WebClient();
//Client.DownloadFile("http://inplay.goalserve.com/inplay-soccer.gz", "C:\\temp\\inplay-soccer-jecoute-toutes-les-1sec"+e.Name+");
if (e.ChangeType == WatcherChangeTypes.Created)
{
var str = e.Name;
str = str.Remove(str.Length - 5);
//Client.DownloadFile("http://inplay.goalserve.com/inplay-soccer.gz", "C:\\temp\\"+str);
// ...enqueue it's file name so it can be processed...
fileProcessor.EnqueueFileName(str);
}
}
// File processor class
class FileProcessor : IDisposable
{
// Create an AutoResetEvent EventWaitHandle
private EventWaitHandle eventWaitHandle = new AutoResetEvent(false);
private Thread worker;
private readonly object locker = new object();
private Queue<string> fileNamesQueue = new Queue<string>();
public FileProcessor()
{
// Create worker thread
worker = new Thread(Work);
// Start worker thread
worker.Start();
}
public void EnqueueFileName(string FileName)
{
// Enqueue the file name
// This statement is secured by lock to prevent other thread to mess with queue while enqueuing file name
lock (locker) fileNamesQueue.Enqueue(FileName);
// Signal worker that file name is enqueued and that it can be processed
eventWaitHandle.Set();
}
private void Work()
{
while (true)
{
string fileName = null;
// Dequeue the file name
lock (locker)
if (fileNamesQueue.Count > 0)
{
fileName = fileNamesQueue.Dequeue();
// If file name is null then stop worker thread
if (fileName == null) return;
}
if (fileName != null)
{
// Process file
ProcessFile(fileName);
}
else
{
// No more file names - wait for a signal
eventWaitHandle.WaitOne();
}
}
}
private void ProcessFile(string FileName)
{
PutInDb(FileName);
// Maybe it has to wait for file to stop being used by process that created it before it can continue
// Read content file
// Log file data to database
// Move file to archive folder
}
#region IDisposable Members
public void Dispose()
{
// Signal the FileProcessor to exit
EnqueueFileName(null);
// Wait for the FileProcessor's thread to finish
worker.Join();
eventWaitHandle.Close();
}
#endregion
}
private void DoWork(object state)
{
_logger.LogInformation("Timed Background Service is working.");
WebClient Client = new WebClient();
Client.DownloadFile("http://inplay.goalserve.com/inplay-soccer.gz", "C:\\temp\\inplay-soccer-jecoute-toutes-les-1sec.gz");
}
public Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Timed Background Service is stopping.");
_timer?.Change(Timeout.Infinite, 0);
return Task.CompletedTask;
}
public void Dispose()
{
_timer?.Dispose();
}
}
public static void PutInDb(string save)
{
WebClient Client = new WebClient();
using (StreamReader r = new StreamReader("c:\\temp_json\\" + save + ".json"))
{
var con = new SqlConnection(@"Server=WINDOWS-ASPNET-\SQLEXPRESS01;Database=goalserve;Trusted_Connection=false;User ID=******;Password=*********;");
try
{
con.Open();
}
catch
{
//Client.DownloadFile("http://inplay.goalserve.com/inplay-soccer.gz", "C:\\temp\\inplay-soccer_noooo" + save + ".gz");
}
var cmd = new SqlCommand("INSERT INTO Event (id, league, name, start_date, start_time, start_ts, score, period, minute, seconds, additional_time, state, ball_pos, player) VALUES (@id, @league, @name, @start_date, @start_time, @start_ts, @score, @period, @minute, @seconds, @additional_time, @state, @ball_pos, @player)", con);
using (JsonTextReader reader = new JsonTextReader(r))
{
reader.SupportMultipleContent = true;
var serializer = new JsonSerializer();
while (reader.Read())
{
if (reader.TokenType == JsonToken.StartObject)
{
Goalserve objects = serializer.Deserialize<Goalserve>(reader);
var data = objects.Events.Keys.ToArray();
for (int i = 0; i < data.Count(); i++)
{
// just test all this treatment with one random match data
if (objects.Events[data[i].ToString()].Info.Name == "America RJ vs Macae Esporte FC")
{
cmd.Parameters.Add("@id", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Id;
cmd.Parameters.Add("@league", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.League;
cmd.Parameters.Add("@name", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Name;
cmd.Parameters.Add("@start_date", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.StartDate;
cmd.Parameters.Add("@start_time", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.StartTime;
cmd.Parameters.Add("@start_ts", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.StartTs;
cmd.Parameters.Add("@score", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Score;
cmd.Parameters.Add("@period", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Period;
cmd.Parameters.Add("@minute", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Minute;
cmd.Parameters.Add("@seconds", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Secunds;
cmd.Parameters.Add("@additional_time", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.AddTime;
cmd.Parameters.Add("@state", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.State;
cmd.Parameters.Add("@ball_pos", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.BallPos;
cmd.Parameters.Add("@player", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Player;
cmd.ExecuteNonQuery();
}
}
}
}
}
}
}
}
}
Please give me your own implementation even if I need to change all the method initially used.
thank you in advance !
EDIT2 ValuesController
[HttpPost]
[Consumes("application/gzip")]
public async Task PostAsync()
{
var con = new SqlConnection(@"Server=WINDOWS-ASPNET-\SQLEXPRESS01;Database=goalserve;Trusted_Connection=false;User ID=*****;Password=*****;");
var cmd = new SqlCommand("INSERT INTO Event (id, league, name, start_date, start_time, start_ts, score, period, minute, seconds, additional_time, state, ball_pos, player) VALUES (@id, @league, @name, @start_date, @start_time, @start_ts, @score, @period, @minute, @seconds, @additional_time, @state, @ball_pos, @player)", con);
var timestamp = new DateTimeOffset(DateTime.UtcNow).ToUnixTimeSeconds();
var client = new HttpClient();
var response = await client.GetAsync(@"http://inplay.goalserve.com/inplay-soccer.gz");
using (var stream = await response.Content.ReadAsStreamAsync())
{
var fileInfo = new FileInfo("C:\\temp\\inplay - soccer_"+timestamp+".gz");
using (var fileStream = fileInfo.OpenWrite())
{
await stream.CopyToAsync(fileStream);
}
}
using (FileStream fInStream = new FileStream(@"C:\\temp\\inplay - soccer_" + timestamp + ".gz", FileMode.Open, FileAccess.Read))
{
using (GZipStream zipStream = new GZipStream(fInStream, CompressionMode.Decompress))
{
using (FileStream fOutStream = new FileStream(@"C:\\temp\\inplay - soccer_" + timestamp + ".json", FileMode.Create, FileAccess.Write))
{
byte[] tempBytes = new byte[4096];
int i;
while ((i = zipStream.Read(tempBytes, 0, tempBytes.Length)) != 0)
{
fOutStream.Write(tempBytes, 0, i);
}
}
}
}
using (StreamReader r = new StreamReader("c:\\temp\\inplay - soccer_" + timestamp + ".json"))
{
var objects = new Goalserve();
string json = r.ReadToEnd();
objects = JsonConvert.DeserializeObject<Goalserve>(json);
var data = objects.Events.Keys.ToArray();
for (int i = 0; i < data.Count(); i++)
{
if (objects.Events[data[i].ToString()].Info.Name == "Blackburn vs Wigan")
{
con.Open();
cmd.Parameters.Add("@id", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Id;
cmd.Parameters.Add("@league", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.League;
cmd.Parameters.Add("@name", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Name;
cmd.Parameters.Add("@start_date", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.StartDate;
cmd.Parameters.Add("@start_time", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.StartTime;
cmd.Parameters.Add("@start_ts", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.StartTs;
cmd.Parameters.Add("@score", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Score;
cmd.Parameters.Add("@period", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Period;
cmd.Parameters.Add("@minute", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Minute;
cmd.Parameters.Add("@seconds", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Secunds;
cmd.Parameters.Add("@additional_time", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.AddTime;
cmd.Parameters.Add("@state", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.State;
cmd.Parameters.Add("@ball_pos", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.BallPos;
cmd.Parameters.Add("@player", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Player;
cmd.ExecuteNonQuery();
}
}
来源：https://stackoverflow.com/questions/59454040/problem-of-performance-optimize-json-deserialization-speed-in-c-sharp