Fix Insert large collection data with BulkInsert

mbbipul · Aug 25, 2020 · cfce261 · cfce261
1 parent d0296e0
commit cfce261
Show file tree

Hide file tree

Showing 145 changed files with 2,090 additions and 3,305 deletions.
diff --git a/Controllers/OcrdatasetController.cs b/Controllers/OcrdatasetController.cs
@@ -96,11 +96,14 @@ public async Task<ActionResult<OcrClass>> PostOcrClass(OcrClass ocrClass)
             await repository.AddOcrClass(ocrClass);
             return CreatedAtAction("GetOcrClass", new { id = ocrClass.Id }, ocrClass);
         }
+        [DisableRequestSizeLimit]
         [HttpPost("all")]
         public async Task<ActionResult<OcrClass>> PostOcrClasses(List<OcrClass> ocrClasses)
         {
-            Console.WriteLine(ocrClasses);
+            Console.WriteLine(ocrClasses.Count());
+            // This might speed up things a little aswell
             await repository.AddOcrClasses(ocrClasses);
+            Console.WriteLine("done");
             return CreatedAtAction("GetOcrClasses", new { count = ocrClasses.Count }, ocrClasses);
         }
         [HttpDelete("all")]

diff --git a/Controllers/WeatherForecastController.cs b/Controllers/WeatherForecastController.cs
@@ -1,39 +1,39 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.Logging;
-
-namespace restApiDataset.Controllers
-{
-    [ApiController]
-    [Route("[controller]")]
-    public class WeatherForecastController : ControllerBase
-    {
-        private static readonly string[] Summaries = new[]
-        {
-            "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching"
-        };
-
-        private readonly ILogger<WeatherForecastController> _logger;
-
-        public WeatherForecastController(ILogger<WeatherForecastController> logger)
-        {
-            _logger = logger;
-        }
-
-        [HttpGet]
-        public IEnumerable<WeatherForecast> Get()
-        {
-            var rng = new Random();
-            return Enumerable.Range(1, 5).Select(index => new WeatherForecast
-            {
-                Date = DateTime.Now.AddDays(index),
-                TemperatureC = rng.Next(-20, 55),
-                Summary = Summaries[rng.Next(Summaries.Length)]
-            })
-            .ToArray();
-        }
-    }
-}
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+
+namespace ocrapi.Controllers
+{
+    [ApiController]
+    [Route("[controller]")]
+    public class WeatherForecastController : ControllerBase
+    {
+        private static readonly string[] Summaries = new[]
+        {
+            "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching"
+        };
+
+        private readonly ILogger<WeatherForecastController> _logger;
+
+        public WeatherForecastController(ILogger<WeatherForecastController> logger)
+        {
+            _logger = logger;
+        }
+
+        [HttpGet]
+        public IEnumerable<WeatherForecast> Get()
+        {
+            var rng = new Random();
+            return Enumerable.Range(1, 5).Select(index => new WeatherForecast
+            {
+                Date = DateTime.Now.AddDays(index),
+                TemperatureC = rng.Next(-20, 55),
+                Summary = Summaries[rng.Next(Summaries.Length)]
+            })
+            .ToArray();
+        }
+    }
+}
diff --git a/.../20200415170800_InitialCreate.Designer.cs → .../20200824111432_InitialCreate.Designer.cs b/.../20200415170800_InitialCreate.Designer.cs → .../20200824111432_InitialCreate.Designer.cs
diff --git a/Migrations/20200415170800_InitialCreate.cs → Migrations/20200824111432_InitialCreate.cs b/Migrations/20200415170800_InitialCreate.cs → Migrations/20200824111432_InitialCreate.cs
diff --git a/Migrations/20200825180015_InitialCreate.Designer.cs b/Migrations/20200825180015_InitialCreate.Designer.cs
diff --git a/Migrations/20200825180015_InitialCreate.cs b/Migrations/20200825180015_InitialCreate.cs
@@ -0,0 +1,33 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace ocrapi.Migrations
+{
+    public partial class InitialCreate : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "OcrClasses",
+                columns: table => new
+                {
+                    Id = table.Column<long>(nullable: false)
+                        .Annotation("SqlServer:Identity", "1, 1"),
+                    FileName = table.Column<string>(nullable: true),
+                    ImageData = table.Column<string>(nullable: true),
+                    GraphemeRootId = table.Column<string>(nullable: true),
+                    VowelDiacreticId = table.Column<string>(nullable: true),
+                    ConsonantDiacreticId = table.Column<string>(nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_OcrClasses", x => x.Id);
+                });
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "OcrClasses");
+        }
+    }
+}
diff --git a/Migrations/OcrDbContextModelSnapshot.cs b/Migrations/OcrDbContextModelSnapshot.cs
@@ -0,0 +1,50 @@
+// <auto-generated />
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using restApiDataset.Models;
+
+namespace ocrapi.Migrations
+{
+    [DbContext(typeof(OcrDbContext))]
+    partial class OcrDbContextModelSnapshot : ModelSnapshot
+    {
+        protected override void BuildModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("ProductVersion", "3.1.7")
+                .HasAnnotation("Relational:MaxIdentifierLength", 128)
+                .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn);
+
+            modelBuilder.Entity("restApiDataset.Models.OcrClass", b =>
+                {
+                    b.Property<long>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("bigint")
+                        .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn);
+
+                    b.Property<string>("ConsonantDiacreticId")
+                        .HasColumnType("nvarchar(max)");
+
+                    b.Property<string>("FileName")
+                        .HasColumnType("nvarchar(max)");
+
+                    b.Property<string>("GraphemeRootId")
+                        .HasColumnType("nvarchar(max)");
+
+                    b.Property<string>("ImageData")
+                        .HasColumnType("nvarchar(max)");
+
+                    b.Property<string>("VowelDiacreticId")
+                        .HasColumnType("nvarchar(max)");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("OcrClasses");
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}
diff --git a/Models/ApplicationDbContext.cs b/Models/ApplicationDbContext.cs
@@ -5,7 +5,8 @@ namespace restApiDataset.Models
     public class ApplicationDbContext : DbContext
     {
         public ApplicationDbContext(DbContextOptions<ApplicationDbContext> options) : base(options){
-            this.Database.SetCommandTimeout(180);
+
+            this.Database.SetCommandTimeout(370);
         }
         public DbSet<OcrClass> OcrClasses { get; set;}
     }

diff --git a/Models/EFConsonantDiacretic.cs b/Models/EFConsonantDiacretic.cs
@@ -4,8 +4,8 @@ namespace restApiDataset.Models
 {
     public class EFConsonantDiacretic : IConsonantDiacretic
     {
-        ApplicationDbContext context;
-        public EFConsonantDiacretic(ApplicationDbContext ctx){
+        OcrDbContext context;
+        public EFConsonantDiacretic(OcrDbContext ctx){
             this.context = ctx;
         }
         public IQueryable<OcrClass> GetConsonantDiacretics(){

diff --git a/Models/EFGraphemerootRepository.cs b/Models/EFGraphemerootRepository.cs
@@ -7,8 +7,8 @@ namespace restApiDataset.Models
 {
     public class EFGraphemerootRepository : IGraphemerootRepository
     {
-        private ApplicationDbContext context;   
-        public EFGraphemerootRepository(ApplicationDbContext ctx){
+        private OcrDbContext context;   
+        public EFGraphemerootRepository(OcrDbContext ctx){
             this.context = ctx;
         }
         public IQueryable<OcrClass> GetGraphemeroots(){

diff --git a/Models/EFOcrclassRepository.cs b/Models/EFOcrclassRepository.cs
@@ -7,8 +7,8 @@ namespace restApiDataset.Models
 {
     public class EFOcrclassRepository : IOcrclassRepository
     {
-        private ApplicationDbContext context;   
-        public EFOcrclassRepository(ApplicationDbContext ctx){
+        private OcrDbContext context;   
+        public EFOcrclassRepository(OcrDbContext ctx){
             this.context = ctx;
         }
         public IQueryable<OcrClass> OcrClasses => context.OcrClasses;
@@ -22,8 +22,8 @@ public async Task AddOcrClass(OcrClass ocrClass){
             await context.SaveChangesAsync();
         }
         public async Task AddOcrClasses(List<OcrClass> ocrClass){
-            context.OcrClasses.AddRange(ocrClass);
-            await context.SaveChangesAsync();
+            await context.OcrClasses.BulkInsertAsync(ocrClass);
+            await context.BulkSaveChangesAsync();
         }
         public async Task<bool> UpdateOcrClass(OcrClass ocrClass){
             OcrClass dbEntry = context.OcrClasses.
@@ -51,8 +51,8 @@ public async Task<OcrClass> DeleteOcrClassR(int ocrClassId){
             return null;
         }
         public async Task DeleteAllOcrClass(){
-            context.OcrClasses.RemoveRange(context.OcrClasses);
-            await context.SaveChangesAsync();
+            await context.OcrClasses.BulkDeleteAsync(context.OcrClasses);
+            await context.BulkSaveChangesAsync();
         }
         public IQueryable<OcrClass> GetClassesByFontName(string fontName){
             return context.OcrClasses

diff --git a/Models/EFVowelDiacretic.cs b/Models/EFVowelDiacretic.cs
@@ -4,8 +4,8 @@ namespace restApiDataset.Models
 {
     public class EFVowelDiacretic : IVowelDiacretic
     {
-        private ApplicationDbContext context;
-        public EFVowelDiacretic(ApplicationDbContext ctx){
+        private OcrDbContext context;
+        public EFVowelDiacretic(OcrDbContext ctx){
             this.context = ctx;
         }
         public IQueryable<OcrClass> GetVowelDiacretics(){

diff --git a/Models/OcrDbContext.cs b/Models/OcrDbContext.cs
@@ -0,0 +1,14 @@
+using Microsoft.EntityFrameworkCore;
+
+namespace restApiDataset.Models
+{
+    public class OcrDbContext : DbContext
+    {
+        public OcrDbContext(){}
+        public OcrDbContext(DbContextOptions<OcrDbContext> options) : base(options){
+            Database.SetCommandTimeout(150000);
+
+        }
+        public DbSet<OcrClass> OcrClasses { get; set;}
+    }
+}
diff --git a/Program.cs b/Program.cs
@@ -1,28 +1,26 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Hosting;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-
-namespace restApiDataset
-{
-    public class Program
-    {
-        public static void Main(string[] args)
-        {
-            CreateHostBuilder(args).Build().Run();
-        }
-
-        public static IHostBuilder CreateHostBuilder(string[] args) =>
-            Host.CreateDefaultBuilder(args)
-                .ConfigureWebHostDefaults(webBuilder =>
-                {
-                    webBuilder.UseStartup<Startup>()
-                    .UseDefaultServiceProvider(options =>
-                        options.ValidateScopes = false);
-                });
-    }
-}
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace ocrapi
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            CreateHostBuilder(args).Build().Run();
+        }
+
+        public static IHostBuilder CreateHostBuilder(string[] args) =>
+            Host.CreateDefaultBuilder(args)
+                .ConfigureWebHostDefaults(webBuilder =>
+                {
+                    webBuilder.UseStartup<Startup>();
+                });
+    }
+}