From 253901d03a38a3ddc4ea0cdedb8832f8c1509d35 Mon Sep 17 00:00:00 2001 From: Barncastle Date: Thu, 1 Mar 2018 12:10:44 +0000 Subject: [PATCH 01/11] tweaks --- WDBXEditor/Definitions/BfA 8.0.1 (25902).xml | 5 +++-- WDBXEditor/Definitions/WoD 6.2.4 (21742).xml | 16 +--------------- WDBXEditor/Reader/FileTypes/WDC1.cs | 2 +- WDBXEditor/Storage/DBEntry.cs | 8 ++++---- 4 files changed, 9 insertions(+), 22 deletions(-) diff --git a/WDBXEditor/Definitions/BfA 8.0.1 (25902).xml b/WDBXEditor/Definitions/BfA 8.0.1 (25902).xml index 31e1985..6931a25 100644 --- a/WDBXEditor/Definitions/BfA 8.0.1 (25902).xml +++ b/WDBXEditor/Definitions/BfA 8.0.1 (25902).xml @@ -5230,12 +5230,13 @@ - + + - + diff --git a/WDBXEditor/Definitions/WoD 6.2.4 (21742).xml b/WDBXEditor/Definitions/WoD 6.2.4 (21742).xml index b968e4f..8be586e 100644 --- a/WDBXEditor/Definitions/WoD 6.2.4 (21742).xml +++ b/WDBXEditor/Definitions/WoD 6.2.4 (21742).xml @@ -3285,6 +3285,7 @@ + @@ -3341,21 +3342,6 @@ - - - - - - - - - - - - - - -
diff --git a/WDBXEditor/Reader/FileTypes/WDC1.cs b/WDBXEditor/Reader/FileTypes/WDC1.cs index 990285a..f86a43b 100644 --- a/WDBXEditor/Reader/FileTypes/WDC1.cs +++ b/WDBXEditor/Reader/FileTypes/WDC1.cs @@ -365,7 +365,7 @@ public void SetColumnMinMaxValues(DBEntry entry) } int bits = ColumnMeta[i].CompressionType == CompressionType.None ? FieldStructure[i].BitCount : ColumnMeta[i].BitWidth; - if ((bits & (bits - 1)) == 0) // power of two so standard type + if ((bits & (bits - 1)) == 0 && bits >= 8) // power of two and >= sizeof(byte) means a standard type { column += ColumnMeta[i].ArraySize; continue; diff --git a/WDBXEditor/Storage/DBEntry.cs b/WDBXEditor/Storage/DBEntry.cs index f112f4c..e2afb73 100644 --- a/WDBXEditor/Storage/DBEntry.cs +++ b/WDBXEditor/Storage/DBEntry.cs @@ -295,7 +295,7 @@ public int[] GetPadding() int c = 0; - foreach(var field in header.ColumnMeta) + foreach (var field in header.ColumnMeta) { Type type = Data.Columns[c].DataType; bool isneeded = field.CompressionType >= CompressionType.Sparse; @@ -539,12 +539,12 @@ public void ToSQLTable(string connectionstring) /// public string ToCSV() { + Func EncodeCsv = s => { return string.Concat("\"", s.Replace(Environment.NewLine, string.Empty).Replace("\"", "\"\""), "\""); }; + StringBuilder sb = new StringBuilder(); - IEnumerable columnNames = Data.Columns.Cast().Select(column => column.ColumnName); + IEnumerable columnNames = Data.Columns.Cast().Select(column => EncodeCsv(column.ColumnName)); sb.AppendLine(string.Join(",", columnNames)); - Func EncodeCsv = s => { return string.Concat("\"", s.Replace(Environment.NewLine, string.Empty).Replace("\"", "\"\""), "\""); }; - foreach (DataRow row in Data.Rows) { IEnumerable fields = row.ItemArray.Select(field => EncodeCsv(field.ToString())); From 5ce104488aa31457b189d641b63be3b970a7ddf3 Mon Sep 17 00:00:00 2001 From: Barncastle Date: Sat, 3 Mar 2018 16:32:56 +0000 Subject: [PATCH 02/11] remap of 7.3.5 defs --- .../Definitions/Legion 7.3.5 (25632).xml | 2611 +++++++++++++++-- 1 file changed, 2307 insertions(+), 304 deletions(-) diff --git a/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml b/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml index 9726f2e..1b7b74b 100644 --- a/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml +++ b/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml @@ -1,21 +1,21 @@ - +
- - - - - - - - - - - - - - - + + + + + + + + + + + + + + +
@@ -64,6 +64,16 @@
+ + + + + + + + + +
@@ -71,6 +81,13 @@
+ + + + + + +
@@ -90,6 +107,12 @@
+ + + + + +
@@ -164,6 +187,10 @@
+ + + +
@@ -216,6 +243,32 @@
+ + + + + + + + + + + + + + + + + + + + + + + + + +
@@ -233,6 +286,23 @@
+ + + + + + + + + + + + + + + + +
@@ -259,18 +329,22 @@
+ + + +
- - - - - - - - - - + + + + + + + + + +
@@ -289,6 +363,23 @@
+ + + + + + + + + + + + + + + + +
@@ -299,11 +390,26 @@
+ + + + + + + + + +
+ + + + +
@@ -313,15 +419,33 @@
+ + + + + + + + +
+ + + + +
+ + + +
@@ -329,10 +453,21 @@
+ + + + + + +
+ + + +
@@ -355,16 +490,33 @@
+ + + + + + +
+ + + +
+ + + + + +
@@ -375,6 +527,16 @@
+ + + + + + + + + +
@@ -395,6 +557,26 @@
+ + + + + + + + + + + + + + + + + + + +
@@ -432,11 +614,21 @@
+ + + + +
+ + + + +
@@ -463,11 +655,29 @@
+ + + + + + + + + + + + +
+ + + + +
@@ -537,6 +747,18 @@
+ + + + + + + + + + + +
@@ -631,6 +853,13 @@
+ + + + + + +
@@ -667,6 +896,12 @@
+ + + + + +
@@ -696,14 +931,24 @@
- - - - - - - - + + + + + + + + +
+ + + + + + + + +
@@ -745,6 +990,15 @@
+ + + + + + + + +
@@ -752,6 +1006,13 @@
+ + + + + + +
@@ -759,6 +1020,13 @@
+ + + + + + +
@@ -786,10 +1054,36 @@
+ + + + + + + + + + + + + + + + + + + + + +
+ + + +
@@ -826,44 +1120,44 @@
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
@@ -880,6 +1174,21 @@
+ + + + + + + + + + + + + + +
@@ -901,11 +1210,23 @@
+ + + + + + +
+ + + + +
@@ -978,6 +1299,17 @@
+ + + + + + + + + + +
@@ -1002,29 +1334,54 @@
- - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + +
@@ -1050,19 +1407,34 @@
- - - - - - - - - - - - - + + + + + + + + + + + + + +
+ + + + + + + + + + + + + +
@@ -1089,6 +1461,18 @@
+ + + + + + + + + + + +
@@ -1102,36 +1486,36 @@
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
@@ -1182,6 +1566,11 @@
+ + + + +
@@ -1210,6 +1599,16 @@
+ + + + + + + + + +
@@ -1233,10 +1632,23 @@
+ + + + + + + + + + + + +
- - + +
@@ -1244,6 +1656,13 @@
+ + + + + + +
@@ -1295,6 +1714,31 @@
+ + + + + + + + + + + + + + + + + + + + + + + + +
@@ -1308,19 +1752,19 @@
- - - - - - - - - - - - - + + + + + + + + + + + + +
@@ -1360,6 +1804,17 @@
+ + + + + + + + + + +
@@ -1383,10 +1838,19 @@
+ + + + +
+ + + +
@@ -1405,21 +1869,26 @@
- - - - - - - - - + + + + + + + + +
+ + + + +
@@ -1432,6 +1901,13 @@
+ + + + + + +
@@ -1448,22 +1924,24 @@
- - - + + + - - - - - - - - - - - - + + + + + + + + + + + + + +
@@ -1483,6 +1961,16 @@
+ + + + + + + + + +
@@ -1557,6 +2045,15 @@
+ + + + + + + + +
@@ -1576,6 +2073,23 @@
+ + + + + + + + + + + + + + + + +
@@ -1593,6 +2107,16 @@
+ + + + + + + + + +
@@ -1638,6 +2162,33 @@
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
@@ -1653,6 +2204,13 @@
+ + + + + + +
@@ -1663,6 +2221,16 @@
+ + + + + + + + + +
@@ -1730,6 +2298,40 @@
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
@@ -1770,6 +2372,11 @@
+ + + + +
@@ -1800,6 +2407,36 @@
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
@@ -1870,16 +2507,37 @@
+ + + + + + + + + + +
+ + + + +
+ + + + +
@@ -1897,6 +2555,18 @@
+ + + + + + + + + + + +
@@ -1904,6 +2574,13 @@
+ + + + + + +
@@ -1918,6 +2595,28 @@
+ + + + + + + + + + + + + + + + + + + + + +
@@ -1987,6 +2686,12 @@
+ + + + + +
@@ -1997,6 +2702,10 @@
+ + + +
@@ -2008,10 +2717,21 @@
+ + + + + + +
+ + + +
@@ -2077,32 +2797,53 @@
+ + + + + +
+ + + + + +
+ + + + + +
+ + + +
- - - - - - - - - + + + + + + + +
@@ -2132,17 +2873,17 @@
- - - - - - - - - - - + + + + + + + + + + +
@@ -2151,18 +2892,37 @@
+ + + + + + +
+ + + +
+ + + +
+ + + +
@@ -2176,14 +2936,14 @@
- - - - - - - - + + + + + + + +
@@ -2192,6 +2952,13 @@
+ + + + + + +
@@ -2202,11 +2969,21 @@
+ + + + +
+ + + + +
@@ -2215,10 +2992,19 @@
+ + + + +
+ + + +
@@ -2226,10 +3012,21 @@
+ + + + + + +
+ + + +
@@ -2237,11 +3034,23 @@
+ + + + + + +
+ + + + +
@@ -2249,6 +3058,13 @@
+ + + + + + +
@@ -2261,31 +3077,60 @@
+ + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
@@ -2295,23 +3140,32 @@
+ + + + + + + + +
- - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + +
@@ -2334,6 +3188,17 @@
+ + + + + + + + + + +
@@ -2347,6 +3212,19 @@
+ + + + + + + + + + + + +
@@ -2356,22 +3234,43 @@
+ + + + +
+ + + + +
+ + + + +
+ + + + + +
@@ -2385,6 +3284,14 @@
+ + + + + + + +
@@ -2408,17 +3315,34 @@
+ + + + + +
+ + + + +
+ + + + + +
@@ -2442,6 +3366,22 @@
+ + + + + + + + + + + + + + + +
@@ -2450,12 +3390,26 @@
+ + + + + + + +
+ + + + + +
@@ -2523,6 +3477,73 @@
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
@@ -2532,10 +3553,23 @@
+ + + + + + + + +
+ + + +
@@ -2563,6 +3597,14 @@
+ + + + + + + +
@@ -2571,6 +3613,10 @@
+ + + +
@@ -2610,7 +3656,7 @@ - + @@ -2665,7 +3711,7 @@
- +
@@ -2734,6 +3780,42 @@
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
@@ -2745,12 +3827,12 @@
- - - - - - + + + + + +
@@ -2790,17 +3872,17 @@
- - - - - - - - - - - + + + + + + + + + + +
@@ -2844,6 +3926,28 @@
+ + + + + + + + + + + + + + + + + + + + + +
@@ -2871,11 +3975,11 @@
- - - - - + + + + +
@@ -2909,6 +4013,10 @@
+ + + +
@@ -2953,27 +4061,27 @@
- - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + +
@@ -2998,6 +4106,17 @@
+ + + + + + + + + + +
@@ -3072,18 +4191,27 @@
+ + + + + + + + +
- - - - - - - - + + + + + + + + - - + +
@@ -3095,23 +4223,44 @@
+ + + + + + + + + +
+ + + + + +
+ + + + +
- - - - + + + +
@@ -3128,20 +4277,40 @@
+ + + + + +
+ + + + +
+ + + +
+ + + + +
@@ -3190,6 +4359,12 @@
+ + + + + +
@@ -3243,7 +4418,7 @@
- +
@@ -3265,6 +4440,93 @@
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
@@ -3388,6 +4650,14 @@
+ + + + + + + +
@@ -3403,6 +4673,21 @@
+ + + + + + + + + + + + + + +
@@ -3410,6 +4695,13 @@
+ + + + + + +
@@ -3421,6 +4713,12 @@
+ + + + + +
@@ -3432,6 +4730,12 @@
+ + + + + +
@@ -3465,6 +4769,10 @@
+ + + +
@@ -3495,6 +4803,10 @@
+ + + +
@@ -3512,6 +4824,13 @@
+ + + + + + +
@@ -3532,10 +4851,19 @@
+ + + + +
+ + + +
@@ -3560,7 +4888,7 @@ - +
@@ -3571,12 +4899,22 @@
+ + + +
+ + + + + +
@@ -3638,6 +4976,15 @@
+ + + + + + + + +
@@ -3648,6 +4995,11 @@
+ + + + +
@@ -3661,24 +5013,41 @@
+ + + + +
+ + + + + +
+ + + + + +
- - - - + + + +
@@ -3697,20 +5066,46 @@
+ + + + + + + + + + + +
+ + + + +
+ + + + +
+ + + +
@@ -3723,6 +5118,11 @@
+ + + + +
@@ -3808,6 +5208,17 @@
+ + + + + + + + + + +
@@ -3824,6 +5235,22 @@
+ + + + + + + + + + + + + + + +
@@ -3834,6 +5261,16 @@
+ + + + + + + + + +
@@ -3919,6 +5356,24 @@
+ + + + + + + + + + + + + + + + + +
@@ -4033,12 +5488,20 @@
+ + + + + + + +
- - - - - + + + + +
@@ -4066,6 +5529,16 @@
+ + + + + + + + + +
@@ -4078,6 +5551,18 @@
+ + + + + + + + + + + +
@@ -4097,12 +5582,28 @@
+ + + + + + + + + +
+ + + + + +
@@ -4114,6 +5615,17 @@
+ + + + + + + + + + +
@@ -4123,6 +5635,15 @@
+ + + + + + + + +
@@ -4193,6 +5714,13 @@
+ + + + + + +
@@ -4200,6 +5728,13 @@
+ + + + + + +
@@ -4217,6 +5752,12 @@
+ + + + + +
@@ -4248,6 +5789,39 @@
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
@@ -4262,6 +5836,13 @@
+ + + + + + +
@@ -4280,6 +5861,10 @@
+ + + +
@@ -4287,6 +5872,13 @@
+ + + + + + +
@@ -4309,6 +5901,37 @@
+ + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + +
@@ -4325,6 +5948,12 @@
+ + + + + +
@@ -4333,6 +5962,14 @@
+ + + + + + + +
@@ -4350,6 +5987,32 @@
+ + + + + + + + + + + + + + + + + + + + + + + + + +
@@ -4390,11 +6053,31 @@
+ + + + + + + + + + + + + + +
+ + + + +
@@ -4405,12 +6088,23 @@
+ + + + +
+ + + + + +
@@ -4418,6 +6112,13 @@
+ + + + + + +
@@ -4426,12 +6127,28 @@
+ + + + + + + + + +
+ + + + + +
@@ -4446,6 +6163,14 @@
+ + + + + + + +
@@ -4453,6 +6178,13 @@
+ + + + + + +
@@ -4466,6 +6198,19 @@
+ + + + + + + + + + + + +
@@ -4481,12 +6226,28 @@
+ + + + + + + + + +
+ + + + + +
@@ -4632,6 +6393,20 @@
+ + + + + + + + + + + + + +
@@ -4661,21 +6436,25 @@
+ + + +
- - - - - - - - - + + + + + + + + +
@@ -4692,12 +6471,33 @@
+ + + + + + + + + + + + + + +
+ + + + + +
@@ -4709,6 +6509,17 @@
+ + + + + + + + + + +
@@ -4751,11 +6562,17 @@
+ + + + + +
- - - - + + + +
@@ -4783,6 +6600,10 @@
+ + + +
@@ -4795,22 +6616,50 @@
+ + + + + + + + + + + +
+ + + +
+ + + + + +
+ + + + + +
@@ -4829,6 +6678,14 @@
+ + + + + + + +
@@ -4990,6 +6847,25 @@
+ + + + + + + + + + + + + + + + + + +
@@ -5075,6 +6951,68 @@
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
@@ -5219,6 +7157,22 @@
+ + + + + + + + + + + + + + + +
@@ -5259,6 +7213,15 @@
+ + + + + + + + +
@@ -5284,6 +7247,25 @@
+ + + + + + + + + + + + + + + + + + +
@@ -5327,6 +7309,20 @@
+ + + + + + + + + + + + + +
@@ -5334,6 +7330,13 @@
+ + + + + + +
From b48deb99ede4739f2f38e8b5dcd2da3c09319499 Mon Sep 17 00:00:00 2001 From: Barncastle Date: Sat, 3 Mar 2018 16:33:18 +0000 Subject: [PATCH 03/11] formatting --- WDBXEditor/Reader/FileTypes/WDC1.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/WDBXEditor/Reader/FileTypes/WDC1.cs b/WDBXEditor/Reader/FileTypes/WDC1.cs index f86a43b..2fc4fd9 100644 --- a/WDBXEditor/Reader/FileTypes/WDC1.cs +++ b/WDBXEditor/Reader/FileTypes/WDC1.cs @@ -204,7 +204,7 @@ public override void ReadHeader(ref BinaryReader dbReader, string signature) { id = m_indexes[CopyTable.Count]; var map = offsetmap[i]; - + dbReader.BaseStream.Position = map.Item1; byte[] data = dbReader.ReadBytes(map.Item2); @@ -308,7 +308,7 @@ public override void ReadHeader(ref BinaryReader dbReader, string signature) else data.AddRange(new byte[4]); } - + CopyTable.Add(id, data.ToArray()); if (Copies.ContainsKey(id)) @@ -487,8 +487,8 @@ public void WriteData(BinaryWriter bw, DBEntry entry) foreach (DataRow r in entry.Data.Rows) { int id = r.Field(entry.Key); - if(!copyIds.Contains(id)) - relationData.Add(id, r.Field(index)); + if (!copyIds.Contains(id)) + relationData.Add(id, r.Field(index)); } RelationShipData = new RelationShipData() @@ -758,7 +758,8 @@ private object[] ExtractFields(Queue rowData, StringTable stringTable, B } else { - values = values.Select(x => (object)stringTable.Write((string)x)).ToArray(); + for (int i = 0; i < values.Length; i++) + values[i] = stringTable.Write((string)values[i]); } } From b88ec42beb1822d9baa7546987a7fa0d7c191c0f Mon Sep 17 00:00:00 2001 From: Barncastle Date: Sat, 3 Mar 2018 17:22:02 +0000 Subject: [PATCH 04/11] fixes --- .../Definitions/Legion 7.3.5 (25632).xml | 1840 +---------------- 1 file changed, 8 insertions(+), 1832 deletions(-) diff --git a/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml b/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml index 1b7b74b..10c5f26 100644 --- a/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml +++ b/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml @@ -1,5 +1,5 @@ - + @@ -101,12 +101,6 @@
- - - - - -
@@ -183,10 +177,6 @@
- - - -
@@ -217,32 +207,6 @@
- - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -269,23 +233,6 @@
- - - - - - - - - - - - - - - - -
@@ -321,14 +268,6 @@
- - - - - - - -
@@ -346,23 +285,6 @@
- - - - - - - - - - - - - - - - -
@@ -380,16 +302,6 @@
- - - - - - - - - -
@@ -400,25 +312,11 @@
- - - - -
- - - - - - - - -
@@ -428,31 +326,15 @@
- - - - -
- - - -
- - - - - - -
@@ -460,10 +342,6 @@
- - - -
@@ -483,13 +361,6 @@
- - - - - - -
@@ -497,36 +368,16 @@
- - - -
- - - - - -
- - - - - - - - - -
@@ -537,26 +388,6 @@
- - - - - - - - - - - - - - - - - - - -
@@ -609,21 +440,11 @@
- - - - -
- - - - -
@@ -642,19 +463,6 @@
- - - - - - - - - - - - -
@@ -668,11 +476,6 @@
- - - - -
@@ -735,18 +538,6 @@
- - - - - - - - - - - -
@@ -846,13 +637,6 @@
- - - - - - -
@@ -890,12 +674,6 @@
- - - - - -
@@ -930,16 +708,6 @@
- - - - - - - - - -
@@ -981,15 +749,6 @@
- - - - - - - - -
@@ -999,13 +758,6 @@
- - - - - - -
@@ -1013,13 +765,6 @@
- - - - - - -
@@ -1032,28 +777,6 @@
- - - - - - - - - - - - - - - - - - - - - -
@@ -1076,10 +799,6 @@
- - - -
@@ -1159,21 +878,6 @@
- - - - - - - - - - - - - - -
@@ -1203,25 +907,13 @@
- - - - - - -
- +
- - - - -
@@ -1288,17 +980,6 @@
- - - - - - - - - - -
@@ -1333,31 +1014,6 @@
- - - - - - - - - - - - - - - - - - - - - - - - -
@@ -1406,21 +1062,6 @@
- - - - - - - - - - - - - - -
@@ -1449,18 +1090,6 @@
- - - - - - - - - - - -
@@ -1561,11 +1190,6 @@
- - - - -
@@ -1589,16 +1213,6 @@
- - - - - - - - - -
@@ -1619,19 +1233,6 @@
- - - - - - - - - - - - -
@@ -1650,12 +1251,6 @@
- - - - - -
@@ -1689,31 +1284,6 @@
- - - - - - - - - - - - - - - - - - - - - - - - -
@@ -1793,17 +1363,6 @@
- - - - - - - - - - -
@@ -1833,20 +1392,11 @@
- - - - -
- - - -
@@ -1879,11 +1429,6 @@
- - - - -
@@ -1894,13 +1439,6 @@
- - - - - - -
@@ -1951,16 +1489,6 @@
- - - - - - - - - -
@@ -2037,14 +1565,6 @@
- - - - - - - -
@@ -2059,20 +1579,6 @@
- - - - - - - - - - - - - -
@@ -2097,16 +1603,6 @@
- - - - - - - - - -
@@ -2135,33 +1631,6 @@
- - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -2197,13 +1666,6 @@
- - - - - - -
@@ -2211,16 +1673,6 @@
- - - - - - - - - -
@@ -2264,40 +1716,6 @@
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -2367,11 +1785,6 @@
- - - - -
@@ -2497,16 +1910,6 @@
- - - - - - - - - -
@@ -2518,21 +1921,11 @@
- - - - -
- - - - -
@@ -2543,19 +1936,7 @@
- - - - - - - - - - - -
- +
@@ -2567,13 +1948,6 @@
- - - - - - -
@@ -2680,12 +2054,6 @@
- - - - - -
@@ -2698,10 +2066,6 @@
- - - -
@@ -2710,13 +2074,6 @@
- - - - - - -
@@ -2724,10 +2081,6 @@
- - - -
@@ -2791,46 +2144,24 @@
- - - - - -
- - - - - -
- - - - - -
- - - -
@@ -2885,13 +2216,6 @@
- - - - - - -
@@ -2899,26 +2223,14 @@
- - - -
- - - -
- - - -
@@ -2945,13 +2257,6 @@
- - - - - - -
@@ -2964,54 +2269,25 @@
- - - - -
- - - - -
- - - - - - - -
- - - -
- - - - - - -
@@ -3019,21 +2295,10 @@
- - - -
- - - - - - -
@@ -3041,23 +2306,11 @@
- - - - -
- - - - - - -
@@ -3073,73 +2326,35 @@
- - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - - - - - -
@@ -3177,17 +2392,6 @@
- - - - - - - - - - -
@@ -3199,19 +2403,6 @@
- - - - - - - - - - - - -
@@ -3229,42 +2420,21 @@
- - - - -
- - - - -
- - - - -
- - - - - -
@@ -3276,14 +2446,6 @@
- - - - - - - -
@@ -3309,34 +2471,17 @@
- - - - - -
- - - - -
- - - - - -
@@ -3350,22 +2495,6 @@
- - - - - - - - - - - - - - - -
@@ -3382,14 +2511,6 @@
- - - - - - - -
@@ -3398,85 +2519,12 @@
- - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -3544,15 +2592,6 @@
- - - - - - - - -
@@ -3562,10 +2601,6 @@
- - - -
@@ -3589,14 +2624,6 @@
- - - - - - - -
@@ -3609,10 +2636,6 @@
- - - -
@@ -3744,42 +2767,6 @@
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -3904,28 +2891,6 @@
- - - - - - - - - - - - - - - - - - - - - -
@@ -4009,10 +2974,6 @@
- - - -
@@ -4095,17 +3056,6 @@
- - - - - - - - - - -
@@ -4181,16 +3131,6 @@
- - - - - - - - - -
@@ -4213,16 +3153,6 @@
- - - - - - - - - -
@@ -4233,23 +3163,12 @@
- - - - - -
- - - - -
@@ -4271,41 +3190,21 @@
- - - - - -
- - - - -
- - - -
- - - - -
@@ -4353,12 +3252,6 @@
- - - - - -
@@ -4436,10 +3329,6 @@
- - - -
@@ -4642,14 +3531,6 @@
- - - - - - - -
@@ -4658,21 +3539,6 @@
- - - - - - - - - - - - - - -
@@ -4688,13 +3554,6 @@
- - - - - - -
@@ -4707,12 +3566,6 @@
- - - - - -
@@ -4724,12 +3577,6 @@
- - - - - -
@@ -4765,10 +3612,6 @@
- - - -
@@ -4799,10 +3642,6 @@
- - - -
@@ -4817,13 +3656,6 @@
- - - - - - -
@@ -4846,20 +3678,11 @@
- - - - -
- - - -
@@ -4895,20 +3718,10 @@
- - - -
- - - - - -
@@ -4967,15 +3780,6 @@
- - - - - - - - -
@@ -4990,11 +3794,6 @@
- - - - -
@@ -5008,33 +3807,16 @@
- - - - -
- - - - - -
- - - - -
- - - - - + + + +
@@ -5054,18 +3836,6 @@
- - - - - - - - - - - -
@@ -5078,30 +3848,16 @@
- - - - -
- - - - -
- - - -
@@ -5113,11 +3869,6 @@
- - - - -
@@ -5197,17 +3948,6 @@
- - - - - - - - - - -
@@ -5219,22 +3959,6 @@
- - - - - - - - - - - - - - - -
@@ -5251,16 +3975,6 @@
- - - - - - - - - -
@@ -5338,24 +4052,6 @@
- - - - - - - - - - - - - - - - - -
@@ -5480,14 +4176,6 @@
- - - - - - - -
@@ -5519,16 +4207,6 @@
- - - - - - - - - -
@@ -5539,18 +4217,6 @@
- - - - - - - - - - - -
@@ -5572,16 +4238,6 @@
- - - - - - - - - -
@@ -5592,29 +4248,12 @@
- - - - - -
- - - - - - - - - - -
@@ -5626,15 +4265,6 @@
- - - - - - - - -
@@ -5707,13 +4337,6 @@
- - - - - - -
@@ -5721,13 +4344,6 @@
- - - - - - -
@@ -5746,49 +4362,12 @@
- - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -5829,13 +4408,6 @@
- - - - - - -
@@ -5857,21 +4429,10 @@
- - - -
- - - - - - -
@@ -5879,28 +4440,6 @@
- - - - - - - - - - - - - - - - - - - - - -
@@ -5942,26 +4481,12 @@
- - - - - -
- - - - - - - -
@@ -5974,19 +4499,6 @@
- - - - - - - - - - - - -
@@ -6038,21 +4550,6 @@
- - - - - - - - - - - - - - -
@@ -6066,12 +4563,7 @@ - -
- - - - +
@@ -6083,35 +4575,17 @@
- - - - -
- - - - - -
- - - - - - -
@@ -6119,14 +4593,6 @@
- - - - - - - -
@@ -6137,12 +4603,6 @@
- - - - - -
@@ -6155,14 +4615,6 @@
- - - - - - - -
@@ -6171,13 +4623,6 @@
- - - - - - -
@@ -6185,19 +4630,6 @@
- - - - - - - - - - - - -
@@ -6216,16 +4648,6 @@
- - - - - - - - - -
@@ -6236,12 +4658,6 @@
- - - - - -
@@ -6379,20 +4795,6 @@
- - - - - - - - - - - - - -
@@ -6432,10 +4834,6 @@
- - - -
@@ -6456,21 +4854,6 @@
- - - - - - - - - - - - - - -
@@ -6486,29 +4869,12 @@
- - - - - -
- - - - - - - - - - -
@@ -6556,12 +4922,6 @@
- - - - - -
@@ -6596,26 +4956,10 @@
- - - -
- - - - - - - - - - - -
@@ -6628,32 +4972,16 @@
- - - -
- - - - - -
- - - - - -
@@ -6673,11 +5001,6 @@
- - - - -
@@ -6828,25 +5151,6 @@
- - - - - - - - - - - - - - - - - - -
@@ -6887,70 +5191,6 @@
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -7141,22 +5381,6 @@
- - - - - - - - - - - - - - - -
@@ -7204,15 +5428,6 @@
- - - - - - - - -
@@ -7228,25 +5443,6 @@
- - - - - - - - - - - - - - - - - - -
@@ -7296,19 +5492,6 @@
- - - - - - - - - - - - -
@@ -7323,13 +5506,6 @@
- - - - - - -
From 01a3e5cd27c8a0214117347d5488c109ef41a69f Mon Sep 17 00:00:00 2001 From: Barncastle Date: Sat, 3 Mar 2018 17:22:26 +0000 Subject: [PATCH 05/11] performance tweaks --- WDBXEditor.sln | 10 +++++++-- WDBXEditor/Reader/BitStream.cs | 35 +++++++++++++++-------------- WDBXEditor/Reader/FileTypes/WDB5.cs | 2 +- WDBXEditor/Reader/FileTypes/WDC1.cs | 8 +++---- 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/WDBXEditor.sln b/WDBXEditor.sln index 979cebf..60b05f8 100644 --- a/WDBXEditor.sln +++ b/WDBXEditor.sln @@ -1,13 +1,16 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 14 -VisualStudioVersion = 14.0.24720.0 +# Visual Studio 15 +VisualStudioVersion = 15.0.27130.2036 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WDBXEditor", "WDBXEditor\WDBXEditor.csproj", "{E1F42B5C-7A40-4539-AD92-E24CAD7041F5}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AdvancedDataGridView", "AdvancedDataGridView\AdvancedDataGridView.csproj", "{6EBA0A55-B390-4479-A564-58D46094998D}" EndProject Global + GlobalSection(Performance) = preSolution + HasPerformanceSessions = true + EndGlobalSection GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Release|Any CPU = Release|Any CPU @@ -25,4 +28,7 @@ Global GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {645EB3D9-4619-4DF0-BDC5-9A0B95065969} + EndGlobalSection EndGlobal diff --git a/WDBXEditor/Reader/BitStream.cs b/WDBXEditor/Reader/BitStream.cs index 2c365c5..044c458 100644 --- a/WDBXEditor/Reader/BitStream.cs +++ b/WDBXEditor/Reader/BitStream.cs @@ -19,7 +19,6 @@ public class BitStream : IDisposable public long Length => stream.Length; public long BitPosition => bit; public long Offset => offset; - private bool ValidPosition => offset < Length; public BitStream(int capacity = 0) @@ -146,7 +145,7 @@ private void UpdateCurrentByte() private Bit ReadBit() { - if (!ValidPosition) + if (offset >= stream.Length) throw new IOException("Cannot read in an offset bigger than the length of the stream"); byte value = (byte)((currentByte >> (bit)) & 1); @@ -160,7 +159,7 @@ private void WriteBit(Bit data) currentByte &= (byte)~(1 << bit); currentByte |= (byte)(data << bit); - if (!ValidPosition) + if (offset >= stream.Length) { if (!canWrite || !ChangeLength(Length + (offset - Length) + 1)) throw new IOException("Attempted to write past the length of the stream."); @@ -272,11 +271,8 @@ private int NextPow2(int v) #region Write - public void WriteBytes(byte[] data, long length, bool isBytes = false) + public void WriteBits(byte[] data, long length) { - if (isBytes) - length *= 8; - int position = 0; for (long i = 0; i < length;) { @@ -290,65 +286,70 @@ public void WriteBytes(byte[] data, long length, bool isBytes = false) } } + public void WriteBytes(byte[] data, long length) + { + WriteBits(data, length * 8); + } + public void WriteByte(byte value, int bits = 8) { bits = Math.Min(Math.Max(bits, 0), 8); // clamp values - WriteBytes(new byte[] { value }, bits); + WriteBits(new byte[] { value }, bits); } public void WriteChar(char value) { byte[] bytes = encoding.GetBytes(new char[] { value }, 0, 1); - WriteBytes(bytes, bytes.Length * 8); + WriteBits(bytes, bytes.Length * 8); } public void WriteString(string value) { byte[] bytes = encoding.GetBytes(value); - WriteBytes(bytes, bytes.Length * 8); + WriteBits(bytes, bytes.Length * 8); } public void WriteCString(string value) { byte[] bytes = encoding.GetBytes(value); Array.Resize(ref bytes, bytes.Length + 1); - WriteBytes(bytes, bytes.Length * 8); + WriteBits(bytes, bytes.Length * 8); } public void WriteInt16(short value, int bits = 16) { bits = Math.Min(Math.Max(bits, 0), 16); // clamp values - WriteBytes(BitConverter.GetBytes(value), bits); + WriteBits(BitConverter.GetBytes(value), bits); } public void WriteInt32(int value, int bits = 32) { bits = Math.Min(Math.Max(bits, 0), 32); // clamp values - WriteBytes(BitConverter.GetBytes(value), bits); + WriteBits(BitConverter.GetBytes(value), bits); } public void WriteInt64(long value, int bits = 64) { bits = Math.Min(Math.Max(bits, 0), 64); // clamp values - WriteBytes(BitConverter.GetBytes(value), bits); + WriteBits(BitConverter.GetBytes(value), bits); } public void WriteUInt16(ushort value, int bits = 16) { bits = Math.Min(Math.Max(bits, 0), 16); // clamp values - WriteBytes(BitConverter.GetBytes(value), bits); + WriteBits(BitConverter.GetBytes(value), bits); } public void WriteUInt32(uint value, int bits = 32) { bits = Math.Min(Math.Max(bits, 0), 32); // clamp values - WriteBytes(BitConverter.GetBytes(value), bits); + WriteBits(BitConverter.GetBytes(value), bits); } public void WriteUInt64(ulong value, int bits = 64) { bits = Math.Min(Math.Max(bits, 0), 64); // clamp values - WriteBytes(BitConverter.GetBytes(value), bits); + WriteBits(BitConverter.GetBytes(value), bits); } #endregion diff --git a/WDBXEditor/Reader/FileTypes/WDB5.cs b/WDBXEditor/Reader/FileTypes/WDB5.cs index 11fbc2b..85eb71e 100644 --- a/WDBXEditor/Reader/FileTypes/WDB5.cs +++ b/WDBXEditor/Reader/FileTypes/WDB5.cs @@ -232,7 +232,7 @@ public override void WriteHeader(BinaryWriter bw, DBEntry entry) public override void WriteOffsetMap(BinaryWriter bw, DBEntry entry, List> OffsetMap) { var minmax = entry.MinMax(); - var ids = entry.GetPrimaryKeys().ToList(); + var ids = new HashSet(entry.GetPrimaryKeys()); var duplicates = entry.Header.OffsetDuplicates; int m = 0; diff --git a/WDBXEditor/Reader/FileTypes/WDC1.cs b/WDBXEditor/Reader/FileTypes/WDC1.cs index 2fc4fd9..ce0b1b2 100644 --- a/WDBXEditor/Reader/FileTypes/WDC1.cs +++ b/WDBXEditor/Reader/FileTypes/WDC1.cs @@ -554,11 +554,11 @@ public void WriteData(BinaryWriter bw, DBEntry entry) { case CompressionType.None: for (int i = 0; i < arraySize; i++) - bitStream.WriteBytes(data[i], bitSize); + bitStream.WriteBits(data[i], bitSize); break; case CompressionType.Immediate: - bitStream.WriteBytes(data[0], bitWidth); + bitStream.WriteBits(data[0], bitWidth); break; case CompressionType.Sparse: @@ -599,7 +599,7 @@ public void WriteData(BinaryWriter bw, DBEntry entry) short size = (short)(bitStream.Offset - offset + bw.BaseStream.Position); int remaining = size % 4 == 0 ? 0 : 4 - size % 4; if (remaining != 0) - bitStream.WriteBytes(new byte[remaining], remaining, true); + bitStream.WriteBytes(new byte[remaining], remaining); offsetMap.Add(new Tuple((int)offset, (short)(bw.BaseStream.Position + bitStream.Offset - offset))); } @@ -608,7 +608,7 @@ public void WriteData(BinaryWriter bw, DBEntry entry) bitStream.SeekNextOffset(); short size = (short)(bitStream.Offset - offset + bw.BaseStream.Position); if (size < RecordSize) - bitStream.WriteBytes(new byte[RecordSize - size], RecordSize - size, true); + bitStream.WriteBytes(new byte[RecordSize - size], RecordSize - size); } } bitStream.CopyStreamTo(bw.BaseStream); // write to the filestream From 6ebf1bbc249809bd5c3c7f1356414147156ddc1f Mon Sep 17 00:00:00 2001 From: Barncastle Date: Sat, 3 Mar 2018 18:44:25 +0000 Subject: [PATCH 06/11] performance tweaks --- AdvancedDataGridView/DataCache.cs | 554 +++++++++++++++--------------- WDBXEditor/Storage/DBEntry.cs | 2 +- 2 files changed, 278 insertions(+), 278 deletions(-) diff --git a/AdvancedDataGridView/DataCache.cs b/AdvancedDataGridView/DataCache.cs index 38fa6b3..6a79eb3 100644 --- a/AdvancedDataGridView/DataCache.cs +++ b/AdvancedDataGridView/DataCache.cs @@ -13,296 +13,296 @@ namespace ADGV { - partial class AdvancedDataGridView : DataGridView - { - public DataColumn PrimaryKey { get; private set; } + partial class AdvancedDataGridView : DataGridView + { + public DataColumn PrimaryKey { get; private set; } - private ConcurrentDictionary Cache = new ConcurrentDictionary(); - private int[] DataCount; - private string _tag = string.Empty; + private ConcurrentDictionary Cache = new ConcurrentDictionary(); + private int[] DataCount; + private string _tag = string.Empty; - public void Init(bool force = false) - { - if (!force && _tag == this.Parent.Tag + "") - return; + public void Init(bool force = false) + { + if (!force && _tag == this.Parent.Tag + "") + return; - _tag = this.Parent.Tag + ""; - Cache.Clear(); - DataCount = new int[0]; + _tag = this.Parent.Tag + ""; + Cache.Clear(); + DataCount = new int[0]; - var table = (DataTable)((BindingSource)this.DataSource).DataSource; - if ((table?.Rows.Count ?? 0) == 0) - return; + var table = (DataTable)((BindingSource)this.DataSource).DataSource; + if ((table?.Rows.Count ?? 0) == 0) + return; - PrimaryKey = table.PrimaryKey[0]; //Store the primary key + PrimaryKey = table.PrimaryKey[0]; //Store the primary key #if DEBUG - Stopwatch sw = new Stopwatch(); - sw.Start(); + Stopwatch sw = new Stopwatch(); + sw.Start(); #endif - DataCount = new int[table.Columns.Count]; + DataCount = new int[table.Columns.Count]; - Parallel.ForEach(table.Rows.Cast(), row => - { - if ((row?.ItemArray.Length ?? 0) == 0) - return; + Parallel.ForEach(table.Rows.Cast(), row => + { + if ((row?.ItemArray.Length ?? 0) == 0) + return; - if (row.RowState == DataRowState.Deleted || row.RowState == DataRowState.Detached) - return; + if (row.RowState == DataRowState.Deleted || row.RowState == DataRowState.Detached) + return; - if (row[PrimaryKey.ColumnName] == DBNull.Value) - return; + if (row[PrimaryKey.ColumnName] == DBNull.Value) + return; - int key = (int)row[PrimaryKey.ColumnName]; - var data = row.ItemArray; - Cache.TryAdd(key, SerializeObject(data)); //Store the JSON variant - GetColumnDataCount(data); //Update the empty column count - }); + int key = (int)row[PrimaryKey.ColumnName]; + var data = row.ItemArray; + Cache.TryAdd(key, SerializeObject(data)); //Store the JSON variant + GetColumnDataCount(data); //Update the empty column count + }); #if DEBUG - sw.Stop(); - Debug.WriteLine(sw.ElapsedMilliseconds); + sw.Stop(); + Debug.WriteLine(sw.ElapsedMilliseconds); #endif - } - - #region Cache Changes - - public void ChangeValue(DataRow row) - { - int key = (int)row[PrimaryKey.ColumnName]; - var data = row.ItemArray.Select(x => x + "").ToArray(); - - if (Cache.ContainsKey(key)) - { - UpdateColumnDataCount(key, data); - Cache[key] = SerializeObject(data); - } - else - AddRow(row); - } - - public void AddRow(DataRow row) - { - int key = (int)(row[PrimaryKey.ColumnName] ?? -1); - if (key == -1) return; - - var data = row.ItemArray; - Cache.TryAdd(key, SerializeObject(data)); - GetColumnDataCount(data); - } - - public void RemoveRow(DataRow row) - { - int key = (int)row[PrimaryKey.ColumnName]; - - if (Cache.ContainsKey(key)) - { - string _dump; - if (Cache.TryRemove(key, out _dump)) - { - string[] vals = DeserializeObject(_dump); - for (int i = 0; i < vals.Length; i++) - if (vals[i].Length > 0) - DataCount[i]--; - } - } - } - - #endregion - - #region Column Value Count - - public IEnumerable GetEmptyColumns() - { - for (int i = 0; i < DataCount.Length; i++) - { - if (DataCount[i] <= 0) - { - DataCount[i] = 0; - yield return i; - } - } - } - - private void GetColumnDataCount(object[] data) - { - for (int x = 0; x < data.Length; x++) - if ((data[x] + "").Length > 0) - DataCount[x]++; - } - - private void UpdateColumnDataCount(int key, string[] data) - { - string[] prev = DeserializeObject(Cache[key]); - for (int i = 0; i < data.Length; i++) - { - if (prev[i].Length > 0 && data[i].Length == 0) - DataCount[i]--; - else if (prev[i].Length == 0 && data[i].Length > 0) - DataCount[i]++; - } - } - - #endregion - - #region Search - public Point Search(string text, bool exact, StringComparison comparison = StringComparison.CurrentCultureIgnoreCase, bool includestart = false) - { - if (this.CurrentCell == null) - this.SetSelectedCellCore(0, 0, true); - - var startcell = this.CurrentCell; //Our original - var startindex = startcell.RowIndex; - var startcolumn = startcell.ColumnIndex; - bool looped = false; - var hidden = this.Columns.Cast().Where(x => !x.Visible).Select(x => x.Index).ToList(); - - BindingSource bs = (BindingSource)this.DataSource; - - //Get actual match - FinalSearch: - for (int i = startindex; i < this.Rows.Count - 1; i++) - { - var pk = (int)((DataRowView)bs[i]).Row.ItemArray[PrimaryKey.Ordinal]; //Get the Id value - if (startcolumn > 0 && i != startindex) - startcolumn = 0; - - if (!Cache.ContainsKey(pk)) //Ignore non-cached - shouldn't happen - continue; - if (Cache[pk].IndexOf(text, comparison) == -1) //Check the JSON haystack contains the needle - continue; - - var data = DeserializeObject(Cache[pk]); - for (int x = startcolumn; x < data.Length; x++) - { - //Don't search hidden columns - if (hidden.Contains(x)) - continue; - - //Completed a full loop - if (i == startcell.RowIndex && x == startcell.ColumnIndex && looped) - return new Point(-1, -1); - - //Ignore start cell check - if (i == startcell.RowIndex && x == startcell.ColumnIndex && !includestart && !looped) - continue; - - if (exact && data[x].Equals(text, comparison)) - return new Point(i, x); - else if (!exact && data[x].IndexOf(text, comparison) >= 0) - return new Point(i, x); - } - } - - //Restart from the beginning - if (!looped && startcell.RowIndex > 0) - { - startindex = 0; - looped = true; - goto FinalSearch; - } - - return new Point(-1, -1); //No matches - } - - public Point SearchFlag(long flag, bool includestart = false, ICollection ignore = null) - { - if (this.CurrentCell == null) - this.SetSelectedCellCore(0, 0, true); - - var startcell = this.CurrentCell; //Our original - var startindex = startcell.RowIndex; - var startcolumn = startcell.ColumnIndex; - bool looped = false; - var hidden = this.Columns.Cast().Where(x => !x.Visible).Select(x => x.Index).ToList(); - - BindingSource bs = (BindingSource)this.DataSource; - - //Get actual match - FinalSearch: - for (int i = startindex; i < this.Rows.Count - 1; i++) - { - var pk = (int)((DataRowView)bs[i]).Row.ItemArray[PrimaryKey.Ordinal]; //Get the Id value - - if (!Cache.ContainsKey(pk)) //Ignore non-cached - shouldn't happen - continue; - - var data = DeserializeObject(Cache[pk]); - //Don't search hidden columns - if (hidden.Contains(startcolumn)) - return new Point(-1, -1); //No matches - - //Completed a full loop - if (i >= startcell.RowIndex && looped) - return new Point(-1, -1); - - //Ignore start cell check - if (i == startcell.RowIndex && !includestart && !looped) - continue; - - - if (long.TryParse(data[startcolumn], out long value) && (value & flag) == flag) - { - Point result = new Point(i, startcolumn); - if (ignore == null || !ignore.Contains(result)) - return result; - } - } - - //Restart from the beginning - if (!looped && startcell.RowIndex > 0) - { - startindex = 0; - looped = true; - goto FinalSearch; - } - - return new Point(-1, -1); //No matches - } - #endregion - - - #region Serialization - private string SerializeObject(Array array) - { - using (MemoryStream ms = new MemoryStream()) - { - foreach (var obj in array) - { - var b = Encoding.UTF8.GetBytes(obj + ""); - var s = BitConverter.GetBytes((ushort)b.Length); - - ms.Write(s, 0, s.Length); - ms.Write(b, 0, b.Length); - } - - return Encoding.UTF8.GetString(ms.ToArray()); - } - } - - private string[] DeserializeObject(string value) - { - var bytes = Encoding.UTF8.GetBytes(value); - string[] _output = new string[DataCount.Length]; - - using (MemoryStream ms = new MemoryStream(bytes)) - { - byte[] s = new byte[sizeof(ushort)]; - byte[] b; - - for (int i = 0; i < _output.Length; i++) - { - ms.Read(s, 0, s.Length); - int size = BitConverter.ToUInt16(s, 0); - - b = new byte[size]; - ms.Read(b, 0, b.Length); - _output[i] = Encoding.UTF8.GetString(b); - } - - return _output; - } - } - #endregion - } + } + + #region Cache Changes + + public void ChangeValue(DataRow row) + { + int key = (int)row[PrimaryKey.ColumnName]; + var data = row.ItemArray.Select(x => x + "").ToArray(); + + if (Cache.ContainsKey(key)) + { + UpdateColumnDataCount(key, data); + Cache[key] = SerializeObject(data); + } + else + AddRow(row); + } + + public void AddRow(DataRow row) + { + int key = (int)(row[PrimaryKey.ColumnName] ?? -1); + if (key == -1) return; + + var data = row.ItemArray; + Cache.TryAdd(key, SerializeObject(data)); + GetColumnDataCount(data); + } + + public void RemoveRow(DataRow row) + { + int key = (int)row[PrimaryKey.ColumnName]; + + if (Cache.ContainsKey(key)) + { + string _dump; + if (Cache.TryRemove(key, out _dump)) + { + string[] vals = DeserializeObject(_dump); + for (int i = 0; i < vals.Length; i++) + if (vals[i].Length > 0) + DataCount[i]--; + } + } + } + + #endregion + + #region Column Value Count + + public IEnumerable GetEmptyColumns() + { + for (int i = 0; i < DataCount.Length; i++) + { + if (DataCount[i] <= 0) + { + DataCount[i] = 0; + yield return i; + } + } + } + + private void GetColumnDataCount(object[] data) + { + for (int x = 0; x < data.Length; x++) + if ((data[x] + "").Length > 0) + DataCount[x]++; + } + + private void UpdateColumnDataCount(int key, string[] data) + { + string[] prev = DeserializeObject(Cache[key]); + for (int i = 0; i < data.Length; i++) + { + if (prev[i].Length > 0 && data[i].Length == 0) + DataCount[i]--; + else if (prev[i].Length == 0 && data[i].Length > 0) + DataCount[i]++; + } + } + + #endregion + + #region Search + public Point Search(string text, bool exact, StringComparison comparison = StringComparison.CurrentCultureIgnoreCase, bool includestart = false) + { + if (this.CurrentCell == null) + this.SetSelectedCellCore(0, 0, true); + + var startcell = this.CurrentCell; //Our original + var startindex = startcell.RowIndex; + var startcolumn = startcell.ColumnIndex; + bool looped = false; + var hidden = new HashSet(this.Columns.Cast().Where(x => !x.Visible).Select(x => x.Index)); + + BindingSource bs = (BindingSource)this.DataSource; + + //Get actual match + FinalSearch: + for (int i = startindex; i < this.Rows.Count - 1; i++) + { + var pk = (int)((DataRowView)bs[i]).Row.ItemArray[PrimaryKey.Ordinal]; //Get the Id value + if (startcolumn > 0 && i != startindex) + startcolumn = 0; + + if (!Cache.ContainsKey(pk)) //Ignore non-cached - shouldn't happen + continue; + if (Cache[pk].IndexOf(text, comparison) == -1) //Check the JSON haystack contains the needle + continue; + + var data = DeserializeObject(Cache[pk]); + for (int x = startcolumn; x < data.Length; x++) + { + //Don't search hidden columns + if (hidden.Contains(x)) + continue; + + //Completed a full loop + if (i == startcell.RowIndex && x == startcell.ColumnIndex && looped) + return new Point(-1, -1); + + //Ignore start cell check + if (i == startcell.RowIndex && x == startcell.ColumnIndex && !includestart && !looped) + continue; + + if (exact && data[x].Equals(text, comparison)) + return new Point(i, x); + else if (!exact && data[x].IndexOf(text, comparison) >= 0) + return new Point(i, x); + } + } + + //Restart from the beginning + if (!looped && startcell.RowIndex > 0) + { + startindex = 0; + looped = true; + goto FinalSearch; + } + + return new Point(-1, -1); //No matches + } + + public Point SearchFlag(long flag, bool includestart = false, ICollection ignore = null) + { + if (this.CurrentCell == null) + this.SetSelectedCellCore(0, 0, true); + + var startcell = this.CurrentCell; //Our original + var startindex = startcell.RowIndex; + var startcolumn = startcell.ColumnIndex; + bool looped = false; + var hidden = new HashSet(this.Columns.Cast().Where(x => !x.Visible).Select(x => x.Index)); + + BindingSource bs = (BindingSource)this.DataSource; + + //Get actual match + FinalSearch: + for (int i = startindex; i < this.Rows.Count - 1; i++) + { + var pk = (int)((DataRowView)bs[i]).Row.ItemArray[PrimaryKey.Ordinal]; //Get the Id value + + if (!Cache.ContainsKey(pk)) //Ignore non-cached - shouldn't happen + continue; + + var data = DeserializeObject(Cache[pk]); + //Don't search hidden columns + if (hidden.Contains(startcolumn)) + return new Point(-1, -1); //No matches + + //Completed a full loop + if (i >= startcell.RowIndex && looped) + return new Point(-1, -1); + + //Ignore start cell check + if (i == startcell.RowIndex && !includestart && !looped) + continue; + + + if (long.TryParse(data[startcolumn], out long value) && (value & flag) == flag) + { + Point result = new Point(i, startcolumn); + if (ignore == null || !ignore.Contains(result)) + return result; + } + } + + //Restart from the beginning + if (!looped && startcell.RowIndex > 0) + { + startindex = 0; + looped = true; + goto FinalSearch; + } + + return new Point(-1, -1); //No matches + } + #endregion + + + #region Serialization + private string SerializeObject(Array array) + { + using (MemoryStream ms = new MemoryStream()) + { + foreach (var obj in array) + { + var b = Encoding.UTF8.GetBytes(obj + ""); + var s = BitConverter.GetBytes((ushort)b.Length); + + ms.Write(s, 0, s.Length); + ms.Write(b, 0, b.Length); + } + + return Encoding.UTF8.GetString(ms.ToArray()); + } + } + + private string[] DeserializeObject(string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + string[] _output = new string[DataCount.Length]; + + using (MemoryStream ms = new MemoryStream(bytes)) + { + byte[] s = new byte[sizeof(ushort)]; + byte[] b; + + for (int i = 0; i < _output.Length; i++) + { + ms.Read(s, 0, s.Length); + int size = BitConverter.ToUInt16(s, 0); + + b = new byte[size]; + ms.Read(b, 0, b.Length); + _output[i] = Encoding.UTF8.GetString(b); + } + + return _output; + } + } + #endregion + } } diff --git a/WDBXEditor/Storage/DBEntry.cs b/WDBXEditor/Storage/DBEntry.cs index e2afb73..6bb238e 100644 --- a/WDBXEditor/Storage/DBEntry.cs +++ b/WDBXEditor/Storage/DBEntry.cs @@ -643,7 +643,7 @@ public bool ImportCSV(string filename, bool headerrow, UpdateMode mode, out stri DataTable importTable = Data.Clone(); //Clone table structure to help with mapping - List usedids = new List(); + HashSet usedids = new HashSet(); int idcolumn = Data.Columns[Key].Ordinal; int maxid = int.MinValue; From 80e81c30eb3e2d769f6521022d6fbc81602b4da3 Mon Sep 17 00:00:00 2001 From: Barncastle Date: Sun, 4 Mar 2018 13:38:27 +0000 Subject: [PATCH 07/11] tweaks + fixes --- AdvancedDataGridView/AdvancedDataGridView.cs | 2 +- AdvancedDataGridView/DataCache.cs | 2 +- .../Definitions/Legion 7.3.5 (25632).xml | 6 +- WDBXEditor/Reader/FileTypes/WDC1.cs | 39 ++-- WDBXEditor/Reader/StringTable.cs | 173 +++++++++--------- 5 files changed, 117 insertions(+), 105 deletions(-) diff --git a/AdvancedDataGridView/AdvancedDataGridView.cs b/AdvancedDataGridView/AdvancedDataGridView.cs index 88f91d6..cf5b19d 100644 --- a/AdvancedDataGridView/AdvancedDataGridView.cs +++ b/AdvancedDataGridView/AdvancedDataGridView.cs @@ -395,7 +395,7 @@ public void PasteCopyData(DataRow row) public void ClearCopyData() { - _copydata = new object[0]; + Array.Resize(ref _copydata, 0); } #endregion diff --git a/AdvancedDataGridView/DataCache.cs b/AdvancedDataGridView/DataCache.cs index 6a79eb3..523d121 100644 --- a/AdvancedDataGridView/DataCache.cs +++ b/AdvancedDataGridView/DataCache.cs @@ -28,7 +28,7 @@ public void Init(bool force = false) _tag = this.Parent.Tag + ""; Cache.Clear(); - DataCount = new int[0]; + Array.Resize(ref DataCount, 0); var table = (DataTable)((BindingSource)this.DataSource).DataSource; if ((table?.Rows.Count ?? 0) == 0) diff --git a/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml b/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml index 10c5f26..c48912f 100644 --- a/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml +++ b/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml @@ -1419,7 +1419,7 @@
- + @@ -2527,7 +2527,7 @@
- + @@ -3977,7 +3977,7 @@
- + diff --git a/WDBXEditor/Reader/FileTypes/WDC1.cs b/WDBXEditor/Reader/FileTypes/WDC1.cs index ce0b1b2..f72644e 100644 --- a/WDBXEditor/Reader/FileTypes/WDC1.cs +++ b/WDBXEditor/Reader/FileTypes/WDC1.cs @@ -88,9 +88,14 @@ public override void ReadHeader(ref BinaryReader dbReader, string signature) if (CopyTableSize == 0) { if (!firstindex.ContainsKey(offset)) + { firstindex.Add(offset, new OffsetDuplicate(offsetmap.Count, firstindex.Count)); + } else + { OffsetDuplicates.Add(MinId + i, firstindex[offset].VisibleIndex); + continue; + } } offsetmap.Add(new Tuple(offset, length)); @@ -205,6 +210,9 @@ public override void ReadHeader(ref BinaryReader dbReader, string signature) id = m_indexes[CopyTable.Count]; var map = offsetmap[i]; + if (CopyTableSize == 0 && firstindex[map.Item1].HiddenIndex != i) //Ignore duplicates + continue; + dbReader.BaseStream.Position = map.Item1; byte[] data = dbReader.ReadBytes(map.Item2); @@ -328,13 +336,12 @@ public override void ReadHeader(ref BinaryReader dbReader, string signature) FieldStructure.Insert(0, new FieldStructureEntry(0, 0)); ColumnMeta.Insert(0, new ColumnStructureEntry()); } - + offsetmap.Clear(); firstindex.Clear(); OffsetDuplicates.Clear(); Copies.Clear(); - recordData = new byte[0]; - recordData = null; + Array.Resize(ref recordData, 0); bitStream.Dispose(); ColumnMeta.ForEach(x => { x.PalletValues?.Clear(); x.SparseValues?.Clear(); }); @@ -436,7 +443,7 @@ public override void WriteHeader(BinaryWriter bw, DBEntry entry) bw.Write(0); // PalletDataSize bw.Write(0); // RelationshipDataSize - //Write the field_structure bits + // Write the field_structure bits for (int i = 0; i < FieldStructure.Count; i++) { if (HasIndexTable && i == 0) continue; @@ -536,7 +543,7 @@ public void WriteData(BinaryWriter bw, DBEntry entry) bitStream.SeekNextOffset(); // each row starts at a 0 bit position - long offset = bw.BaseStream.Position + bitStream.Offset; // used for offset map calcs + long offset = pos + bitStream.Offset; // used for offset map calcs for (int fieldIndex = 0; fieldIndex < FieldCount; fieldIndex++) { @@ -593,22 +600,26 @@ public void WriteData(BinaryWriter bw, DBEntry entry) } } - if (IsSparse) // needs to be padded to % 4 and offsetmap record needs to be created. this isn't true but doesn't matter + short size = (short)(pos + bitStream.Offset - offset); + + if (IsSparse) // matches itemsparse padding { bitStream.SeekNextOffset(); - short size = (short)(bitStream.Offset - offset + bw.BaseStream.Position); - int remaining = size % 4 == 0 ? 0 : 4 - size % 4; - if (remaining != 0) + + int remaining = size % 8 == 0 ? 0 : 8 - (size % 8); + if (remaining > 0) + { + size += (short)remaining; bitStream.WriteBytes(new byte[remaining], remaining); + } - offsetMap.Add(new Tuple((int)offset, (short)(bw.BaseStream.Position + bitStream.Offset - offset))); + offsetMap.Add(new Tuple((int)offset, size)); } else // needs to be padded to the record size regardless of the byte count - weird eh? { bitStream.SeekNextOffset(); - short size = (short)(bitStream.Offset - offset + bw.BaseStream.Position); - if (size < RecordSize) - bitStream.WriteBytes(new byte[RecordSize - size], RecordSize - size); + //if (size < RecordSize) + // bitStream.WriteBytes(new byte[RecordSize - size], RecordSize - size); } } bitStream.CopyStreamTo(bw.BaseStream); // write to the filestream @@ -759,7 +770,7 @@ private object[] ExtractFields(Queue rowData, StringTable stringTable, B else { for (int i = 0; i < values.Length; i++) - values[i] = stringTable.Write((string)values[i]); + values[i] = stringTable.Write((string)values[i], false, false); } } diff --git a/WDBXEditor/Reader/StringTable.cs b/WDBXEditor/Reader/StringTable.cs index 682da20..63d88a0 100644 --- a/WDBXEditor/Reader/StringTable.cs +++ b/WDBXEditor/Reader/StringTable.cs @@ -7,90 +7,91 @@ namespace WDBXEditor.Reader { - public class StringTable : IDisposable - { - public Dictionary Table = new Dictionary(); - public int Size => (int)StringStream.Length; - - public MemoryStream StringStream = new MemoryStream(); - private Dictionary _stringlookup = new Dictionary(); - - public StringTable() { } - - public StringTable(bool extended) - { - StringStream.WriteByte(0); - if (extended) - StringStream.WriteByte(0); - } - - /// - /// Writes the string into the stringtable and return's it's position. - /// If unqiue then it will check if the string is existent and return the previous position. - /// - /// - /// - /// - public int Write(string str, bool duplicates = false) - { - str = str.Replace("\r\n", "\n").Replace(Environment.NewLine, "\n"); - - int offset = 0; - if (str == "") //Empty string always 0 - return offset; - - //WDB2 with MaxId allows duplicates else distinct strings - if (duplicates || !_stringlookup.TryGetValue(str, out offset)) - { - byte[] strBytes = Encoding.UTF8.GetBytes(str); - offset = (int)StringStream.Position; - - if (!duplicates) - _stringlookup.Add(str, offset); - - StringStream.Write(strBytes, 0, strBytes.Length); - StringStream.WriteByte(0); - } - - return offset; - } - - public void CopyTo(Stream s) - { - StringStream.Position = 0; - StringStream.CopyTo(s); - } - - - /// - /// Reads the binary data from start to end returning a list of strings and their positions. - /// - /// - /// - /// - /// - public Dictionary Read(BinaryReader dbReader, long stringTableStart, long stringTableEnd) - { - if (dbReader.BaseStream.Position > stringTableEnd) - return Table; - - while (dbReader.BaseStream.Position < stringTableEnd) - { - int index = (int)(dbReader.BaseStream.Position - stringTableStart); - Table[index] = dbReader.ReadStringNull(); //Extract all the strings to the string table - } - - return Table; - } - - public Dictionary Read(BinaryReader dbReader, long stringTableStart) - { - return Read(dbReader, stringTableStart, dbReader.BaseStream.Length); - } - - public void Dispose() - { - ((IDisposable)StringStream).Dispose(); - } - } + public class StringTable : IDisposable + { + public Dictionary Table = new Dictionary(); + public int Size => (int)StringStream.Length; + + public MemoryStream StringStream = new MemoryStream(); + private Dictionary _stringlookup = new Dictionary(); + + public StringTable() { } + + public StringTable(bool extended) + { + StringStream.WriteByte(0); + if (extended) + StringStream.WriteByte(0); + } + + /// + /// Writes the string into the stringtable and return's it's position. + /// If unique then it will check if the string is existent and return the previous position. + /// + /// + /// + /// + public int Write(string str, bool duplicates = false, bool stripCR = true) + { + if (stripCR) + str = str.Replace("\r\n", "\n").Replace(Environment.NewLine, "\n"); + + int offset = 0; + if (str == "") //Empty string always 0 + return offset; + + //WDB2 with MaxId allows duplicates else distinct strings + if (duplicates || !_stringlookup.TryGetValue(str, out offset)) + { + byte[] strBytes = Encoding.UTF8.GetBytes(str); + offset = (int)StringStream.Position; + + if (!duplicates) + _stringlookup.Add(str, offset); + + StringStream.Write(strBytes, 0, strBytes.Length); + StringStream.WriteByte(0); + } + + return offset; + } + + public void CopyTo(Stream s) + { + StringStream.Position = 0; + StringStream.CopyTo(s); + } + + + /// + /// Reads the binary data from start to end returning a list of strings and their positions. + /// + /// + /// + /// + /// + public Dictionary Read(BinaryReader dbReader, long stringTableStart, long stringTableEnd) + { + if (dbReader.BaseStream.Position > stringTableEnd) + return Table; + + while (dbReader.BaseStream.Position < stringTableEnd) + { + int index = (int)(dbReader.BaseStream.Position - stringTableStart); + Table[index] = dbReader.ReadStringNull(); //Extract all the strings to the string table + } + + return Table; + } + + public Dictionary Read(BinaryReader dbReader, long stringTableStart) + { + return Read(dbReader, stringTableStart, dbReader.BaseStream.Length); + } + + public void Dispose() + { + ((IDisposable)StringStream).Dispose(); + } + } } From 652d996cca44c5a5948d7536507242dee9eca817 Mon Sep 17 00:00:00 2001 From: Barncastle Date: Sun, 4 Mar 2018 17:17:45 +0000 Subject: [PATCH 08/11] extra WDC1 field value validation --- AdvancedDataGridView/AdvancedDataGridView.cs | 22 ++++++++++ WDBXEditor/Main.cs | 10 +++-- WDBXEditor/Reader/FileTypes/WDC1.cs | 20 +++++++++ WDBXEditor/Storage/DBEntry.cs | 43 ++++++++++++++++++++ 4 files changed, 92 insertions(+), 3 deletions(-) diff --git a/AdvancedDataGridView/AdvancedDataGridView.cs b/AdvancedDataGridView/AdvancedDataGridView.cs index cf5b19d..4a85603 100644 --- a/AdvancedDataGridView/AdvancedDataGridView.cs +++ b/AdvancedDataGridView/AdvancedDataGridView.cs @@ -187,6 +187,28 @@ protected override void OnCellValidating(DataGridViewCellValidatingEventArgs e) base.OnCellValidating(e); } + public bool ValidValue(int index, object value) + { + if (BitCounts.ContainsKey(index)) + { + var bitcount = BitCounts[index]; + if (bitcount.IsSingle && float.TryParse(value.ToString(), out float fVal)) + { + return fVal >= (float)bitcount.MinVal && fVal <= (float)bitcount.MaxVal; + } + if (bitcount.Signed && long.TryParse(value.ToString(), out long val)) + { + return (val >= (long)bitcount.MinVal && val <= (long)bitcount.MaxVal); + } + else if (ulong.TryParse(value.ToString(), out ulong val2)) + { + return val2 <= (ulong)bitcount.MaxVal; + } + } + + return true; + } + #endregion diff --git a/WDBXEditor/Main.cs b/WDBXEditor/Main.cs index bc300e8..09f2320 100644 --- a/WDBXEditor/Main.cs +++ b/WDBXEditor/Main.cs @@ -1206,7 +1206,13 @@ private void InsertLine() string res = ""; if (ShowInputDialog("Id:", "Id to insert", "1", ref res) == DialogResult.OK) { - if (int.TryParse(res, out int id) && id > 0) //Ensure the result is an integer + int keyIndex = advancedDataGridView.Columns[LoadedEntry.Key].Index; + + if(!int.TryParse(res, out int id) || id < 0 || !advancedDataGridView.ValidValue(keyIndex, id)) + { + MessageBox.Show($"Invalid Id. Out of range of the column min/max value."); + } + else { int index = _bindingsource.Find(LoadedEntry.Key, id); //See if the Id exists if (index < 0) @@ -1223,8 +1229,6 @@ private void InsertLine() advancedDataGridView.SelectRow(index); } - else - MessageBox.Show($"Invalid Id."); } } diff --git a/WDBXEditor/Reader/FileTypes/WDC1.cs b/WDBXEditor/Reader/FileTypes/WDC1.cs index f72644e..3c10da2 100644 --- a/WDBXEditor/Reader/FileTypes/WDC1.cs +++ b/WDBXEditor/Reader/FileTypes/WDC1.cs @@ -23,6 +23,7 @@ public class WDC1 : WDB6 public List ColumnMeta; public RelationShipData RelationShipData; + public Dictionary MinMaxValues; private byte[] recordData; @@ -360,6 +361,7 @@ public override byte[] ReadData(BinaryReader dbReader, long pos) public void SetColumnMinMaxValues(DBEntry entry) { + MinMaxValues = new Dictionary(); int column = 0; for (int i = 0; i < ColumnMeta.Count; i++) { @@ -395,6 +397,16 @@ public void SetColumnMinMaxValues(DBEntry entry) entry.Data.Columns[column].ExtendedProperties.Add("MaxValue", max); if (signed || isfloat) entry.Data.Columns[column].ExtendedProperties.Add("MinValue", min); + + MinMax minmax = new MinMax() + { + Signed = signed, + MaxVal = max, + MinVal = min, + IsSingle = isfloat + }; + + MinMaxValues[column] = minmax; column++; } } @@ -780,4 +792,12 @@ private object[] ExtractFields(Queue rowData, StringTable stringTable, B #endregion } + + public class MinMax + { + public object MinVal; + public object MaxVal; + public bool Signed; + public bool IsSingle; + } } diff --git a/WDBXEditor/Storage/DBEntry.cs b/WDBXEditor/Storage/DBEntry.cs index 6bb238e..253693a 100644 --- a/WDBXEditor/Storage/DBEntry.cs +++ b/WDBXEditor/Storage/DBEntry.cs @@ -770,6 +770,9 @@ public bool ImportCSV(string filename, bool headerrow, UpdateMode mode, out stri return false; } + if (!ValidateMinMaxValues(importTable, out error)) + return false; + UpdateData(importTable, mode); return true; } @@ -823,10 +826,50 @@ public bool ImportSQL(UpdateMode mode, string connectionstring, string table, ou return false; } + if (!ValidateMinMaxValues(importTable, out error)) + return false; + UpdateData(importTable, mode); return true; } + private bool ValidateMinMaxValues(DataTable importTable, out string error) + { + error = ""; + + if (Header is WDC1 header) + { + foreach (var minmax in header.MinMaxValues) + { + Func compare = (x, min, max) => x < min || x > max; + + bool errored = false; + + var values = importTable.Rows.Cast().Select(x => x.ItemArray[minmax.Key]); + if (minmax.Value.IsSingle) + { + errored = values.Any(x => compare((float)Convert.ChangeType(x, typeof(float)), minmax.Value.MinVal, minmax.Value.MaxVal)); + } + else if (minmax.Value.Signed) + { + errored = values.Any(x => compare((long)Convert.ChangeType(x, typeof(long)), minmax.Value.MinVal, minmax.Value.MaxVal)); + } + else + { + errored = values.Any(x => compare((ulong)Convert.ChangeType(x, typeof(ulong)), minmax.Value.MinVal, minmax.Value.MaxVal)); + } + + if (errored) + { + error = $"Import Failed: Imported data has out of range values for column {minmax.Key} (Min {minmax.Value.MinVal}, Max {minmax.Value.MaxVal})"; + return false; + } + } + } + + return true; + } + private void UpdateData(DataTable importTable, UpdateMode mode) { switch (mode) From 5e71a0c24cb3e0a28148a75538633e735df49190 Mon Sep 17 00:00:00 2001 From: Barncastle Date: Sun, 4 Mar 2018 20:15:18 +0000 Subject: [PATCH 09/11] fixes --- WDBXEditor/Reader/FileTypes/WDC1.cs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/WDBXEditor/Reader/FileTypes/WDC1.cs b/WDBXEditor/Reader/FileTypes/WDC1.cs index 3c10da2..0d24637 100644 --- a/WDBXEditor/Reader/FileTypes/WDC1.cs +++ b/WDBXEditor/Reader/FileTypes/WDC1.cs @@ -612,12 +612,11 @@ public void WriteData(BinaryWriter bw, DBEntry entry) } } + bitStream.SeekNextOffset(); short size = (short)(pos + bitStream.Offset - offset); if (IsSparse) // matches itemsparse padding - { - bitStream.SeekNextOffset(); - + { int remaining = size % 8 == 0 ? 0 : 8 - (size % 8); if (remaining > 0) { @@ -629,9 +628,8 @@ public void WriteData(BinaryWriter bw, DBEntry entry) } else // needs to be padded to the record size regardless of the byte count - weird eh? { - bitStream.SeekNextOffset(); - //if (size < RecordSize) - // bitStream.WriteBytes(new byte[RecordSize - size], RecordSize - size); + if (size < RecordSize) + bitStream.WriteBytes(new byte[RecordSize - size], RecordSize - size); } } bitStream.CopyStreamTo(bw.BaseStream); // write to the filestream From 98d390a4348384d0ef69a26bc07308df295a8ee3 Mon Sep 17 00:00:00 2001 From: Barncastle Date: Mon, 5 Mar 2018 13:45:56 +0000 Subject: [PATCH 10/11] twekas --- .../Definitions/Legion 7.3.5 (25632).xml | 26 +++++++-------- WDBXEditor/Forms/WotLKItemFix.cs | 32 +++++++++---------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml b/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml index c48912f..46ec69c 100644 --- a/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml +++ b/WDBXEditor/Definitions/Legion 7.3.5 (25632).xml @@ -4370,24 +4370,24 @@
- + - - + + - + - + - - + + - + - + @@ -4395,11 +4395,9 @@ - - - - - + + +
diff --git a/WDBXEditor/Forms/WotLKItemFix.cs b/WDBXEditor/Forms/WotLKItemFix.cs index 5242d9c..94fa07b 100644 --- a/WDBXEditor/Forms/WotLKItemFix.cs +++ b/WDBXEditor/Forms/WotLKItemFix.cs @@ -186,26 +186,26 @@ private void PopulateTable() private readonly Dictionary MangosLookup = new Dictionary() { - {"m_ID","entry" }, - {"m_classID","class" }, - {"m_subclassID","subclass" }, - {"m_sound_override_subclassid","unk0" }, - {"m_material","Material" }, - {"m_displayInfoID","displayid" }, - {"m_inventoryType","InventoryType" }, - {"m_sheatheType","sheath" } + {"ID","entry" }, + {"ClassID","class" }, + {"SubclassID","subclass" }, + {"Sound_override_subclassid","unk0" }, + {"Material","Material" }, + {"DisplayInfoID","displayid" }, + {"InventoryType","InventoryType" }, + {"SheatheType","sheath" } }; private readonly Dictionary TrinityLookup = new Dictionary() { - {"m_ID","entry" }, - {"m_classID","class" }, - {"m_subclassID","subclass" }, - {"m_sound_override_subclassid","SoundOverrideSubclass" }, - {"m_material","Material" }, - {"m_displayInfoID","displayid" }, - {"m_inventoryType","InventoryType" }, - {"m_sheatheType","sheath" } + {"ID","entry" }, + {"ClassID","class" }, + {"SubclassID","subclass" }, + {"Sound_override_subclassid","SoundOverrideSubclass" }, + {"Material","Material" }, + {"DisplayInfoID","displayid" }, + {"InventoryType","InventoryType" }, + {"SheatheType","sheath" } }; } From eace3af03ffa18e4a48ebd414de7e8fe1ed78732 Mon Sep 17 00:00:00 2001 From: Barncastle Date: Wed, 7 Mar 2018 12:20:10 +0000 Subject: [PATCH 11/11] tweaks --- WDBXEditor/Forms/WotLKItemFix.cs | 373 +++++++++++++++---------------- WDBXEditor/Storage/DBEntry.cs | 4 +- 2 files changed, 180 insertions(+), 197 deletions(-) diff --git a/WDBXEditor/Forms/WotLKItemFix.cs b/WDBXEditor/Forms/WotLKItemFix.cs index 94fa07b..3cc4b4c 100644 --- a/WDBXEditor/Forms/WotLKItemFix.cs +++ b/WDBXEditor/Forms/WotLKItemFix.cs @@ -13,200 +13,181 @@ namespace WDBXEditor { - public partial class WotLKItemFix : Form - { - public string ConnectionString => $"Server={txtHost.Text};Port={txtPort.Text};Database={ddlDatabases.Text};Uid={txtUser.Text};Pwd={txtPass.Text};"; - public DBEntry Entry { get; set; } - - private bool testedconnection = false; - - - public WotLKItemFix() - { - InitializeComponent(); - } - - private void WotLKItemFix_Load(object sender, EventArgs e) - { - LoadSettings(); - PopulateTable(); - } - - #region Buttons - private void btnRefresh_Click(object sender, EventArgs e) - { - ddlDatabases.Items.Clear(); - testedconnection = false; - ddlDatabases.Enabled = false; - - try - { - string sql = "SHOW DATABASES;"; - using (MySqlConnection connection = new MySqlConnection(ConnectionString)) - { - connection.Open(); - MySqlCommand command = new MySqlCommand(sql, connection); - using (var rdr = command.ExecuteReader()) - { - ddlDatabases.Items.Add(""); - - while (rdr.Read()) - ddlDatabases.Items.Add(rdr[0].ToString()); - - testedconnection = true; - ddlDatabases.Enabled = true; - SaveSettings(); - } - } - - } - catch (MySqlException ex) - { - MessageBox.Show(ex.Message); - } - } - - private void btnClose_Click(object sender, EventArgs e) - { - this.DialogResult = DialogResult.Cancel; - this.Close(); - } - - private void btnLoad_Click(object sender, EventArgs e) - { - string columns = string.Join(",", dgvSchema.Rows.Cast().Select(x => $"`{x.Cells["Column"].Value.ToString()}` AS `{x.Cells["Field"].Value.ToString()}` ")); - if (columns.IndexOf("``") >= 0) - { - MessageBox.Show("Some columns are unmapped."); - return; - } - - string ErrorMessage = string.Empty; - Entry.ImportSQL(UpdateMode.Update, ConnectionString, ddlTable.Text, out ErrorMessage, columns); - if (!string.IsNullOrWhiteSpace(ErrorMessage)) - { - MessageBox.Show(ErrorMessage); - this.DialogResult = DialogResult.Abort; - } - else - this.DialogResult = DialogResult.OK; - - this.Close(); - } - #endregion - - #region Dropdown Methods - private void ddlDatabases_SelectedIndexChanged(object sender, EventArgs e) - { - if (ddlTable.Enabled) - { - ddlTable.Items.Clear(); - - try - { - string sql = $"USE {ddlDatabases.Text}; SHOW TABLES;"; - using (MySqlConnection connection = new MySqlConnection(ConnectionString)) - { - connection.Open(); - MySqlCommand command = new MySqlCommand(sql, connection); - using (var rdr = command.ExecuteReader()) - { - ddlTable.Items.Add(""); - while (rdr.Read()) - ddlTable.Items.Add(rdr[0].ToString()); - } - } - } - catch { return; } - } - - btnLoad.Enabled = !string.IsNullOrWhiteSpace(ddlDatabases.Text) && //Database selected - testedconnection && //Connection works - (!string.IsNullOrWhiteSpace(ddlTable.Text) || !ddlTable.Enabled); //Table selected/not applicable - } - - private void ddlTable_SelectedIndexChanged(object sender, EventArgs e) - { - btnLoad.Enabled = !string.IsNullOrWhiteSpace(ddlDatabases.Text) && //Database selected - testedconnection && //Connection works - (!string.IsNullOrWhiteSpace(ddlTable.Text) || !ddlTable.Enabled); //Table selected/not applicable - } - - private void ddlTemplate_SelectedIndexChanged(object sender, EventArgs e) - { - PopulateTable(); - } - #endregion - - private void SaveSettings() - { - Properties.Settings.Default["Host"] = txtHost.Text; - Properties.Settings.Default["Port"] = txtPort.Text; - Properties.Settings.Default["User"] = txtUser.Text; - Properties.Settings.Default["Password"] = txtPass.Text; - Properties.Settings.Default.Save(); - } - - private void LoadSettings() - { - txtHost.Text = Properties.Settings.Default["Host"].ToString(); - txtPort.Text = Properties.Settings.Default["Port"].ToString(); - txtUser.Text = Properties.Settings.Default["User"].ToString(); - txtPass.Text = Properties.Settings.Default["Password"].ToString(); - } - - private void PopulateTable() - { - DataTable schema = new DataTable(); - schema.Columns.Add("Field"); - schema.Columns.Add("Column"); - - if (ddlTemplate.Text == "Trinity") - { - foreach (var col in TrinityLookup) - schema.Rows.Add(col.Key, col.Value); - } - else if (ddlTemplate.Text.IndexOf("Mangos") > -1) - { - foreach (var col in MangosLookup) - schema.Rows.Add(col.Key, col.Value); - } - else - { - foreach (var col in MangosLookup) - schema.Rows.Add(col.Key, ""); - } - - dgvSchema.DataSource = schema; - dgvSchema.Columns[0].ReadOnly = true; - dgvSchema.Columns[0].HeaderText = "DBC Field"; - dgvSchema.Columns[1].HeaderText = "Database Column"; - } - - - private readonly Dictionary MangosLookup = new Dictionary() - { - {"ID","entry" }, - {"ClassID","class" }, - {"SubclassID","subclass" }, - {"Sound_override_subclassid","unk0" }, - {"Material","Material" }, - {"DisplayInfoID","displayid" }, - {"InventoryType","InventoryType" }, - {"SheatheType","sheath" } - }; - - private readonly Dictionary TrinityLookup = new Dictionary() - { - {"ID","entry" }, - {"ClassID","class" }, - {"SubclassID","subclass" }, - {"Sound_override_subclassid","SoundOverrideSubclass" }, - {"Material","Material" }, - {"DisplayInfoID","displayid" }, - {"InventoryType","InventoryType" }, - {"SheatheType","sheath" } - }; - - } + public partial class WotLKItemFix : Form + { + public string ConnectionString => $"Server={txtHost.Text};Port={txtPort.Text};Database={ddlDatabases.Text};Uid={txtUser.Text};Pwd={txtPass.Text};"; + public DBEntry Entry { get; set; } + + private bool validConn = false; + + + public WotLKItemFix() + { + InitializeComponent(); + } + + private void WotLKItemFix_Load(object sender, EventArgs e) + { + LoadSettings(); + PopulateTable(); + } + + #region Buttons + private void btnRefresh_Click(object sender, EventArgs e) + { + ddlDatabases.Items.Clear(); + validConn = false; + ddlDatabases.Enabled = false; + + try + { + string sql = "SHOW DATABASES;"; + using (MySqlConnection connection = new MySqlConnection(ConnectionString)) + { + connection.Open(); + MySqlCommand command = new MySqlCommand(sql, connection); + using (var rdr = command.ExecuteReader()) + { + ddlDatabases.Items.Add(""); + + while (rdr.Read()) + ddlDatabases.Items.Add(rdr[0].ToString()); + + validConn = true; + ddlDatabases.Enabled = true; + SaveSettings(); + } + } + + } + catch (MySqlException ex) + { + MessageBox.Show(ex.Message); + } + } + + private void btnClose_Click(object sender, EventArgs e) + { + this.DialogResult = DialogResult.Cancel; + this.Close(); + } + + private void btnLoad_Click(object sender, EventArgs e) + { + string columns = string.Join(",", dgvSchema.Rows.Cast().Select(x => $"`{x.Cells["Column"].Value.ToString()}` AS `{x.Cells["Field"].Value.ToString()}` ")); + if (columns.IndexOf("``") >= 0) + { + MessageBox.Show("Some columns are unmapped."); + return; + } + + string ErrorMessage = string.Empty; + Entry.ImportSQL(UpdateMode.Update, ConnectionString, ddlTable.Text, out ErrorMessage, columns); + if (!string.IsNullOrWhiteSpace(ErrorMessage)) + { + MessageBox.Show(ErrorMessage); + this.DialogResult = DialogResult.Abort; + } + else + this.DialogResult = DialogResult.OK; + + this.Close(); + } + #endregion + + #region Dropdown Methods + private void ddlDatabases_SelectedIndexChanged(object sender, EventArgs e) + { + if (ddlTable.Enabled) + { + ddlTable.Items.Clear(); + + try + { + string sql = $"USE {ddlDatabases.Text}; SHOW TABLES;"; + using (MySqlConnection connection = new MySqlConnection(ConnectionString)) + { + connection.Open(); + MySqlCommand command = new MySqlCommand(sql, connection); + using (var rdr = command.ExecuteReader()) + { + ddlTable.Items.Add(""); + while (rdr.Read()) + ddlTable.Items.Add(rdr[0].ToString()); + } + } + } + catch { return; } + } + + btnLoad.Enabled = !string.IsNullOrWhiteSpace(ddlDatabases.Text) && //Database selected + validConn && //Connection works + (!string.IsNullOrWhiteSpace(ddlTable.Text) || !ddlTable.Enabled); //Table selected/not applicable + } + + private void ddlTable_SelectedIndexChanged(object sender, EventArgs e) + { + btnLoad.Enabled = !string.IsNullOrWhiteSpace(ddlDatabases.Text) && //Database selected + validConn && //Connection works + (!string.IsNullOrWhiteSpace(ddlTable.Text) || !ddlTable.Enabled); //Table selected/not applicable + } + + private void ddlTemplate_SelectedIndexChanged(object sender, EventArgs e) + { + PopulateTable(); + } + #endregion + + private void SaveSettings() + { + Properties.Settings.Default["Host"] = txtHost.Text; + Properties.Settings.Default["Port"] = txtPort.Text; + Properties.Settings.Default["User"] = txtUser.Text; + Properties.Settings.Default["Password"] = txtPass.Text; + Properties.Settings.Default.Save(); + } + + private void LoadSettings() + { + txtHost.Text = Properties.Settings.Default["Host"].ToString(); + txtPort.Text = Properties.Settings.Default["Port"].ToString(); + txtUser.Text = Properties.Settings.Default["User"].ToString(); + txtPass.Text = Properties.Settings.Default["Password"].ToString(); + } + + private void PopulateTable() + { + DataTable schema = new DataTable(); + schema.Columns.Add("Field"); + schema.Columns.Add("Column"); + + var fields = Database.Definitions.Tables.First(x => x.Name.ToLower() == "item" && x.Build == (int)ExpansionFinalBuild.WotLK).Fields; + + if (ddlTemplate.Text == "Trinity") + { + for (int i = 0; i < fields.Length; i++) + schema.Rows.Add(fields[i].Name, TrinityLookup[i]); + } + else if (ddlTemplate.Text.IndexOf("Mangos") > -1) + { + for (int i = 0; i < fields.Length; i++) + schema.Rows.Add(fields[i].Name, MangosLookup[i]); + } + else + { + for (int i = 0; i < fields.Length; i++) + schema.Rows.Add(fields[i].Name, ""); + } + + dgvSchema.DataSource = schema; + dgvSchema.Columns[0].ReadOnly = true; + dgvSchema.Columns[0].HeaderText = "DBC Field"; + dgvSchema.Columns[1].HeaderText = "Database Column"; + } + + private readonly string[] MangosLookup = new[] { "Entry", "Class", "SubClass", "Unk0", "Material", "DisplayId", "InventoryType", "Sheath" }; + + private readonly string[] TrinityLookup = new[] { "Entry", "Class", "SubClass", "SoundOverrideSubclass", "Material", "DisplayId", "InventoryType", "Sheath" }; + + } } diff --git a/WDBXEditor/Storage/DBEntry.cs b/WDBXEditor/Storage/DBEntry.cs index 253693a..5b65811 100644 --- a/WDBXEditor/Storage/DBEntry.cs +++ b/WDBXEditor/Storage/DBEntry.cs @@ -861,7 +861,9 @@ private bool ValidateMinMaxValues(DataTable importTable, out string error) if (errored) { - error = $"Import Failed: Imported data has out of range values for column {minmax.Key} (Min {minmax.Value.MinVal}, Max {minmax.Value.MaxVal})"; + error = $"Import Failed: Imported data has out of range values for {Data.Columns[minmax.Key].ColumnName}.\n" + + $"(Min: {minmax.Value.MinVal}, Max: {minmax.Value.MaxVal})"; + return false; } }