new DB fields and shorter fields

src/core/Odin.Core.Storage/Database/Identity/Abstractions/MainIndexMeta.cs

@@ -1,8 +1,13 @@
 using System;
 using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
 using System.Data;
+using System.IO;
 using System.Linq;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using Microsoft.VisualBasic.FileIO;
+using System.Xml;
 using Odin.Core.Exceptions;
 using Odin.Core.Storage.Database.Identity.Connection;
 using Odin.Core.Storage.Database.Identity.Table;
@@ -19,6 +24,19 @@ public class MainIndexMeta(
         TableDriveMainIndex driveMainIndex)
     {
         private readonly DatabaseType _databaseType = scopedConnectionFactory.DatabaseType;
+        private static readonly string selectOutputFields;
+
+        static MainIndexMeta()
+        {
+            // Initialize selectOutputFields statically
+            selectOutputFields = string.Join(",",
+                TableDriveMainIndex.GetColumnNames()
+                    .Where(name => !name.Equals("identityId", StringComparison.OrdinalIgnoreCase)
+                                && !name.Equals("driveId", StringComparison.OrdinalIgnoreCase))
+                    .Select(name => name.Equals("fileId", StringComparison.OrdinalIgnoreCase)
+                                    ? "driveMainIndex.fileId"
+                                    : name));
+        }
 
         public async Task<int> DeleteEntryAsync(Guid driveId, Guid fileId)
         {
@@ -308,13 +326,13 @@ private string SharedWhereAnd(List<string> listWhere, IntRange requiredSecurityG
             string leftJoin = SharedWhereAnd(listWhereAnd, requiredSecurityGroup, aclAnyOf, filetypesAnyOf, datatypesAnyOf, globalTransitIdAnyOf,
                 uniqueIdAnyOf, tagsAnyOf, archivalStatusAnyOf, senderidAnyOf, groupIdAnyOf, userdateSpan, tagsAllOf,
                 fileSystemType, driveId);
-
             if (IsSet(fileStateAnyOf))
             {
                 listWhereAnd.Add($"fileState IN ({IntList(fileStateAnyOf)})");
             }
 
-            string selectOutputFields = "driveMainIndex.fileId, globalTransitId, fileState, requiredSecurityGroup, fileSystemType, userDate, fileType, dataType, archivalStatus, historyStatus, senderId, groupId, uniqueId, byteCount, hdrEncryptedKeyHeader, hdrVersionTag, hdrAppData, hdrReactionSummary, hdrServerData, hdrTransferHistory, hdrFileMetaData, hdrTmpDriveAlias, hdrTmpDriveType, created, modified";
+            // string selectOutputFields    = "driveMainIndex.fileId, globalTransitId, fileState, requiredSecurityGroup, fileSystemType, userDate, fileType, dataType, archivalStatus, historyStatus, senderId, groupId, uniqueId, byteCount, hdrEncryptedKeyHeader, hdrVersionTag, hdrAppData, hdrLocalVersionTag,hdrLocalAppData,hdrReactionSummary, hdrServerData, hdrTransferHistory, hdrFileMetaData, hdrTmpDriveAlias, hdrTmpDriveType, created, modified";
+            // string selectOutputFields = "driveMainIndex.fileId, globalTransitId, fileState, requiredSecurityGroup, fileSystemType, userDate, fileType, dataType, archivalStatus, historyStatus, senderId, groupId, uniqueId, byteCount, hdrEncryptedKeyHeader, hdrVersionTag, hdrAppData,                                    hdrReactionSummary, hdrServerData, hdrTransferHistory, hdrFileMetaData, hdrTmpDriveAlias, hdrTmpDriveType, created, modified";
             /*if (fileIdSort)
                 selectOutputFields = "driveMainIndex.fileId";
             else
@@ -332,7 +350,6 @@ private string SharedWhereAnd(List<string> listWhere, IntRange requiredSecurityG
 
             // Read +1 more than requested to see if we're at the end of the dataset
             string stm = $"SELECT DISTINCT {selectOutputFields} FROM driveMainIndex {leftJoin} WHERE " + string.Join(" AND ", listWhereAnd) + $" ORDER BY {order} LIMIT {noOfItems + 1}";
-
             await using var cn = await scopedConnectionFactory.CreateScopedConnectionAsync();
             await using var cmd = cn.CreateCommand();
 
@@ -578,7 +595,7 @@ private string SharedWhereAnd(List<string> listWhere, IntRange requiredSecurityG
                 uniqueIdAnyOf, tagsAnyOf, archivalStatusAnyOf, senderidAnyOf, groupIdAnyOf, userdateSpan, tagsAllOf,
                 fileSystemType, driveId);
 
-            string selectOutputFields = "driveMainIndex.fileId, globalTransitId, fileState, requiredSecurityGroup, fileSystemType, userDate, fileType, dataType, archivalStatus, historyStatus, senderId, groupId, uniqueId, byteCount, hdrEncryptedKeyHeader, hdrVersionTag, hdrAppData, hdrReactionSummary, hdrServerData, hdrTransferHistory, hdrFileMetaData, hdrTmpDriveAlias, hdrTmpDriveType, created, modified";
+            // string selectOutputFields =  "driveMainIndex.fileId, globalTransitId, fileState, requiredSecurityGroup, fileSystemType, userDate, fileType, dataType, archivalStatus, historyStatus, senderId, groupId, uniqueId, byteCount, hdrEncryptedKeyHeader, hdrVersionTag, hdrAppData, hdrReactionSummary, hdrServerData, hdrTransferHistory, hdrFileMetaData, hdrTmpDriveAlias, hdrTmpDriveType, created, modified";
             string stm = $"SELECT DISTINCT {selectOutputFields} FROM drivemainindex {leftJoin} WHERE " + string.Join(" AND ", listWhereAnd) + $" ORDER BY modified ASC LIMIT {noOfItems + 1}";
             // string stm = $"SELECT DISTINCT driveMainIndex.fileid, modified FROM drivemainindex {leftJoin} WHERE " + string.Join(" AND ", listWhereAnd) + $" ORDER BY modified ASC LIMIT {noOfItems + 1}";
 

src/core/Odin.Core.Storage/Database/Identity/Table/TableAppGrantsCRUD.cs

@@ -69,6 +69,16 @@ public byte[] data
                   _data = value;
                }
         }
+        internal byte[] dataNoLengthCheck
+        {
+           get {
+                   return _data;
+               }
+           set {
+                    if (value?.Length < 0) throw new Exception("Too short");
+                  _data = value;
+               }
+        }
     } // End of class AppGrantsRecord
 
     public abstract class TableAppGrantsCRUD
@@ -288,7 +298,7 @@ protected virtual async Task<int> GetCountAsync()
             }
         }
 
-        public List<string> GetColumnNames()
+        public static List<string> GetColumnNames()
         {
             var sl = new List<string>();
             sl.Add("identityId");
@@ -309,16 +319,11 @@ protected AppGrantsRecord ReadRecordFromReaderAll(DbDataReader rdr)
 #pragma warning restore CS0168
             var guid = new byte[16];
             var item = new AppGrantsRecord();
-            item.identityId = rdr.IsDBNull(0) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[0]);
-            item.odinHashId = rdr.IsDBNull(1) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[1]);
-            item.appId = rdr.IsDBNull(2) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[2]);
-            item.circleId = rdr.IsDBNull(3) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[3]);
-            item.data = rdr.IsDBNull(4) ? 
-                null : (byte[])(rdr[4]);
+            item.identityId = rdr.IsDBNull(0) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[0]);
+            item.odinHashId = rdr.IsDBNull(1) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[1]);
+            item.appId = rdr.IsDBNull(2) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[2]);
+            item.circleId = rdr.IsDBNull(3) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[3]);
+            item.dataNoLengthCheck = rdr.IsDBNull(4) ? null : (byte[])(rdr[4]);
             if (item.data?.Length > 65535)
                 throw new Exception("Too much data in data...");
             if (item.data?.Length < 0)
@@ -368,15 +373,9 @@ protected AppGrantsRecord ReadRecordFromReader0(DbDataReader rdr, Guid identityI
             var item = new AppGrantsRecord();
             item.identityId = identityId;
             item.odinHashId = odinHashId;
-
-            item.appId = rdr.IsDBNull(0) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[0]);
-
-            item.circleId = rdr.IsDBNull(1) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[1]);
-
-            item.data = rdr.IsDBNull(2) ? 
-                null : (byte[])(rdr[2]);
+            item.appId = rdr.IsDBNull(0) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[0]);
+            item.circleId = rdr.IsDBNull(1) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[1]);
+            item.dataNoLengthCheck = rdr.IsDBNull(2) ? null : (byte[])(rdr[2]);
             if (item.data?.Length > 65535)
                 throw new Exception("Too much data in data...");
             if (item.data?.Length < 0)
@@ -434,9 +433,7 @@ protected AppGrantsRecord ReadRecordFromReader1(DbDataReader rdr, Guid identityI
             item.odinHashId = odinHashId;
             item.appId = appId;
             item.circleId = circleId;
-
-            item.data = rdr.IsDBNull(0) ? 
-                null : (byte[])(rdr[0]);
+            item.dataNoLengthCheck = rdr.IsDBNull(0) ? null : (byte[])(rdr[0]);
             if (item.data?.Length > 65535)
                 throw new Exception("Too much data in data...");
             if (item.data?.Length < 0)

src/core/Odin.Core.Storage/Database/Identity/Table/TableAppNotificationsCRUD.cs

@@ -59,6 +59,16 @@ public string senderId
                   _senderId = value;
                }
         }
+        internal string senderIdNoLengthCheck
+        {
+           get {
+                   return _senderId;
+               }
+           set {
+                    if (value?.Length < 0) throw new Exception("Too short");
+                  _senderId = value;
+               }
+        }
         private UnixTimeUtc _timestamp;
         public UnixTimeUtc timestamp
         {
@@ -81,6 +91,16 @@ public byte[] data
                   _data = value;
                }
         }
+        internal byte[] dataNoLengthCheck
+        {
+           get {
+                   return _data;
+               }
+           set {
+                    if (value?.Length < 0) throw new Exception("Too short");
+                  _data = value;
+               }
+        }
         private UnixTimeUtcUnique _created;
         public UnixTimeUtcUnique created
         {
@@ -384,7 +404,7 @@ protected virtual async Task<int> GetCountAsync()
             }
         }
 
-        public List<string> GetColumnNames()
+        public static List<string> GetColumnNames()
         {
             var sl = new List<string>();
             sl.Add("identityId");
@@ -408,26 +428,18 @@ protected AppNotificationsRecord ReadRecordFromReaderAll(DbDataReader rdr)
 #pragma warning restore CS0168
             var guid = new byte[16];
             var item = new AppNotificationsRecord();
-            item.identityId = rdr.IsDBNull(0) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[0]);
-            item.notificationId = rdr.IsDBNull(1) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[1]);
-            item.unread = rdr.IsDBNull(2) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : (int)(long)rdr[2];
-            item.senderId = rdr.IsDBNull(3) ? 
-                null : (string)rdr[3];
-            item.timestamp = rdr.IsDBNull(4) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new UnixTimeUtc((long)rdr[4]);
-            item.data = rdr.IsDBNull(5) ? 
-                null : (byte[])(rdr[5]);
+            item.identityId = rdr.IsDBNull(0) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[0]);
+            item.notificationId = rdr.IsDBNull(1) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[1]);
+            item.unread = rdr.IsDBNull(2) ? throw new Exception("item is NULL, but set as NOT NULL") : (int)(long)rdr[2];
+            item.senderIdNoLengthCheck = rdr.IsDBNull(3) ? null : (string)rdr[3];
+            item.timestamp = rdr.IsDBNull(4) ? throw new Exception("item is NULL, but set as NOT NULL") : new UnixTimeUtc((long)rdr[4]);
+            item.dataNoLengthCheck = rdr.IsDBNull(5) ? null : (byte[])(rdr[5]);
             if (item.data?.Length > 65000)
                 throw new Exception("Too much data in data...");
             if (item.data?.Length < 0)
                 throw new Exception("Too little data in data...");
-            item.created = rdr.IsDBNull(6) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new UnixTimeUtcUnique((long)rdr[6]);
-            item.modified = rdr.IsDBNull(7) ? 
-                null : new UnixTimeUtcUnique((long)rdr[7]);
+            item.created = rdr.IsDBNull(6) ? throw new Exception("item is NULL, but set as NOT NULL") : new UnixTimeUtcUnique((long)rdr[6]);
+            item.modified = rdr.IsDBNull(7) ? null : new UnixTimeUtcUnique((long)rdr[7]);
             return item;
        }
 
@@ -465,28 +477,16 @@ protected AppNotificationsRecord ReadRecordFromReader0(DbDataReader rdr, Guid id
             var item = new AppNotificationsRecord();
             item.identityId = identityId;
             item.notificationId = notificationId;
-
-            item.unread = rdr.IsDBNull(0) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : (int)(long)rdr[0];
-
-            item.senderId = rdr.IsDBNull(1) ? 
-                null : (string)rdr[1];
-
-            item.timestamp = rdr.IsDBNull(2) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new UnixTimeUtc((long)rdr[2]);
-
-            item.data = rdr.IsDBNull(3) ? 
-                null : (byte[])(rdr[3]);
+            item.unread = rdr.IsDBNull(0) ? throw new Exception("item is NULL, but set as NOT NULL") : (int)(long)rdr[0];
+            item.senderIdNoLengthCheck = rdr.IsDBNull(1) ? null : (string)rdr[1];
+            item.timestamp = rdr.IsDBNull(2) ? throw new Exception("item is NULL, but set as NOT NULL") : new UnixTimeUtc((long)rdr[2]);
+            item.dataNoLengthCheck = rdr.IsDBNull(3) ? null : (byte[])(rdr[3]);
             if (item.data?.Length > 65000)
                 throw new Exception("Too much data in data...");
             if (item.data?.Length < 0)
                 throw new Exception("Too little data in data...");
-
-            item.created = rdr.IsDBNull(4) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new UnixTimeUtcUnique((long)rdr[4]);
-
-            item.modified = rdr.IsDBNull(5) ? 
-                null : new UnixTimeUtcUnique((long)rdr[5]);
+            item.created = rdr.IsDBNull(4) ? throw new Exception("item is NULL, but set as NOT NULL") : new UnixTimeUtcUnique((long)rdr[4]);
+            item.modified = rdr.IsDBNull(5) ? null : new UnixTimeUtcUnique((long)rdr[5]);
             return item;
        }
 

src/core/Odin.Core.Storage/Database/Identity/Table/TableCircleCRUD.cs

@@ -40,6 +40,17 @@ public string circleName
                   _circleName = value;
                }
         }
+        internal string circleNameNoLengthCheck
+        {
+           get {
+                   return _circleName;
+               }
+           set {
+                    if (value == null) throw new Exception("Cannot be null");
+                    if (value?.Length < 2) throw new Exception("Too short");
+                  _circleName = value;
+               }
+        }
         private Guid _circleId;
         public Guid circleId
         {
@@ -62,6 +73,16 @@ public byte[] data
                   _data = value;
                }
         }
+        internal byte[] dataNoLengthCheck
+        {
+           get {
+                   return _data;
+               }
+           set {
+                    if (value?.Length < 0) throw new Exception("Too short");
+                  _data = value;
+               }
+        }
     } // End of class CircleRecord
 
     public abstract class TableCircleCRUD
@@ -256,7 +277,7 @@ protected virtual async Task<int> GetCountAsync()
             }
         }
 
-        public List<string> GetColumnNames()
+        public static List<string> GetColumnNames()
         {
             var sl = new List<string>();
             sl.Add("identityId");
@@ -276,14 +297,10 @@ protected CircleRecord ReadRecordFromReaderAll(DbDataReader rdr)
 #pragma warning restore CS0168
             var guid = new byte[16];
             var item = new CircleRecord();
-            item.identityId = rdr.IsDBNull(0) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[0]);
-            item.circleName = rdr.IsDBNull(1) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : (string)rdr[1];
-            item.circleId = rdr.IsDBNull(2) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[2]);
-            item.data = rdr.IsDBNull(3) ? 
-                null : (byte[])(rdr[3]);
+            item.identityId = rdr.IsDBNull(0) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[0]);
+            item.circleNameNoLengthCheck = rdr.IsDBNull(1) ? throw new Exception("item is NULL, but set as NOT NULL") : (string)rdr[1];
+            item.circleId = rdr.IsDBNull(2) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[2]);
+            item.dataNoLengthCheck = rdr.IsDBNull(3) ? null : (byte[])(rdr[3]);
             if (item.data?.Length > 65000)
                 throw new Exception("Too much data in data...");
             if (item.data?.Length < 0)
@@ -325,12 +342,8 @@ protected CircleRecord ReadRecordFromReader0(DbDataReader rdr, Guid identityId,G
             var item = new CircleRecord();
             item.identityId = identityId;
             item.circleId = circleId;
-
-            item.circleName = rdr.IsDBNull(0) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : (string)rdr[0];
-
-            item.data = rdr.IsDBNull(1) ? 
-                null : (byte[])(rdr[1]);
+            item.circleNameNoLengthCheck = rdr.IsDBNull(0) ? throw new Exception("item is NULL, but set as NOT NULL") : (string)rdr[0];
+            item.dataNoLengthCheck = rdr.IsDBNull(1) ? null : (byte[])(rdr[1]);
             if (item.data?.Length > 65000)
                 throw new Exception("Too much data in data...");
             if (item.data?.Length < 0)