Newfields

src/core/Odin.Core.Storage/Database/Identity/Abstractions/MainIndexMeta.cs

@@ -1,8 +1,13 @@
 using System;
 using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
 using System.Data;
+using System.IO;
 using System.Linq;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using Microsoft.VisualBasic.FileIO;
+using System.Xml;
 using Odin.Core.Exceptions;
 using Odin.Core.Storage.Database.Identity.Connection;
 using Odin.Core.Storage.Database.Identity.Table;
@@ -16,9 +21,23 @@
         IdentityKey identityKey,
         TableDriveAclIndex driveAclIndex,
         TableDriveTagIndex driveTagIndex,
+        TableDriveLocalTagIndex driveLocalTagIndex,
         TableDriveMainIndex driveMainIndex)
     {
         private readonly DatabaseType _databaseType = scopedConnectionFactory.DatabaseType;
+        private static readonly string selectOutputFields;
+
+        static MainIndexMeta()
+        {
+            // Initialize selectOutputFields statically
+            selectOutputFields = string.Join(",",
+                TableDriveMainIndex.GetColumnNames()
+                    .Where(name => !name.Equals("identityId", StringComparison.OrdinalIgnoreCase)
+                                && !name.Equals("driveId", StringComparison.OrdinalIgnoreCase))
+                    .Select(name => name.Equals("fileId", StringComparison.OrdinalIgnoreCase)
+                                    ? "driveMainIndex.fileId"
+                                    : name));
+        }
 
         public async Task<int> DeleteEntryAsync(Guid driveId, Guid fileId)
         {
@@ -94,12 +113,13 @@
         }
 
         private string SharedWhereAnd(List<string> listWhere, IntRange requiredSecurityGroup, List<Guid> aclAnyOf, List<int> filetypesAnyOf,
-            List<int> datatypesAnyOf, List<Guid> globalTransitIdAnyOf, List<Guid> uniqueIdAnyOf, List<Guid> tagsAnyOf,
+            List<int> datatypesAnyOf, List<Guid> globalTransitIdAnyOf, List<Guid> uniqueIdAnyOf, List<Guid> tagsAnyOf, List<Guid> localTagsAnyOf,
             List<Int32> archivalStatusAnyOf,
             List<string> senderidAnyOf,
             List<Guid> groupIdAnyOf,
             UnixTimeUtcRange userdateSpan,
             List<Guid> tagsAllOf,
+            List<Guid> localTagsAllOf,
             Int32? fileSystemType,
             Guid driveId)
         {
@@ -155,6 +175,11 @@
                 listWhere.Add($"driveMainIndex.fileid IN (SELECT DISTINCT fileid FROM drivetagindex WHERE drivetagindex.identityId=driveMainIndex.identityId AND tagId IN ({HexList(tagsAnyOf)}))");
             }
 
+            if (IsSet(localTagsAnyOf))
+            {
+                listWhere.Add($"driveMainIndex.fileId IN (SELECT DISTINCT fileId FROM driveLocalTagIndex WHERE driveLocalTagIndex.identityId=driveMainIndex.identityId AND TagId IN ({HexList(localTagsAnyOf)}))");
+            }
+
             if (IsSet(archivalStatusAnyOf))
             {
                 listWhere.Add($"archivalStatus IN ({IntList(archivalStatusAnyOf)})");
@@ -179,7 +204,13 @@
             if (IsSet(tagsAllOf))
             {
                 // TODO: This will return 0 matches. Figure out the right query.
-                listWhere.Add($"{AndIntersectHexList(tagsAllOf)}");
+                listWhere.Add($"{AndIntersectHexList(tagsAllOf, "driveTagIndex")}");
+            }
+
+            if (IsSet(localTagsAllOf))
+            {
+                // TODO: This will return 0 matches. Figure out the right query.
+                listWhere.Add($"{AndIntersectHexList(localTagsAllOf, "driveLocalTagIndex")}");
             }
 
             return leftJoin;
@@ -227,7 +258,9 @@
             UnixTimeUtcRange userdateSpan = null,
             List<Guid> aclAnyOf = null,
             List<Guid> tagsAnyOf = null,
-            List<Guid> tagsAllOf = null)
+            List<Guid> tagsAllOf = null,
+            List<Guid> localTagsAnyOf = null,
+            List<Guid> localTagsAllOf = null)
         {
             if (null == fileSystemType)
             {
@@ -306,15 +339,15 @@
             }
 
             string leftJoin = SharedWhereAnd(listWhereAnd, requiredSecurityGroup, aclAnyOf, filetypesAnyOf, datatypesAnyOf, globalTransitIdAnyOf,
-                uniqueIdAnyOf, tagsAnyOf, archivalStatusAnyOf, senderidAnyOf, groupIdAnyOf, userdateSpan, tagsAllOf,
+                uniqueIdAnyOf, tagsAnyOf, localTagsAnyOf, archivalStatusAnyOf, senderidAnyOf, groupIdAnyOf, userdateSpan, tagsAllOf, localTagsAllOf,
                 fileSystemType, driveId);
-
             if (IsSet(fileStateAnyOf))
             {
                 listWhereAnd.Add($"fileState IN ({IntList(fileStateAnyOf)})");
             }
 
-            string selectOutputFields = "driveMainIndex.fileId, globalTransitId, fileState, requiredSecurityGroup, fileSystemType, userDate, fileType, dataType, archivalStatus, historyStatus, senderId, groupId, uniqueId, byteCount, hdrEncryptedKeyHeader, hdrVersionTag, hdrAppData, hdrReactionSummary, hdrServerData, hdrTransferHistory, hdrFileMetaData, hdrTmpDriveAlias, hdrTmpDriveType, created, modified";
+            // string selectOutputFields    = "driveMainIndex.fileId, globalTransitId, fileState, requiredSecurityGroup, fileSystemType, userDate, fileType, dataType, archivalStatus, historyStatus, senderId, groupId, uniqueId, byteCount, hdrEncryptedKeyHeader, hdrVersionTag, hdrAppData, hdrLocalVersionTag,hdrLocalAppData,hdrReactionSummary, hdrServerData, hdrTransferHistory, hdrFileMetaData, hdrTmpDriveAlias, hdrTmpDriveType, created, modified";
+            // string selectOutputFields = "driveMainIndex.fileId, globalTransitId, fileState, requiredSecurityGroup, fileSystemType, userDate, fileType, dataType, archivalStatus, historyStatus, senderId, groupId, uniqueId, byteCount, hdrEncryptedKeyHeader, hdrVersionTag, hdrAppData,                                    hdrReactionSummary, hdrServerData, hdrTransferHistory, hdrFileMetaData, hdrTmpDriveAlias, hdrTmpDriveType, created, modified";
             /*if (fileIdSort)
                 selectOutputFields = "driveMainIndex.fileId";
             else
@@ -332,7 +365,6 @@
 
             // Read +1 more than requested to see if we're at the end of the dataset
             string stm = $"SELECT DISTINCT {selectOutputFields} FROM driveMainIndex {leftJoin} WHERE " + string.Join(" AND ", listWhereAnd) + $" ORDER BY {order} LIMIT {noOfItems + 1}";
-
             await using var cn = await scopedConnectionFactory.CreateScopedConnectionAsync();
             await using var cmd = cn.CreateCommand();
 
@@ -406,7 +438,9 @@
             UnixTimeUtcRange userdateSpan = null,
             List<Guid> aclAnyOf = null,
             List<Guid> tagsAnyOf = null,
-            List<Guid> tagsAllOf = null)
+            List<Guid> tagsAllOf = null,
+            List<Guid> localTagsAnyOf = null,
+            List<Guid> localTagsAllOf = null)
         {
             bool pagingCursorWasNull = ((cursor == null) || (cursor.pagingCursor == null));
 
@@ -428,7 +462,9 @@
                     userdateSpan,
                     aclAnyOf,
                     tagsAnyOf,
-                    tagsAllOf);
+                    tagsAllOf,
+                    localTagsAnyOf,
+                    localTagsAllOf);
 
             //
             // OldToNew:
@@ -482,7 +518,9 @@
                         userdateSpan,
                         aclAnyOf,
                         tagsAnyOf,
-                        tagsAllOf);
+                        tagsAllOf,
+                        localTagsAnyOf,
+                        localTagsAllOf);
 
                     // There was more data
                     if (r2.Count > 0)
@@ -512,7 +550,9 @@
                         uniqueIdAnyOf,
                         archivalStatusAnyOf,
                         userdateSpan,
-                        aclAnyOf, tagsAnyOf, tagsAllOf);
+                        aclAnyOf,
+                        tagsAnyOf, tagsAllOf,
+                        localTagsAnyOf, localTagsAllOf);
                 }
                 else
                 {
@@ -548,7 +588,9 @@
             UnixTimeUtcRange userdateSpan = null,
             List<Guid> aclAnyOf = null,
             List<Guid> tagsAnyOf = null,
-            List<Guid> tagsAllOf = null)
+            List<Guid> tagsAllOf = null,
+            List<Guid> localTagsAnyOf = null,
+            List<Guid> localTagsAllOf = null)
         {
             if (null == fileSystemType)
             {
@@ -575,10 +617,10 @@
             }
 
             string leftJoin = SharedWhereAnd(listWhereAnd, requiredSecurityGroup, aclAnyOf, filetypesAnyOf, datatypesAnyOf, globalTransitIdAnyOf,
-                uniqueIdAnyOf, tagsAnyOf, archivalStatusAnyOf, senderidAnyOf, groupIdAnyOf, userdateSpan, tagsAllOf,
+                uniqueIdAnyOf, tagsAnyOf, localTagsAnyOf, archivalStatusAnyOf, senderidAnyOf, groupIdAnyOf, userdateSpan, tagsAllOf, localTagsAllOf,
                 fileSystemType, driveId);
 
-            string selectOutputFields = "driveMainIndex.fileId, globalTransitId, fileState, requiredSecurityGroup, fileSystemType, userDate, fileType, dataType, archivalStatus, historyStatus, senderId, groupId, uniqueId, byteCount, hdrEncryptedKeyHeader, hdrVersionTag, hdrAppData, hdrReactionSummary, hdrServerData, hdrTransferHistory, hdrFileMetaData, hdrTmpDriveAlias, hdrTmpDriveType, created, modified";
+            // string selectOutputFields =  "driveMainIndex.fileId, globalTransitId, fileState, requiredSecurityGroup, fileSystemType, userDate, fileType, dataType, archivalStatus, historyStatus, senderId, groupId, uniqueId, byteCount, hdrEncryptedKeyHeader, hdrVersionTag, hdrAppData, hdrReactionSummary, hdrServerData, hdrTransferHistory, hdrFileMetaData, hdrTmpDriveAlias, hdrTmpDriveType, created, modified";
             string stm = $"SELECT DISTINCT {selectOutputFields} FROM drivemainindex {leftJoin} WHERE " + string.Join(" AND ", listWhereAnd) + $" ORDER BY modified ASC LIMIT {noOfItems + 1}";
             // string stm = $"SELECT DISTINCT driveMainIndex.fileid, modified FROM drivemainindex {leftJoin} WHERE " + string.Join(" AND ", listWhereAnd) + $" ORDER BY modified ASC LIMIT {noOfItems + 1}";
 
@@ -692,7 +734,7 @@
             return list?.Count > 0;
         }
 
-        private string AndIntersectHexList(List<Guid> list)
+        private string AndIntersectHexList(List<Guid> list, string tableName)
         {
             int len = list.Count;
             string s = "";
@@ -710,11 +752,11 @@
             //   SELECT DISTINCT HEX(fileid) FROM tagindex WHERE fileid in (SELECT DISTINCT fileid FROM tagindex WHERE fileid IN(SELECT DISTINCT fileid FROM tagindex WHERE tagid = x'189820F6018C218FA0F0F18E86139565') AND tagid = x'189820F6018B51349CC07ED86B02C8F6') and tagid = x'189820F6018C7F083F50CFCD32AF2B7F';
             //
 
-            s = $"driveMainIndex.fileid IN (SELECT DISTINCT fileid FROM drivetagindex WHERE tagid = {list[0].BytesToSql(_databaseType)} ";
+            s = $"driveMainIndex.fileid IN (SELECT DISTINCT fileid FROM {tableName} WHERE tagid = {list[0].BytesToSql(_databaseType)} ";
 
             for (int i = 0 + 1; i < len; i++)
             {
-                s += $"INTERSECT SELECT DISTINCT fileid FROM drivetagindex WHERE tagid = {list[i].BytesToSql(_databaseType)} ";
+                s += $"INTERSECT SELECT DISTINCT fileid FROM {tableName} WHERE tagid = {list[i].BytesToSql(_databaseType)} ";
             }
 
             s += ") ";

src/core/Odin.Core.Storage/Database/Identity/IdentityDatabase.cs

@@ -25,6 +25,7 @@ public class IdentityDatabase(ILifetimeScope lifetimeScope) : AbstractDatabase<I
         typeof(TableDriveMainIndex),
         typeof(TableDriveReactions),
         typeof(TableDriveTagIndex),
+        typeof(TableDriveLocalTagIndex),
         typeof(TableFollowsMe),
         typeof(TableImFollowing),
         typeof(TableInbox),
@@ -63,9 +64,14 @@ public class IdentityDatabase(ILifetimeScope lifetimeScope) : AbstractDatabase<I
     private Lazy<TableDriveMainIndex> _driveMainIndex;
     public TableDriveMainIndex DriveMainIndex => LazyResolve(ref _driveMainIndex);
     private Lazy<TableDriveReactions> _driveReactions;
-    public TableDriveReactions DriveReactions => LazyResolve(ref _driveReactions);
+
     private Lazy<TableDriveTagIndex> _driveTagIndex;
     public TableDriveTagIndex DriveTagIndex => LazyResolve(ref _driveTagIndex);
+
+    private Lazy<TableDriveLocalTagIndex> _driveLocalTagIndex;
+    public TableDriveLocalTagIndex DriveLocalTagIndex => LazyResolve(ref _driveLocalTagIndex);
+
+    public TableDriveReactions DriveReactions => LazyResolve(ref _driveReactions);
     private Lazy<TableFollowsMe> _followsMe;
     public TableFollowsMe FollowsMe => LazyResolve(ref _followsMe);
     private Lazy<TableImFollowing> _imFollowing;

src/core/Odin.Core.Storage/Database/Identity/Table/TableAppGrantsCRUD.cs

@@ -69,6 +69,16 @@ public byte[] data
                   _data = value;
                }
         }
+        internal byte[] dataNoLengthCheck
+        {
+           get {
+                   return _data;
+               }
+           set {
+                    if (value?.Length < 0) throw new Exception("Too short");
+                  _data = value;
+               }
+        }
     } // End of class AppGrantsRecord
 
     public abstract class TableAppGrantsCRUD
@@ -288,7 +298,7 @@ protected virtual async Task<int> GetCountAsync()
             }
         }
 
-        public List<string> GetColumnNames()
+        public static List<string> GetColumnNames()
         {
             var sl = new List<string>();
             sl.Add("identityId");
@@ -309,16 +319,11 @@ protected AppGrantsRecord ReadRecordFromReaderAll(DbDataReader rdr)
 #pragma warning restore CS0168
             var guid = new byte[16];
             var item = new AppGrantsRecord();
-            item.identityId = rdr.IsDBNull(0) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[0]);
-            item.odinHashId = rdr.IsDBNull(1) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[1]);
-            item.appId = rdr.IsDBNull(2) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[2]);
-            item.circleId = rdr.IsDBNull(3) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[3]);
-            item.data = rdr.IsDBNull(4) ? 
-                null : (byte[])(rdr[4]);
+            item.identityId = rdr.IsDBNull(0) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[0]);
+            item.odinHashId = rdr.IsDBNull(1) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[1]);
+            item.appId = rdr.IsDBNull(2) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[2]);
+            item.circleId = rdr.IsDBNull(3) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[3]);
+            item.dataNoLengthCheck = rdr.IsDBNull(4) ? null : (byte[])(rdr[4]);
             if (item.data?.Length > 65535)
                 throw new Exception("Too much data in data...");
             if (item.data?.Length < 0)
@@ -368,15 +373,9 @@ protected AppGrantsRecord ReadRecordFromReader0(DbDataReader rdr, Guid identityI
             var item = new AppGrantsRecord();
             item.identityId = identityId;
             item.odinHashId = odinHashId;
-
-            item.appId = rdr.IsDBNull(0) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[0]);
-
-            item.circleId = rdr.IsDBNull(1) ? 
-                throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[1]);
-
-            item.data = rdr.IsDBNull(2) ? 
-                null : (byte[])(rdr[2]);
+            item.appId = rdr.IsDBNull(0) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[0]);
+            item.circleId = rdr.IsDBNull(1) ? throw new Exception("item is NULL, but set as NOT NULL") : new Guid((byte[])rdr[1]);
+            item.dataNoLengthCheck = rdr.IsDBNull(2) ? null : (byte[])(rdr[2]);
             if (item.data?.Length > 65535)
                 throw new Exception("Too much data in data...");
             if (item.data?.Length < 0)
@@ -434,9 +433,7 @@ protected AppGrantsRecord ReadRecordFromReader1(DbDataReader rdr, Guid identityI
             item.odinHashId = odinHashId;
             item.appId = appId;
             item.circleId = circleId;
-
-            item.data = rdr.IsDBNull(0) ? 
-                null : (byte[])(rdr[0]);
+            item.dataNoLengthCheck = rdr.IsDBNull(0) ? null : (byte[])(rdr[0]);
             if (item.data?.Length > 65535)
                 throw new Exception("Too much data in data...");
             if (item.data?.Length < 0)