Mark index meta as deleted when segment is compacted (milvus-io#19163)

Signed-off-by: cai.zhang <[email protected]> Signed-off-by: cai.zhang <[email protected]>
sunby · Sep 15, 2022 · 525201d · 525201d
1 parent e72e722
commit 525201d
Show file tree

Hide file tree

Showing 10 changed files with 968 additions and 1,024 deletions.
diff --git a/internal/indexcoord/flush_segment_watcher.go b/internal/indexcoord/flush_segment_watcher.go
diff --git a/internal/indexcoord/flushed_segment_watcher_test.go b/internal/indexcoord/flushed_segment_watcher_test.go
diff --git a/internal/indexcoord/garbage_collector.go b/internal/indexcoord/garbage_collector.go
@@ -22,12 +22,13 @@ import (
 	"sync"
 	"time"
 
+	"github.com/milvus-io/milvus/internal/proto/commonpb"
+	"github.com/milvus-io/milvus/internal/proto/datapb"
+
 	"go.uber.org/zap"
 
 	"github.com/milvus-io/milvus/internal/common"
 	"github.com/milvus-io/milvus/internal/log"
-	"github.com/milvus-io/milvus/internal/proto/commonpb"
-	"github.com/milvus-io/milvus/internal/proto/datapb"
 	"github.com/milvus-io/milvus/internal/storage"
 )
 
@@ -39,8 +40,6 @@ type garbageCollector struct {
 	gcFileDuration time.Duration
 	gcMetaDuration time.Duration
 
-	notify chan struct{}
-
 	metaTable        *metaTable
 	chunkManager     storage.ChunkManager
 	indexCoordClient *IndexCoord
@@ -75,13 +74,6 @@ func (gc *garbageCollector) Stop() {
 	gc.wg.Wait()
 }
 
-func (gc *garbageCollector) Notify() {
-	select {
-	case gc.notify <- struct{}{}:
-	default:
-	}
-}
-
 func (gc *garbageCollector) recycleUnusedIndexes() {
 	defer gc.wg.Done()
 	log.Info("IndexCoord garbageCollector recycleUnusedIndexes start")
@@ -122,6 +114,8 @@ func (gc *garbageCollector) recycleUnusedIndexes() {
 								zap.Int64("nodeID", segIdx.NodeID), zap.Error(err))
 							continue
 						}
+						log.Info("IndexCoord remove segment index meta success", zap.Int64("buildID", segIdx.BuildID),
+							zap.Int64("nodeID", segIdx.NodeID))
 					}
 				}
 			}
@@ -170,11 +164,9 @@ func (gc *garbageCollector) recycleSegIndexesMeta() {
 			}
 		}
 	}
-
+	//segIndexes := gc.metaTable.GetDeletedSegmentIndexes()
 	for _, meta := range segIndexes {
 		if meta.IsDeleted || gc.metaTable.IsIndexDeleted(meta.CollectionID, meta.IndexID) {
-			log.Info("index meta need to recycle it", zap.Int64("buildID", meta.BuildID),
-				zap.Int64("nodeID", meta.NodeID))
 			if meta.NodeID != 0 {
 				// wait for releasing reference lock
 				continue
@@ -187,6 +179,7 @@ func (gc *garbageCollector) recycleSegIndexesMeta() {
 					zap.Int64("nodeID", meta.NodeID), zap.Error(err))
 				continue
 			}
+			log.Debug("index meta recycle success", zap.Int64("buildID", meta.BuildID))
 		}
 	}
 }
@@ -205,8 +198,6 @@ func (gc *garbageCollector) recycleUnusedSegIndexes() {
 			return
 		case <-ticker.C:
 			gc.recycleSegIndexesMeta()
-		case <-gc.notify:
-			gc.recycleSegIndexesMeta()
 		}
 	}
 }