pharr117/filter and logic improvements

.github/workflows/go-test.yaml

@@ -11,7 +11,7 @@ on:
       - main
 
 env:
-  GO_VERSION: '^1.19'
+  GO_VERSION: '^1.22'
 
 jobs:
   go-test:

.github/workflows/golangci-lint.yaml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.19'
+          go-version: '1.22'
           cache: false
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3

.github/workflows/pre-commit.yaml

@@ -10,7 +10,7 @@ on:
     branches:
       - main
 env:
-  GO_VERSION: '^1.19'
+  GO_VERSION: '^1.22'
 
 jobs:
   pre-commit:

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
     rev: master
     hooks:
       - id: go-fmt
-      - id: go-vet
+      # - id: go-vet
       # - id: go-lint
       - id: go-imports
       # - id: go-cyclo

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19-alpine3.16 AS build-env
+FROM golang:1.22-alpine3.18 AS build-env
 
 # Customize to your build env
 
@@ -67,8 +67,8 @@ COPY --from=build-env /usr/lib/libonig.so.5 /lib
 COPY --from=build-env /usr/lib/libcurl.so.4 /lib
 COPY --from=build-env /lib/libz.so.1 /lib
 COPY --from=build-env /usr/lib/libnghttp2.so.14 /lib
-COPY --from=build-env /lib/libssl.so.1.1 /lib
-COPY --from=build-env /lib/libcrypto.so.1.1 /lib
+COPY --from=build-env /lib/libssl.so.3 /lib
+COPY --from=build-env /lib/libcrypto.so.3 /lib
 COPY --from=build-env /usr/lib/libbrotlidec.so.1 /lib
 COPY --from=build-env /usr/lib/libbrotlicommon.so.1 /lib
 

README.md

@@ -23,7 +23,7 @@ Before you can start indexing a chain, you need to set up the application's depe
 The application requires a PostgreSQL server with an established database and an owner user/role with password login. Here's a simple example of setting up a containerized database locally [here](https://towardsdatascience.com/local-development-set-up-of-postgresql-with-docker-c022632f13ea).
 
 #### Go
-The application is written in Go, so you need to build it from source. This requires a system installation of at minimum Go 1.19. Instructions for installing and configuring Go can be found [here](https://go.dev/doc/install).
+The application is written in Go, so you need to build it from source. This requires a system installation of at minimum Go 1.22. Instructions for installing and configuring Go can be found [here](https://go.dev/doc/install).
 
 ## Indexing and Querying
 

cmd/index.go

@@ -147,7 +147,6 @@ func setupIndex(cmd *cobra.Command, args []string) error {
 			indexer.BlockEventFilterRegistries.EndBlockEventFilterRegistry.RollingWindowEventFilters,
 			fileMessageTypeFilters,
 			err = config.ParseJSONFilterConfig(b)
-
 		if err != nil {
 			safeCleanupSetupExit(&indexer)
 			config.Log.Fatal("Failed to parse block event filter config", err)
@@ -233,10 +232,10 @@ func setupIndexer() *indexerPackage.Indexer {
 
 	if indexer.PostSetupCustomFunction != nil {
 		err = indexer.PostSetupCustomFunction(indexerPackage.PostSetupCustomDataset{
-			Config: *indexer.Config,
-			DB:     indexer.DB,
+			ChainClient: indexer.ChainClient,
+			Config:      *indexer.Config,
+			DB:          indexer.DB,
 		})
-
 		if err != nil {
 			config.Log.Fatal("Failed to run post setup custom function", err)
 		}
@@ -343,7 +342,6 @@ func index(cmd *cobra.Command, args []string) {
 			DB:     indexer.DB,
 			DryRun: indexer.DryRun,
 		})
-
 		if err != nil {
 			config.Log.Fatal("Failed to run pre-exit custom function", err)
 		}

config/filter_config.go

@@ -199,7 +199,6 @@ func ParseTXMessageTypeConfig(messageTypeConfigs []json.RawMessage) ([]filter.Me
 		}
 
 		err = validateMessageTypeFilterConfig(newFilter)
-
 		if err != nil {
 			parserError := fmt.Errorf("error parsing filter at index %d: %s", index, err)
 			return nil, parserError
@@ -226,8 +225,7 @@ func ParseTXMessageTypeConfig(messageTypeConfigs []json.RawMessage) ([]filter.Me
 				return nil, err
 			}
 
-			newFilter, err = filter.NewRegexMessageTypeFilter(newFilter.MessageTypeRegexPattern)
-
+			newFilter, err = filter.NewRegexMessageTypeFilter(newFilter.MessageTypeRegexPattern, newFilter.ShouldIgnore)
 			if err != nil {
 				parserError := fmt.Errorf("error parsing filter at index %d: %s", index, err)
 				return nil, parserError

config/index_config.go

@@ -19,26 +19,28 @@ type IndexConfig struct {
 type indexBase struct {
 	throttlingBase
 	retryBase
-	ReindexMessageType         string `mapstructure:"reindex-message-type"`
-	ReattemptFailedBlocks      bool   `mapstructure:"reattempt-failed-blocks"`
-	StartBlock                 int64  `mapstructure:"start-block"`
-	EndBlock                   int64  `mapstructure:"end-block"`
-	BlockInputFile             string `mapstructure:"block-input-file"`
-	ReIndex                    bool   `mapstructure:"reindex"`
-	RPCWorkers                 int64  `mapstructure:"rpc-workers"`
-	BlockTimer                 int64  `mapstructure:"block-timer"`
-	WaitForChain               bool   `mapstructure:"wait-for-chain"`
-	WaitForChainDelay          int64  `mapstructure:"wait-for-chain-delay"`
-	TransactionIndexingEnabled bool   `mapstructure:"index-transactions"`
-	ExitWhenCaughtUp           bool   `mapstructure:"exit-when-caught-up"`
-	BlockEventIndexingEnabled  bool   `mapstructure:"index-block-events"`
-	FilterFile                 string `mapstructure:"filter-file"`
-	Dry                        bool   `mapstructure:"dry"`
+	ReindexMessageType          string `mapstructure:"reindex-message-type"`
+	ReattemptFailedBlocks       bool   `mapstructure:"reattempt-failed-blocks"`
+	StartBlock                  int64  `mapstructure:"start-block"`
+	EndBlock                    int64  `mapstructure:"end-block"`
+	BlockInputFile              string `mapstructure:"block-input-file"`
+	ReIndex                     bool   `mapstructure:"reindex"`
+	RPCWorkers                  int64  `mapstructure:"rpc-workers"`
+	SkipBlockByHeightRPCRequest bool   `mapstructure:"skip-block-by-height-rpc-request"`
+	BlockTimer                  int64  `mapstructure:"block-timer"`
+	WaitForChain                bool   `mapstructure:"wait-for-chain"`
+	WaitForChainDelay           int64  `mapstructure:"wait-for-chain-delay"`
+	TransactionIndexingEnabled  bool   `mapstructure:"index-transactions"`
+	ExitWhenCaughtUp            bool   `mapstructure:"exit-when-caught-up"`
+	BlockEventIndexingEnabled   bool   `mapstructure:"index-block-events"`
+	FilterFile                  string `mapstructure:"filter-file"`
+	Dry                         bool   `mapstructure:"dry"`
 }
 
 // Flags for specific, deeper indexing behavior
 type flags struct {
 	IndexTxMessageRaw        bool `mapstructure:"index-tx-message-raw"`
+	IndexEmptyTransactions   bool `mapstructure:"index-empty-transactions"`
 	BlockEventsBase64Encoded bool `mapstructure:"block-events-base64-encoded"`
 	IndexMessageEvents       bool `mapstructure:"index-message-events"`
 }
@@ -59,6 +61,7 @@ func SetupIndexSpecificFlags(conf *IndexConfig, cmd *cobra.Command) {
 	// other base setting
 	cmd.PersistentFlags().BoolVar(&conf.Base.Dry, "base.dry", false, "index the chain but don't insert data in the DB.")
 	cmd.PersistentFlags().Int64Var(&conf.Base.RPCWorkers, "base.rpc-workers", 1, "the number of concurrent RPC request workers to spin up.")
+	cmd.PersistentFlags().BoolVar(&conf.Base.SkipBlockByHeightRPCRequest, "base.skip-block-by-height-rpc-request", false, "skip the /block?height=<height> RPC request and only attempt the /block_results RPC request. Sometimes pruned nodes will not have return results for the block RPC request, but still return results for the block_result request.")
 	cmd.PersistentFlags().BoolVar(&conf.Base.WaitForChain, "base.wait-for-chain", false, "wait for chain to be in sync?")
 	cmd.PersistentFlags().Int64Var(&conf.Base.WaitForChainDelay, "base.wait-for-chain-delay", 10, "seconds to wait between each check for node to catch up to the chain")
 	cmd.PersistentFlags().Int64Var(&conf.Base.BlockTimer, "base.block-timer", 10000, "print out how long it takes to process this many blocks")
@@ -68,6 +71,7 @@ func SetupIndexSpecificFlags(conf *IndexConfig, cmd *cobra.Command) {
 
 	// flags
 	cmd.PersistentFlags().BoolVar(&conf.Flags.IndexTxMessageRaw, "flags.index-tx-message-raw", false, "if true, this will index the raw message bytes. This will significantly increase the size of the database.")
+	cmd.PersistentFlags().BoolVar(&conf.Flags.IndexEmptyTransactions, "flags.index-empty-transactions", true, "if true, this will index transactions that have no messages. Setting this to false when filtering TX message types will result in no transactions being indexed if all message types are filtered out.")
 	cmd.PersistentFlags().BoolVar(&conf.Flags.BlockEventsBase64Encoded, "flags.block-events-base64-encoded", false, "if true, decode the block event attributes and keys as base64. Some versions of CometBFT encode the block event attributes and keys as base64 in the response from RPC.")
 	cmd.PersistentFlags().BoolVar(&conf.Flags.IndexMessageEvents, "flags.index-message-events", true, "if true, skip indexing message events if they are uneeded. This will save space in the database.")
 }
@@ -81,15 +85,13 @@ func (conf *IndexConfig) Validate() error {
 	probeConf := conf.Probe
 
 	probeConf, err = validateProbeConf(probeConf)
-
 	if err != nil {
 		return err
 	}
 
 	conf.Probe = probeConf
 
 	err = validateThrottlingConf(conf.Base.throttlingBase)
-
 	if err != nil {
 		return err
 	}

core/block_enqueue.go

@@ -34,7 +34,6 @@ func GenerateBlockFileEnqueueFunction(db *gorm.DB, cfg config.IndexConfig, clien
 		}
 		var blocksToIndex []uint64
 		err = json.Unmarshal(plan, &blocksToIndex)
-
 		if err != nil {
 			errString := err.Error()
 
@@ -220,7 +219,6 @@ func GenerateDefaultEnqueueFunction(db *gorm.DB, cfg config.IndexConfig, client
 		config.Log.Info("Reindexing is disabled, skipping blocks that have already been indexed")
 		// We need to pick up where we last left off, find blocks after start and skip already indexed blocks
 		blocksFromStart, err = dbTypes.GetBlocksFromStart(db, chainID, startBlock, endBlock)
-
 		if err != nil {
 			return nil, err
 		}

core/block_events.go

@@ -23,13 +23,11 @@ func ProcessRPCBlockResults(conf config.IndexConfig, block models.Block, blockRe
 
 	var err error
 	blockDBWrapper.BeginBlockEvents, err = ProcessRPCBlockEvents(blockDBWrapper.Block, blockResults.BeginBlockEvents, models.BeginBlockEvent, blockDBWrapper.UniqueBlockEventTypes, blockDBWrapper.UniqueBlockEventAttributeKeys, customBeginBlockParsers, conf)
-
 	if err != nil {
 		return nil, err
 	}
 
 	blockDBWrapper.EndBlockEvents, err = ProcessRPCBlockEvents(blockDBWrapper.Block, blockResults.EndBlockEvents, models.EndBlockEvent, blockDBWrapper.UniqueBlockEventTypes, blockDBWrapper.UniqueBlockEventAttributeKeys, customEndBlockParsers, conf)
-
 	if err != nil {
 		return nil, err
 	}

core/rpc_worker.go

@@ -83,9 +83,13 @@ func BlockRPCWorker(wg *sync.WaitGroup, blockEnqueueChan chan *EnqueueData, chai
 		}
 
 		if block.IndexTransactions {
-			txsEventResp, err := rpc.GetTxsByBlockHeight(chainClient, block.Height)
+			var txsEventResp *txTypes.GetTxsEventResponse
+			var err error
+			if !cfg.Base.SkipBlockByHeightRPCRequest {
+				txsEventResp, err = rpc.GetTxsByBlockHeight(chainClient, block.Height)
+			}
 
-			if err != nil {
+			if err != nil || cfg.Base.SkipBlockByHeightRPCRequest {
 				// Attempt to get block results to attempt an in-app codec decode of transactions.
 				if currentHeightIndexerData.BlockResultsData == nil {