Increase buffer size

streamingfast · Sep 29, 2022 · b2f5e50 · b2f5e50
1 parent 1399f04
commit b2f5e50
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 7 deletions.
diff --git a/codec/consolereader-bt.go b/codec/consolereader-bt.go
@@ -48,9 +48,9 @@ func (cr *BigtableConsoleReader) ProcessData(reader io.Reader) error {
 }
 
 func (cr *BigtableConsoleReader) buildScanner(reader io.Reader) *bufio.Scanner {
-	buf := make([]byte, 50*1024*1024)
+	buf := make([]byte, 100*1024*1024)
 	scanner := bufio.NewScanner(reader)
-	scanner.Buffer(buf, 50*1024*1024)
+	scanner.Buffer(buf, len(buf))
 
 	return scanner
 }

diff --git a/node-manager/superviser.go b/node-manager/superviser.go
@@ -49,8 +49,8 @@ type Options struct {
 }
 
 func NewSuperviser(supervisorName string, appLogger *zap.Logger, nodelogger *zap.Logger, options *Options) (*Superviser, error) {
-	// Ensure process manager line buffer is large enough (50 MiB) for our Firehose instrumentation outputting lot's of text.
-	overseer.DEFAULT_LINE_BUFFER_SIZE = 50 * 1024 * 1024
+	// Ensure process manager line buffer is large enough (100 MiB) for our Firehose instrumentation outputting lot's of text.
+	overseer.DEFAULT_LINE_BUFFER_SIZE = 100 * 1024 * 1024
 
 	client := rpc.NewClient(fmt.Sprintf("http://127.0.0.1:%s", options.RCPPort))
 	s := &Superviser{

diff --git a/tools/bigtable-blocks.go b/tools/bigtable-blocks.go
@@ -7,11 +7,10 @@ import (
 	"strconv"
 
 	"github.com/spf13/viper"
+	"github.com/streamingfast/firehose-solana/bt"
 	pbsolv1 "github.com/streamingfast/firehose-solana/types/pb/sf/solana/type/v1"
 	"google.golang.org/protobuf/proto"
 
-	"github.com/streamingfast/firehose-solana/bt"
-
 	"cloud.google.com/go/bigtable"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
@@ -67,7 +66,10 @@ func bigtableBlocksRunE(cmd *cobra.Command, args []string) error {
 				return fmt.Errorf("failed to proto  marshal pb sol block: %w", err)
 			}
 
-			fmt.Printf("FIRE BLOCK %d %s\n", block.Slot, hex.EncodeToString(cnt))
+			lineCnt := fmt.Sprintf("FIRE BLOCK %d %s", block.Slot, hex.EncodeToString(cnt))
+			if _, err := fmt.Println(lineCnt); err != nil {
+				return fmt.Errorf("failed to write log line (char lenght %d): %w", len(lineCnt), err)
+			}
 			return nil
 		}