From 8832ed5346489f88b1da00178627c99a54f0c216 Mon Sep 17 00:00:00 2001 From: cce <51567+cce@users.noreply.github.com> Date: Wed, 29 Jan 2025 14:52:18 -0500 Subject: [PATCH] tools: add -d and -o args for catchpointdump net, like file, new info command (#6235) --- cmd/catchpointdump/commands.go | 1 + cmd/catchpointdump/info.go | 274 +++++++++++++++++++++++++++++++++ cmd/catchpointdump/net.go | 11 +- 3 files changed, 284 insertions(+), 2 deletions(-) create mode 100644 cmd/catchpointdump/info.go diff --git a/cmd/catchpointdump/commands.go b/cmd/catchpointdump/commands.go index f6ec14efe9..ba68869b61 100644 --- a/cmd/catchpointdump/commands.go +++ b/cmd/catchpointdump/commands.go @@ -44,6 +44,7 @@ func init() { rootCmd.AddCommand(fileCmd) rootCmd.AddCommand(netCmd) rootCmd.AddCommand(databaseCmd) + rootCmd.AddCommand(infoCmd) } var rootCmd = &cobra.Command{ diff --git a/cmd/catchpointdump/info.go b/cmd/catchpointdump/info.go new file mode 100644 index 0000000000..ea62434f26 --- /dev/null +++ b/cmd/catchpointdump/info.go @@ -0,0 +1,274 @@ +// Copyright (C) 2019-2025 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package main + +import ( + "bufio" + "context" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/network" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util" +) + +var infoFile string + +func init() { + infoCmd.Flags().StringVarP(&infoFile, "tar", "t", "", "Specify the catchpoint file (.tar or .tar.gz) to read") + infoCmd.Flags().StringVarP(&networkName, "net", "n", "", "Specify the network name (e.g. mainnet.algorand.network)") + infoCmd.Flags().IntVarP(&round, "round", "r", 0, "Specify the round number (e.g. 7700000). Only used if --relay/-p is given.") + infoCmd.Flags().StringVarP(&relayAddress, "relay", "p", "", "Relay address to download from (e.g. r-ru.algorand-mainnet.network:4160). If specified, fetch instead of reading local --tar.") +} + +// infoCmd defines a new cobra command that only loads and prints the CatchpointFileHeader. +var infoCmd = &cobra.Command{ + Use: "info", + Short: "Show header info from a catchpoint tar file", + Long: "Reads the specified catchpoint tar (or tar.gz) file, locates the content.json block, and prints the CatchpointFileHeader fields without loading the entire ledger.", + Args: validateNoPosArgsFn, + Run: func(cmd *cobra.Command, args []string) { + // If user gave us a relay, stream from the network: + if relayAddress != "" { + // If they gave a relay, they must also give us a valid network and round + if networkName == "" || round == 0 { + cmd.HelpFunc()(cmd, args) + reportErrorf("Must specify --net and --round when using --relay") + } + // Attempt to read the CatchpointFileHeader from the network stream + fileHeader, err := loadCatchpointFileHeaderFromRelay(relayAddress, networkName, round) + if err != nil { + reportErrorf("Error streaming CatchpointFileHeader from relay %s: %v", relayAddress, err) + } + if fileHeader.Version == 0 { + fmt.Printf("No valid header was found streaming from relay '%s'.\n", relayAddress) + return + } + fmt.Printf("Relay: %s\n", relayAddress) + printHeaderFields(fileHeader) + return + } + + // Otherwise, fallback to local file usage: + if infoFile == "" { + cmd.HelpFunc()(cmd, args) + return + } + fi, err := os.Stat(infoFile) + if err != nil { + reportErrorf("Unable to stat file '%s': %v", infoFile, err) + } + if fi.Size() == 0 { + reportErrorf("File '%s' is empty.", infoFile) + } + + // Open the catchpoint file + f, err := os.Open(infoFile) + if err != nil { + reportErrorf("Unable to open file '%s': %v", infoFile, err) + } + defer f.Close() + + // Extract just the file header + fileHeader, err := loadCatchpointFileHeader(f, fi.Size()) + if err != nil { + reportErrorf("Error reading CatchpointFileHeader from '%s': %v", infoFile, err) + } + + // Print out the fields (mimicking the logic in printAccountsDatabase, but simpler) + if fileHeader.Version == 0 { + fmt.Printf("No valid header was found.\n") + return + } + + printHeaderFields(fileHeader) + }, +} + +func printHeaderFields(fileHeader ledger.CatchpointFileHeader) { + fmt.Printf("Version: %d\n", fileHeader.Version) + fmt.Printf("Balances Round: %d\n", fileHeader.BalancesRound) + fmt.Printf("Block Round: %d\n", fileHeader.BlocksRound) + fmt.Printf("Block Header Digest: %s\n", fileHeader.BlockHeaderDigest.String()) + fmt.Printf("Catchpoint: %s\n", fileHeader.Catchpoint) + fmt.Printf("Total Accounts: %d\n", fileHeader.TotalAccounts) + fmt.Printf("Total KVs: %d\n", fileHeader.TotalKVs) + fmt.Printf("Total Online Accounts: %d\n", fileHeader.TotalOnlineAccounts) + fmt.Printf("Total Online Round Params: %d\n", fileHeader.TotalOnlineRoundParams) + fmt.Printf("Total Chunks: %d\n", fileHeader.TotalChunks) + + totals := fileHeader.Totals + fmt.Printf("AccountTotals - Online Money: %d\n", totals.Online.Money.Raw) + fmt.Printf("AccountTotals - Online RewardUnits: %d\n", totals.Online.RewardUnits) + fmt.Printf("AccountTotals - Offline Money: %d\n", totals.Offline.Money.Raw) + fmt.Printf("AccountTotals - Offline RewardUnits: %d\n", totals.Offline.RewardUnits) + fmt.Printf("AccountTotals - Not Participating Money: %d\n", totals.NotParticipating.Money.Raw) + fmt.Printf("AccountTotals - Not Participating RewardUnits: %d\n", totals.NotParticipating.RewardUnits) + fmt.Printf("AccountTotals - Rewards Level: %d\n", totals.RewardsLevel) +} + +// loadCatchpointFileHeader reads only enough of the tar (or tar.gz) to +// decode the ledger.CatchpointFileHeader from the "content.json" chunk. +func loadCatchpointFileHeader(catchpointFile io.Reader, catchpointFileSize int64) (ledger.CatchpointFileHeader, error) { + var fileHeader ledger.CatchpointFileHeader + fmt.Printf("Scanning for CatchpointFileHeader in tar...\n\n") + + catchpointReader := bufio.NewReader(catchpointFile) + tarReader, _, err := getCatchpointTarReader(catchpointReader, catchpointFileSize) + if err != nil { + return fileHeader, err + } + + for { + hdr, err := tarReader.Next() + if err != nil { + if err == io.EOF { + // We reached the end without finding content.json + break + } + return fileHeader, err + } + + // We only need the "content.json" file + if hdr.Name == ledger.CatchpointContentFileName { + // Read exactly hdr.Size bytes + buf := make([]byte, hdr.Size) + _, readErr := io.ReadFull(tarReader, buf) + if readErr != nil && readErr != io.EOF { + return fileHeader, readErr + } + + // Decode into fileHeader + readErr = protocol.Decode(buf, &fileHeader) + if readErr != nil { + return fileHeader, readErr + } + // Once we have the fileHeader, we can break out. + // If you wanted to keep scanning, you could keep going, + // but it’s not needed just for the header. + return fileHeader, nil + } + + // Otherwise skip this chunk + skipBytes := hdr.Size + n, err := io.Copy(io.Discard, tarReader) + if err != nil { + return fileHeader, err + } + + // skip any leftover in case we didn't read the entire chunk + if skipBytes > n { + // keep discarding until we've skipped skipBytes total + _, err := io.CopyN(io.Discard, tarReader, skipBytes-n) + if err != nil { + return fileHeader, err + } + } + } + // If we get here, we never found the content.json entry + return fileHeader, nil +} + +// loadCatchpointFileHeaderFromRelay opens a streaming HTTP connection to the +// given relay for the given round, then scans the (possibly gzip) tar stream +// until it finds `content.json`, decodes the ledger.CatchpointFileHeader, and +// immediately closes the network connection (so we don't download the entire file). +func loadCatchpointFileHeaderFromRelay(relay string, netName string, round int) (ledger.CatchpointFileHeader, error) { + var fileHeader ledger.CatchpointFileHeader + + // Create an HTTP GET to the relay + genesisID := strings.Split(netName, ".")[0] + "-v1.0" + urlTemplate := "http://" + relay + "/v1/" + genesisID + "/%s/" + strconv.FormatUint(uint64(round), 36) + catchpointURL := fmt.Sprintf(urlTemplate, "ledger") + + req, err := http.NewRequest(http.MethodGet, catchpointURL, nil) + if err != nil { + return fileHeader, err + } + // Add a short-ish timeout or rely on default + ctx, cancelFn := context.WithTimeout(context.Background(), 60*time.Second) + defer cancelFn() + req = req.WithContext(ctx) + network.SetUserAgentHeader(req.Header) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fileHeader, err + } + if resp.StatusCode != http.StatusOK { + // e.g. 404 if not found + return fileHeader, fmt.Errorf("HTTP status code %d from relay", resp.StatusCode) + } + defer resp.Body.Close() + + // Wrap with a small "watchdog" so we don't hang if data stops flowing + wdReader := util.MakeWatchdogStreamReader(resp.Body, 4096, 4096, 5*time.Second) + defer wdReader.Close() + + // Use isGzip logic from file.go + // We have to peek the first 2 bytes to see if it's gz + peekReader := bufio.NewReader(wdReader) + // We'll fake a size of "unknown" since we don't truly know the length + tarReader, _, err := getCatchpointTarReader(peekReader, -1 /* unknown size */) + if err != nil { + return fileHeader, err + } + + // Now read each tar entry, ignoring everything except "content.json" + for { + hdr, err := tarReader.Next() + if err != nil { + if err == io.EOF { + // finished the entire tar stream + break + } + return fileHeader, err + } + if hdr.Name == ledger.CatchpointContentFileName { + // We only need "content.json" + buf := make([]byte, hdr.Size) + _, readErr := io.ReadFull(tarReader, buf) + if readErr != nil && readErr != io.EOF { + return fileHeader, readErr + } + + // decode + decodeErr := protocol.Decode(buf, &fileHeader) + if decodeErr != nil { + return fileHeader, decodeErr + } + // Done! We can return immediately. + return fileHeader, nil + } + // If not content.json, skip over this tar chunk + _, err = io.Copy(io.Discard, tarReader) + if err != nil { + return fileHeader, err + } + } + // If we exit the loop, we never found content.json + return fileHeader, nil +} diff --git a/cmd/catchpointdump/net.go b/cmd/catchpointdump/net.go index 079d8e2e9b..24a6ccfe65 100644 --- a/cmd/catchpointdump/net.go +++ b/cmd/catchpointdump/net.go @@ -60,6 +60,8 @@ func init() { netCmd.Flags().BoolVarP(&singleCatchpoint, "single", "s", true, "Download/process only from a single relay") netCmd.Flags().BoolVarP(&loadOnly, "load", "l", false, "Load only, do not dump") netCmd.Flags().VarP(excludedFields, "exclude-fields", "e", "List of fields to exclude from the dump: ["+excludedFields.AllowedString()+"]") + netCmd.Flags().StringVarP(&outFileName, "output", "o", "", "Specify an outfile for the dump ( i.e. tracker.dump.txt )") + netCmd.Flags().BoolVarP(&printDigests, "digest", "d", false, "Print balances and spver digests") } var netCmd = &cobra.Command{ @@ -103,7 +105,7 @@ var netCmd = &cobra.Command{ reportInfof("failed to load/dump from tar file for '%s' : %v", addr, err) continue } - // clear possible errors from previous run: at this point we've been succeed + // clear possible errors from previous run: at this point we've succeeded err = nil if singleCatchpoint { // a catchpoint processes successfully, exit if needed @@ -348,7 +350,12 @@ func loadAndDump(addr string, tarFile string, genesisInitState ledgercore.InitSt if !loadOnly { dirName := "./" + strings.Split(networkName, ".")[0] + "/" + strings.Split(addr, ".")[0] - outFile, err := os.OpenFile(dirName+"/"+strconv.FormatUint(uint64(round), 10)+".dump", os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0755) + // If user provided -o , use that; otherwise use /.dump + dumpFilename := outFileName + if dumpFilename == "" { + dumpFilename = dirName + "/" + strconv.FormatUint(uint64(round), 10) + ".dump" + } + outFile, err := os.OpenFile(dumpFilename, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0755) if err != nil { return err }