Skip to content

Commit

Permalink
Merge branch 'master' into kayan_1.0_to_master
Browse files Browse the repository at this point in the history
  • Loading branch information
@taokayan
taokayan committed Jan 14, 2025
2 parents 7cc1d0b + 8095933 commit b30fd8b
Show file tree
Hide file tree
Showing 3 changed files with 35 additions and 25 deletions.
3 changes: 2 additions & 1 deletion silkworm/silkrpc/commands/eth_api.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2142,7 +2142,8 @@ awaitable<void> EthereumRpcApi::get_logs(ethdb::TransactionDatabase& tx_database
FilterAddresses& addresses, FilterTopics& topics, std::vector<Log>& logs) {
SILK_INFO << "start block: " << start << " end block: " << end;

roaring::Roaring64Map block_numbers(roaring::api::roaring_bitmap_from_range(start, end+1, 1));
roaring::Roaring block_numbers;
block_numbers.addRange(start, end + 1); // [min, max)

SILK_DEBUG << "block_numbers.cardinality(): " << block_numbers.cardinality();

Expand Down
49 changes: 29 additions & 20 deletions silkworm/silkrpc/ethdb/bitmap.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,37 +25,46 @@
#include <gsl/narrow>

#include <silkworm/infra/common/log.hpp>
#include <silkworm/node/db/bitmap.hpp>

namespace silkworm::rpc::ethdb::bitmap {

using roaring_bitmap_t = roaring::api::roaring_bitmap_t;
using Roaring64Map = roaring::Roaring64Map;
using Roaring = roaring::Roaring;

static Roaring64Map fast_or(size_t n, const std::vector<std::unique_ptr<Roaring64Map>>& inputs) {
Roaring64Map result;
static Roaring fast_or(size_t n, const std::vector<std::unique_ptr<Roaring>>& inputs) {
const auto** x = static_cast<const roaring_bitmap_t**>(malloc(n * sizeof(roaring_bitmap_t*)));
if (x == nullptr) {
throw std::runtime_error("failed memory alloc in fast_or");
}
for (size_t k = 0; k < n; ++k) {
result |= *inputs[k];
x[k] = &inputs[k]->roaring;
}

roaring_bitmap_t* c_ans = roaring_bitmap_or_many(n, x);
if (c_ans == nullptr) {
free(x);
throw std::runtime_error("failed memory alloc in fast_or");
}
return result;
Roaring ans(c_ans);
free(x);
return ans;
}

awaitable<Roaring64Map> get(core::rawdb::DatabaseReader& db_reader, const std::string& table, silkworm::Bytes& key,
awaitable<Roaring> get(core::rawdb::DatabaseReader& db_reader, const std::string& table, silkworm::Bytes& key,
uint32_t from_block, uint32_t to_block) {
std::vector<std::unique_ptr<Roaring64Map>> chunks;
std::vector<std::unique_ptr<Roaring>> chunks;

silkworm::Bytes from_key{key.begin(), key.end()};
from_key.resize(key.size() + sizeof(uint16_t));
boost::endian::store_big_u16(&from_key[key.size()], 0);
from_key.resize(key.size() + sizeof(uint32_t));
boost::endian::store_big_u32(&from_key[key.size()], from_block);
SILK_DEBUG << "table: " << table << " key: " << key << " from_key: " << from_key;

core::rawdb::Walker walker = [&](const silkworm::Bytes& k, const silkworm::Bytes& v) {
SILK_TRACE << "k: " << k << " v: " << v;
auto chunck = std::make_unique<Roaring64Map>(silkworm::db::bitmap::parse(v));
auto block = chunck->maximum();
if (block >= from_block && chunck->minimum() <= to_block) {
chunks.push_back(std::move(chunck));
}
auto chunk = std::make_unique<Roaring>(Roaring::readSafe(reinterpret_cast<const char*>(v.data()), v.size()));
SILK_TRACE << "chunk: " << chunk->toString();
chunks.push_back(std::move(chunk));
auto block = boost::endian::load_big_u32(&k[k.size() - sizeof(uint32_t)]);
return block < to_block;
};
co_await db_reader.walk(table, from_key, gsl::narrow<uint32_t>(key.size() * CHAR_BIT), walker);
Expand All @@ -65,13 +74,13 @@ awaitable<Roaring64Map> get(core::rawdb::DatabaseReader& db_reader, const std::s
co_return result;
}

awaitable<Roaring64Map> from_topics(core::rawdb::DatabaseReader& db_reader, const std::string& table, const FilterTopics& topics,
awaitable<Roaring> from_topics(core::rawdb::DatabaseReader& db_reader, const std::string& table, const FilterTopics& topics,
uint64_t start, uint64_t end) {
SILK_DEBUG << "#topics: " << topics.size() << " start: " << start << " end: " << end;
roaring::Roaring64Map result_bitmap;
roaring::Roaring result_bitmap;
for (const auto& subtopics : topics) {
SILK_DEBUG << "#subtopics: " << subtopics.size();
roaring::Roaring64Map subtopic_bitmap;
roaring::Roaring subtopic_bitmap;
for (auto topic : subtopics) {
silkworm::Bytes topic_key{std::begin(topic.bytes), std::end(topic.bytes)};
SILK_TRACE << "topic: " << topic << " topic_key: " << silkworm::to_hex(topic);
Expand All @@ -92,10 +101,10 @@ awaitable<Roaring64Map> from_topics(core::rawdb::DatabaseReader& db_reader, cons
co_return result_bitmap;
}

awaitable<Roaring64Map> from_addresses(core::rawdb::DatabaseReader& db_reader, const std::string& table, const FilterAddresses& addresses,
awaitable<Roaring> from_addresses(core::rawdb::DatabaseReader& db_reader, const std::string& table, const FilterAddresses& addresses,
uint64_t start, uint64_t end) {
SILK_TRACE << "#addresses: " << addresses.size() << " start: " << start << " end: " << end;
roaring::Roaring64Map result_bitmap;
roaring::Roaring result_bitmap;
for (auto address : addresses) {
silkworm::Bytes address_key{std::begin(address.bytes), std::end(address.bytes)};
auto bitmap = co_await ethdb::bitmap::get(db_reader, table, address_key, gsl::narrow<uint32_t>(start), gsl::narrow<uint32_t>(end));
Expand Down
8 changes: 4 additions & 4 deletions silkworm/silkrpc/ethdb/bitmap.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
#pragma GCC diagnostic ignored "-Wold-style-cast"
#pragma GCC diagnostic ignored "-Wconversion"
#pragma GCC diagnostic ignored "-Wsign-conversion"
#include <roaring/roaring64map.hh>
#include <roaring/roaring.hh>
#pragma GCC diagnostic pop

#include <silkworm/core/common/util.hpp>
Expand All @@ -36,13 +36,13 @@ namespace silkworm::rpc::ethdb::bitmap {

using boost::asio::awaitable;

awaitable<roaring::Roaring64Map> get(core::rawdb::DatabaseReader& db_reader, const std::string& table,
awaitable<roaring::Roaring> get(core::rawdb::DatabaseReader& db_reader, const std::string& table,
silkworm::Bytes& key, uint32_t from_block, uint32_t to_block);

awaitable<roaring::Roaring64Map> from_topics(core::rawdb::DatabaseReader& db_reader, const std::string& table,
awaitable<roaring::Roaring> from_topics(core::rawdb::DatabaseReader& db_reader, const std::string& table,
const FilterTopics& topics, uint64_t start, uint64_t end);

awaitable<roaring::Roaring64Map> from_addresses(core::rawdb::DatabaseReader& db_reader, const std::string& table,
awaitable<roaring::Roaring> from_addresses(core::rawdb::DatabaseReader& db_reader, const std::string& table,
const FilterAddresses& addresses, uint64_t start, uint64_t end);

} // namespace silkworm::rpc::ethdb::bitmap

0 comments on commit b30fd8b

Please sign in to comment.