Merge branch 'dev' into feat/dvmd-dt-allocated-handler

.dockerignore

@@ -0,0 +1,10 @@
+# Ignore all .env files in any directory or subdirectory
+**/.env
+
+# Ignore node_modules and dist directories in any directory or subdirectory
+**/node_modules
+**/dist
+
+**/.git
+**/.turbo
+**/*.md

.env.example

@@ -3,25 +3,23 @@
 ######################################################
 DATALAYER_POSTGRES_PASSWORD=testing
 DATALAYER_PG_USER=postgres
-DATALAYER_PG_DATABASE=datalayer-postgres
+DATALAYER_PG_DATABASE=datalayer-postgres-db
 DATALAYER_POSTGRES_EXPOSED_PORT=5434
 
 ######################################################
 ############### DATALAYER HASURA API #################
 ######################################################
-#for reference on Hasura configuration, see: https://hasura.io/docs/2.0/deployment/graphql-engine-flags/reference/
 DATALAYER_HASURA_EXPOSED_PORT=8082
-DATALAYER_HASURA_ENABLE_CONSOLE=true
-DATALAYER_HASURA_ADMIN_SECRET=my-admin-secret
-DATALAYER_HASURA_UNAUTHORIZED_ROLE=public
+DATALAYER_HASURA_ENABLE_CONSOLE="true"
+DATALAYER_HASURA_ADMIN_SECRET="my-admin-secret"
+DATALAYER_HASURA_UNAUTHORIZED_ROLE="public"
 DATALAYER_HASURA_CORS_DOMAIN=*
-DATALAYER_HASURA_ENABLE_TELEMETRY=false
-# HASURA_GRAPHQL_METADATA_DATABASE_EXTENSIONS_SCHEMA: "chain_data_schema_1"
+DATALAYER_HASURA_ENABLE_TELEMETRY="false"
 ## enable debugging mode. It is recommended to disable this in production
-DATALAYER_HASURA_DEV_MODE=true
-DATALAYER_HASURA_ADMIN_INTERNAL_ERRORS=true
+DATALAYER_HASURA_DEV_MODE="true"
+DATALAYER_HASURA_ADMIN_INTERNAL_ERRORS="true"
 ## uncomment next line to run console offline (i.e load console assets from server instead of CDN)
-# HASURA_GRAPHQL_CONSOLE_ASSETS_DIR: /srv/console-assets
+# DATALAYER_HASURA_CONSOLE_ASSETS_DIR="/srv/console-assets"
 
 ############################################################
 ############### ENVIO POSTGRES & INDEXER ###################
@@ -38,20 +36,41 @@ ENVIO_POSTGRES_EXPOSED_PORT=5433
 ############################################
 ############### INDEXER ####################
 ############################################
-ENVIO_PG_HOST=envio-postgres
+ENVIO_PG_HOST=indexer-postgres-db
 ENVIO_PG_PORT=5432 
-HASURA_GRAPHQL_ENDPOINT=http://graphql-engine:8080/v1/metadata
+HASURA_GRAPHQL_ENDPOINT=http://indexer-graphql-api:8080/v1/metadata
 TUI_OFF=true
 
 ############################################
 ############### HASURA #####################
 ############################################
 HASURA_EXPOSED_PORT=8080
-HASURA_GRAPHQL_DATABASE_URL=postgres://postgres:testing@envio-postgres:5432/envio-dev
+HASURA_GRAPHQL_DATABASE_URL=postgres://postgres:testing@indexer-postgres-db:5432/envio-dev
 HASURA_GRAPHQL_ENABLE_CONSOLE=true
 HASURA_GRAPHQL_ENABLED_LOG_TYPES=startup, http-log, webhook-log, websocket-log, query-log
 HASURA_GRAPHQL_NO_OF_RETRIES=10
 HASURA_GRAPHQL_ADMIN_SECRET=testing
 HASURA_GRAPHQL_STRINGIFY_NUMERIC_TYPES="true"
 PORT=8080
-HASURA_GRAPHQL_UNAUTHORIZED_ROLE=public
+HASURA_GRAPHQL_UNAUTHORIZED_ROLE=public
+
+############################################
+############### PROCESSING #################
+############################################
+
+RPC_URLS=["https://optimism.llamarpc.com","https://rpc.ankr.com/optimism","https://optimism.gateway.tenderly.co","https://optimism.blockpi.network/v1/rpc/public","https://mainnet.optimism.io","https://opt-mainnet.g.alchemy.com/v2/demo"]
+CHAIN_ID=10
+
+FETCH_LIMIT=1000
+FETCH_DELAY_MS=3000
+
+DATABASE_URL=postgresql://postgres:testing@datalayer-postgres-db:5432/datalayer-postgres-db
+DATABASE_SCHEMA=chain_data_schema_1
+
+INDEXER_GRAPHQL_URL=http://indexer-graphql-api:8080/v1/graphql
+INDEXER_ADMIN_SECRET=testing
+
+IPFS_GATEWAYS_URL=["https://ipfs.io","https://gateway.pinata.cloud","https://dweb.link", "https://ipfs.eth.aragon.network"]
+
+COINGECKO_API_KEY={{YOUR_KEY}}
+COINGECKO_API_TYPE=demo

Dockerfile

@@ -0,0 +1,17 @@
+# Based on example: 
+FROM node:20-slim AS base
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+
+FROM base AS build
+COPY . /usr/src/app
+WORKDIR /usr/src/app
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
+RUN pnpm run -r build
+RUN pnpm deploy --filter=./apps/processing --prod /prod/processing
+
+FROM base AS processing
+COPY --from=build /prod/processing /prod/processing
+WORKDIR /prod/processing
+CMD [ "pnpm", "start" ]

README.md

@@ -4,13 +4,35 @@ Grants Stack Indexer v2 is a tool designed to index blockchain events generated
 
 ## 📖 Overview
 
-This repository is a monorepo that contains 1 package and 2 applications:
+This repository is a monorepo that contains 8 packages and 3 applications:
 
--   @ts-turborepo-boilerplate/sample-lib: A sample library for querying account balances.
--   @ts-turborepo-boilerplate/sample-app: A demo application demonstrating the use of sample-lib.
--   @grants-stack-indexer/indexer: An [`envio`](https://docs.envio.dev/) indexer, that collects all the relevant events from Allo contracts.
+### Applications
 
-## 🚀 Local Deployment (Dockerized)
+-   **@grants-stack-indexer/indexer**: An Envio indexer service that collects relevant events from Allo contracts. It is designed to run on the Envio hosted platform and includes a Dockerfile for deployment.
+
+-   **@grants-stack-indexer/processing**: This service runs the core processing pipeline, coordinating components from various packages to process blockchain events. It manages an Orchestrator per chain.
+
+-   **@grants-stack-indexer/scripts**: Contains scripts for managing the database schema and migrations. It includes scripts for running migrations and resetting the database schema.
+
+### Packages/libs
+
+-   **@grants-stack-indexer/chain-providers**: Provides wrappers of the Viem library to interact with EVM-based blockchains. It includes utilities for reading contracts and making batch requests.
+
+-   **@grants-stack-indexer/data-flow**: Manages the flow of data between different components of the indexer. It includes core components for the processing pipeline.
+
+-   **@grants-stack-indexer/indexer-client**: A client library for interacting with blockchain event indexing services. It provides methods to fetch events by block number and log index.
+
+-   **@grants-stack-indexer/metadata**: Handles metadata related to the indexed events. It provides a metadata provider to retrieve metadata from IPFS.
+
+-   **@grants-stack-indexer/pricing**: Manages pricing data and calculations. It includes providers to get the price of a token at a specific timestamp using chainId and token address.
+
+-   **@grants-stack-indexer/processors**: Contains various data processors used in the indexing pipeline. It includes processors for handling Allo, Strategy, and Registry events.
+
+-   **@grants-stack-indexer/repository**: Manages the storage and retrieval of indexed data. It implements the Repository pattern to abstract database operations.
+
+-   **@grants-stack-indexer/shared**: Provides shared utilities, types, constants, and logger. It is designed to be used across the packages of the monorepo to ensure consistency and reusability.
+
+## 🚀 Local Deployment (Development)
 
 ### Prerequisites
 
@@ -19,15 +41,39 @@ Ensure you have the following installed on your machine:
 -   [Docker](https://www.docker.com/get-started)
 -   [Docker Compose](https://docs.docker.com/compose/install/)
 
-### Run
+### Setup
+
+1. Copy the Example Environment File and edit the `.env` file
+
+```
+cp .env.example .env
+```
 
-To deploy locally with Docker, run the following command:
+2. Build and start the services in detached mode:
 
 ```
 docker-compose up -d --build
 ```
 
-Once the deployment is complete, you can access Hasura by navigating to:
+3. Copy the Example Environment File on `apps/migration-scripts` and edit the `.env` file
+
+```
+cp apps/migration-scripts/.env.example apps/migration-scripts/.env
+```
+
+4. After starting Docker Compose, run the following command to apply the database migrations:
+
+```
+pnpm script:db:migrate
+```
+
+5. Navigate to the processing service directory and start it with:
+
+```
+cd apps/processing && pnpm dev
+```
+
+Once the setup is completed you can access Hasura by navigating to:
 
 ```
 http://localhost:8080/

apps/indexer/package.json

@@ -13,7 +13,7 @@
     },
     "dependencies": {
         "chai": "4.3.10",
-        "envio": "2.7.2",
+        "envio": "2.7.3",
         "ethers": "6.8.0",
         "yaml": "2.5.1"
     },

apps/processing/.env.example

@@ -1,17 +1,17 @@
-# configuration for Optimism
 RPC_URLS=["https://optimism.llamarpc.com","https://rpc.ankr.com/optimism","https://optimism.gateway.tenderly.co","https://optimism.blockpi.network/v1/rpc/public","https://mainnet.optimism.io","https://opt-mainnet.g.alchemy.com/v2/demo"]
 CHAIN_ID=10
+LOG_LEVEL=debug
 
-FETCH_LIMIT=500
-FETCH_DELAY_MS=3000
+FETCH_LIMIT=10000
+FETCH_DELAY_MS=30000
 
-DATABASE_URL=
-DATABASE_SCHEMA=chainDataSchema
+DATABASE_URL=postgresql://postgres:testing@localhost:5434/datalayer-postgres-db
+DATABASE_SCHEMA=chain_data_schema_1
 
-INDEXER_GRAPHQL_URL=
-INDEXER_ADMIN_SECRET=
+INDEXER_GRAPHQL_URL=http://localhost:8080/v1/graphql
+INDEXER_ADMIN_SECRET=testing
 
 IPFS_GATEWAYS_URL=["https://ipfs.io","https://gateway.pinata.cloud","https://dweb.link", "https://ipfs.eth.aragon.network"]
 
-COINGECKO_API_KEY=
-COINGECKO_API_TYPE= #demo or pro
+COINGECKO_API_KEY={{YOUR_KEY}}
+COINGECKO_API_TYPE=demo

apps/processing/src/config/env.ts

@@ -1,6 +1,8 @@
 import dotenv from "dotenv";
 import { z } from "zod";
 
+import { stringify } from "@grants-stack-indexer/shared";
+
 dotenv.config();
 
 const stringToJSONSchema = z.string().transform((str, ctx): z.infer<ReturnType<typeof Object>> => {
@@ -33,7 +35,7 @@ const env = validationSchema.safeParse(process.env);
 if (!env.success) {
     console.error(
         "Invalid environment variables:",
-        env.error.issues.map((issue) => JSON.stringify(issue)).join("\n"),
+        env.error.issues.map((issue) => stringify(issue)).join("\n"),
     );
     process.exit(1);
 }