Skip to content

Commit

Permalink
agent, common: add and use the formatDeploymentName function
Browse files Browse the repository at this point in the history
  • Loading branch information
tilacog committed Mar 10, 2023
1 parent d2decbd commit 5544cf8
Show file tree
Hide file tree
Showing 10 changed files with 249 additions and 48 deletions.
8 changes: 6 additions & 2 deletions packages/indexer-agent/src/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import {
SubgraphIdentifierType,
evaluateDeployments,
AllocationDecision,
formatDeploymentName,
} from '@graphprotocol/indexer-common'
import { Indexer } from './indexer'
import { AgentConfig } from './types'
Expand Down Expand Up @@ -688,8 +689,11 @@ class Agent {
// Index all new deployments worth indexing
await queue.addAll(
deploy.map(deployment => async () => {
const name = `indexer-agent/${deployment.ipfsHash.slice(-10)}`

const subgraphDeployment =
await this.networkMonitor.requireSubgraphDeployment(
deployment.ipfsHash,
)
const name = formatDeploymentName(subgraphDeployment)
this.logger.info(`Index subgraph deployment`, {
name,
deployment: deployment.display,
Expand Down
11 changes: 7 additions & 4 deletions packages/indexer-agent/src/commands/start.ts
Original file line number Diff line number Diff line change
Expand Up @@ -702,14 +702,17 @@ export default {
indexerAddress,
allocationManagementMode,
)
const networkSubgraphDeployment = argv.networkSubgraphDeployment
const networkSubgraphDeploymentId = argv.networkSubgraphDeployment
? new SubgraphDeploymentID(argv.networkSubgraphDeployment)
: undefined
if (networkSubgraphDeployment !== undefined) {
if (networkSubgraphDeploymentId !== undefined) {
// Make sure the network subgraph is being indexed
//
// TODO: once the Network Subgraph is published to the Network, we can use the
// `formatDeploymentName` function instead of using a hardcoded deployment name.
await indexer.ensure(
`indexer-agent/${networkSubgraphDeployment.ipfsHash.slice(-10)}`,
networkSubgraphDeployment,
`graphprotocol/network-subgraph/${networkSubgraphDeploymentId.ipfsHash}`,
networkSubgraphDeploymentId,
)
}

Expand Down
31 changes: 1 addition & 30 deletions packages/indexer-agent/src/indexer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -131,36 +131,7 @@ export class Indexer {
}

async subgraphDeployments(): Promise<SubgraphDeploymentID[]> {
try {
const result = await this.statusResolver.statuses
.query(
gql`
{
indexingStatuses {
subgraphDeployment: subgraph
node
}
}
`,
)
.toPromise()

if (result.error) {
throw result.error
}

return result.data.indexingStatuses
.filter((status: { subgraphDeployment: string; node: string }) => {
return status.node && status.node !== 'removed'
})
.map((status: { subgraphDeployment: string; node: string }) => {
return new SubgraphDeploymentID(status.subgraphDeployment)
})
} catch (error) {
const err = indexerError(IndexerErrorCode.IE018, error)
this.logger.error(`Failed to query indexing status API`, { err })
throw err
}
return await this.statusResolver.subgraphDeployments()
}

async indexNodes(): Promise<indexNode[]> {
Expand Down
88 changes: 88 additions & 0 deletions packages/indexer-common/src/__tests__/subgraphs.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import { formatDeploymentName, cleanDeploymentName } from '../types'
import { BigNumber } from 'ethers'
import { SubgraphDeploymentID, toAddress } from '@graphprotocol/common-ts'

describe('formatDeploymentName function tests', () => {
const creatorAddress = toAddress('0x6d2e03b7EfFEae98BD302A9F836D0d6Ab0002766')
const name = 'testSubgraphName'
const ipfsHash = 'Qmadj8x9km1YEyKmRnJ6EkC2zpJZFCfTyTZpuqC3j6e1QH'
const base = {
id: new SubgraphDeploymentID(ipfsHash),
deniedAt: 0,
stakedTokenns: BigNumber.from(0),
signalledTokens: BigNumber.from(0),
queryFeesAmount: BigNumber.from(0),
stakedTokens: BigNumber.from(0),
activeAllocations: 0,
}

test('formatDeploymentName can handle existing subgraph and owner information', async () => {
const nameAndOwner = {
name,
creatorAddress,
...base,
}
expect(formatDeploymentName(nameAndOwner)).toBe(
`${name}/${ipfsHash}/${creatorAddress}`,
)
})

test('formatDeploymentName can handle missing owner name', async () => {
const noOwner = {
name,
...base,
}
expect(formatDeploymentName(noOwner)).toBe(`${name}/${ipfsHash}/unknownCreator`)
})

test('formatDeploymentName can handle missing subgraph name', async () => {
const noName = {
creatorAddress,
...base,
}
expect(formatDeploymentName(noName)).toBe(
`unknownSubgraph/${ipfsHash}/${creatorAddress}`,
)
})

test('formatDeploymentName can handle missing subgraph and owner names', async () => {
expect(formatDeploymentName(base)).toBe(`unknownSubgraph/${ipfsHash}/unknownCreator`)
})
})

describe('cleanDeploymentName function tests', () => {
test('can handle null input', () => {
expect(cleanDeploymentName(undefined)).toBe('unknownSubgraph')
})
test('can remove invalid characters', () => {
expect(cleanDeploymentName('abc!@"#$%^&*()-def_123')).toBe('abc-def_123')
})
test('can strip invalid charecters from start', () => {
expect(cleanDeploymentName('_abc')).toBe('abc')
expect(cleanDeploymentName('-abc')).toBe('abc')
})
test('can strip invalid charecters from the end', () => {
expect(cleanDeploymentName('abc_')).toBe('abc')
expect(cleanDeploymentName('abc-')).toBe('abc')
})
test('can strip invalid charecters from both ends', () => {
expect(cleanDeploymentName('_abc_')).toBe('abc')
expect(cleanDeploymentName('-abc-')).toBe('abc')
})
test('can clean empty strings', () => {
expect(cleanDeploymentName('')).toBe('unknownSubgraph')
expect(cleanDeploymentName('--')).toBe('unknownSubgraph')
expect(cleanDeploymentName('_')).toBe('unknownSubgraph')
})
test('can clean the special name "graphql"', () => {
expect(cleanDeploymentName('graphql')).toBe('graphql-subgraph')
expect(cleanDeploymentName('-graphql-')).toBe('graphql-subgraph')
expect(cleanDeploymentName('_graphql')).toBe('graphql-subgraph')
expect(cleanDeploymentName('graphql_')).toBe('graphql-subgraph')
})
test('can chop the subgraph name to the adequate size', () => {
const reallyLongName = // 200 chars
'y2feiw6y0eihrau5m5my0g0wvg6e2qbf79k91wzhcoep40hrend3re36jaejomss0goyaxx6yph5rrwieg3gkrvys699riza6kfak1tx9uy46onxt4fs3tp95e05v3xcf0jdldsz5ukqozsefo53wxl2m5rh5cdx8dkxq1fktr'
expect(cleanDeploymentName(reallyLongName)).toHaveLength(165)
})
})
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
CloseAllocationResult,
CreateAllocationResult,
fetchIndexingRules,
formatDeploymentName,
indexerError,
IndexerError,
IndexerErrorCode,
Expand Down Expand Up @@ -330,11 +331,15 @@ export class AllocationManager {
)
}

const subgraphDeployment = await this.networkMonitor.requireSubgraphDeployment(
deployment.ipfsHash,
)

// Ensure subgraph is deployed before allocating
await this.subgraphManager.ensure(
logger,
this.models,
`indexer-agent/${deployment.ipfsHash.slice(-10)}`,
formatDeploymentName(subgraphDeployment),
deployment,
indexNode,
)
Expand Down
24 changes: 24 additions & 0 deletions packages/indexer-common/src/indexer-management/monitor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -440,6 +440,12 @@ export class NetworkMonitor {
id
}
}
versions(first: 1, orderBy: version, orderDirection: desc) {
subgraph {
displayName
creatorAddress
}
}
}
}
`,
Expand Down Expand Up @@ -477,6 +483,18 @@ export class NetworkMonitor {
}
}

// Wrapper function over this.subgraphDeployment that will throw an
// error on missing subgraph deployments
async requireSubgraphDeployment(ipfsHash: string): Promise<SubgraphDeployment> {
const subgraphDeployment = await this.subgraphDeployment(ipfsHash)
if (!subgraphDeployment) {
const errorMessage = `Failed to locate subgraph deployment with id ${ipfsHash} in the Network Subgraph`
this.logger.error(errorMessage, { ipfsHash })
throw indexerError(IndexerErrorCode.IE020, errorMessage)
}
return subgraphDeployment
}

async subgraphDeployments(): Promise<SubgraphDeployment[]> {
const deployments = []
const queryProgress = {
Expand Down Expand Up @@ -513,6 +531,12 @@ export class NetworkMonitor {
id
}
}
versions(first: 1, orderBy: version, orderDirection: desc) {
subgraph {
displayName
creatorAddress
}
}
}
}
`,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
import { NetworkMonitor, epochElapsedBlocks } from '@graphprotocol/indexer-common'
import {
NetworkMonitor,
epochElapsedBlocks,
formatDeploymentName,
} from '@graphprotocol/indexer-common'
/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
/* eslint-disable @typescript-eslint/ban-types */

Expand Down Expand Up @@ -421,13 +425,13 @@ export default {
logger.debug('Execute createAllocation() mutation', { deployment, amount })

const allocationAmount = parseGRT(amount)
const subgraphDeployment = new SubgraphDeploymentID(deployment)
const subgraphDeploymentID = new SubgraphDeploymentID(deployment)

const activeAllocations = await networkMonitor.allocations(AllocationStatus.ACTIVE)

const allocation = activeAllocations.find(
(allocation) =>
allocation.subgraphDeployment.id.toString() === subgraphDeployment.toString(),
allocation.subgraphDeployment.id.toString() === subgraphDeploymentID.toString(),
)
if (allocation) {
logger.warn('Already allocated to deployment', {
Expand Down Expand Up @@ -475,12 +479,16 @@ export default {
)
}

const subgraphDeployment = await networkMonitor.requireSubgraphDeployment(
subgraphDeploymentID.ipfsHash,
)

// Ensure subgraph is deployed before allocating
await subgraphManager.ensure(
logger,
models,
`indexer-agent/${subgraphDeployment.ipfsHash.slice(-10)}`,
subgraphDeployment,
formatDeploymentName(subgraphDeployment),
subgraphDeploymentID,
indexNode,
)

Expand All @@ -490,7 +498,7 @@ export default {
const { allocationSigner, allocationId } = uniqueAllocationID(
transactionManager.wallet.mnemonic.phrase,
currentEpoch.toNumber(),
subgraphDeployment,
subgraphDeploymentID,
activeAllocations.map((allocation) => allocation.id),
)

Expand Down Expand Up @@ -528,7 +536,7 @@ export default {

logger.debug(`Sending allocateFrom transaction`, {
indexer: address,
subgraphDeployment: subgraphDeployment.ipfsHash,
subgraphDeployment: subgraphDeploymentID.ipfsHash,
amount: formatGRT(allocationAmount),
allocation: allocationId,
proof,
Expand All @@ -538,7 +546,7 @@ export default {
async () =>
contracts.staking.estimateGas.allocateFrom(
address,
subgraphDeployment.bytes32,
subgraphDeploymentID.bytes32,
allocationAmount,
allocationId,
utils.hexlify(Array(32).fill(0)),
Expand All @@ -547,7 +555,7 @@ export default {
async (gasLimit) =>
contracts.staking.allocateFrom(
address,
subgraphDeployment.bytes32,
subgraphDeploymentID.bytes32,
allocationAmount,
allocationId,
utils.hexlify(Array(32).fill(0)),
Expand Down Expand Up @@ -594,7 +602,7 @@ export default {
`Updating indexing rules, so indexer-agent will now manage the active allocation`,
)
const indexingRule = {
identifier: subgraphDeployment.ipfsHash,
identifier: subgraphDeploymentID.ipfsHash,
allocationAmount: allocationAmount.toString(),
identifierType: SubgraphIdentifierType.DEPLOYMENT,
decisionBasis: IndexingDecisionBasis.ALWAYS,
Expand Down
2 changes: 2 additions & 0 deletions packages/indexer-common/src/indexer-management/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ export const parseGraphQLSubgraphDeployment = (
signalledTokens: BigNumber.from(subgraphDeployment.signalledTokens),
queryFeesAmount: BigNumber.from(subgraphDeployment.queryFeesAmount),
activeAllocations: subgraphDeployment.indexerAllocations.length,
name: subgraphDeployment.versions[0].subgraph.displayName,
creatorAddress: toAddress(subgraphDeployment.versions[0].subgraph.creatorAddress),
})

/* eslint-disable @typescript-eslint/no-explicit-any */
Expand Down
45 changes: 45 additions & 0 deletions packages/indexer-common/src/indexing-status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ export interface IndexingStatusFetcherOptions {
statusEndpoint: string
}

export interface SubgraphDeploymentAssignment {
id: SubgraphDeploymentID
node: string
}

// eslint-disable-next-line @typescript-eslint/no-explicit-any
export const parseGraphQLIndexingStatus = (indexingStatus: any): IndexingStatus => ({
subgraphDeployment: new SubgraphDeploymentID(indexingStatus.subgraphDeployment),
Expand Down Expand Up @@ -271,4 +276,44 @@ export class IndexingStatusResolver {
throw err
}
}

public async subgraphDeployments(): Promise<SubgraphDeploymentID[]> {
return (await this.subgraphDeploymentsAssignments()).map((details) => details.id)
}

public async subgraphDeploymentsAssignments(): Promise<SubgraphDeploymentAssignment[]> {
try {
const result = await this.statuses
.query(
gql`
{
indexingStatuses {
subgraphDeployment: subgraph
node
}
}
`,
)
.toPromise()

if (result.error) {
throw result.error
}

type QueryResult = { subgraphDeployment: string; node: string }

return result.data.indexingStatuses
.filter((status: QueryResult) => status.node && status.node !== 'removed')
.map((status: QueryResult) => {
return {
id: new SubgraphDeploymentID(status.subgraphDeployment),
node: status.node,
}
})
} catch (error) {
const err = indexerError(IndexerErrorCode.IE018, error)
this.logger.error(`Failed to query indexing status API`, { err })
throw err
}
}
}
Loading

0 comments on commit 5544cf8

Please sign in to comment.