From a165b802b01c4a5a742ba21d3381293a92c294e9 Mon Sep 17 00:00:00 2001 From: Ben Lyaunzon Date: Tue, 10 Aug 2021 13:57:13 -0700 Subject: [PATCH 01/15] staking api div by 0 hotfix --- staking-api/src/queries.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/staking-api/src/queries.ts b/staking-api/src/queries.ts index a5e839c5..3b42d445 100644 --- a/staking-api/src/queries.ts +++ b/staking-api/src/queries.ts @@ -455,12 +455,12 @@ WITH , ts.total_staked , cebs.operator_zrx_delegated AS operator_zrx_staked , cebs.member_zrx_delegated AS member_zrx_staked - , cebs.zrx_delegated / ts.total_staked AS share_of_stake + , cebs.zrx_delegated / NULLIF(ts.total_staked,0) AS share_of_stake , fbp.protocol_fees AS total_protocol_fees_generated_in_eth , fbp.num_fills AS number_of_fills , fbp.protocol_fees / tf.total_protocol_fees AS share_of_fees , fbp.num_fills::FLOAT / tf.total_fills::FLOAT AS share_of_fills - , (cebs.zrx_delegated / ts.total_staked) + , (cebs.zrx_delegated / NULLIF(ts.total_staked,0)) / NULLIF((COALESCE(fbp.protocol_fees,0) / tf.total_protocol_fees),0) AS approximate_stake_ratio FROM events.staking_pool_created_events pce @@ -522,12 +522,12 @@ export const currentEpochPoolStatsQuery = ` , cebs.operator_zrx_delegated AS operator_zrx_staked , cebs.member_zrx_delegated AS member_zrx_staked , ts.total_staked - , cebs.zrx_delegated / ts.total_staked AS share_of_stake + , cebs.zrx_delegated / NULLIF(ts.total_staked,0) AS share_of_stake , fbp.protocol_fees AS total_protocol_fees_generated_in_eth , fbp.num_fills AS number_of_fills , fbp.protocol_fees / tf.total_protocol_fees AS share_of_fees , fbp.num_fills::FLOAT / tf.total_fills::FLOAT AS share_of_fills - , (cebs.zrx_delegated / ts.total_staked) + , (cebs.zrx_delegated / NULLIF(ts.total_staked,0)) / (fbp.protocol_fees / tf.total_protocol_fees) AS approximate_stake_ratio FROM events.staking_pool_created_events pce @@ -613,10 +613,10 @@ export const nextEpochPoolsStatsQuery = ` , cs.operator_zrx_staked , cs.member_zrx_staked , ts.total_staked - , cs.zrx_staked / ts.total_staked AS share_of_stake + , cs.zrx_staked / NULLIF(ts.total_staked,0) AS share_of_stake , 0.00 AS total_protocol_fees_generated_in_eth , 0 AS number_of_fills - , (cs.zrx_staked / ts.total_staked) + , (cs.zrx_staked / NULLIF(ts.total_staked,0)) / NULLIF((COALESCE(fbp.protocol_fees,0) / tr.total_protocol_fees),0) AS approximate_stake_ratio FROM staking.pool_info pi @@ -701,10 +701,10 @@ export const nextEpochPoolStatsQuery = ` , cs.operator_zrx_staked , cs.member_zrx_staked , ts.total_staked - , cs.zrx_staked / ts.total_staked AS share_of_stake + , cs.zrx_staked / NULLIF(ts.total_staked,0)) AS share_of_stake , 0.00 AS total_protocol_fees_generated_in_eth , 0 AS number_of_fills - , (cs.zrx_staked / ts.total_staked) + , (cs.zrx_staked / NULLIF(ts.total_staked,0))) / (fbp.protocol_fees / tr.total_protocol_fees) AS approximate_stake_ratio FROM staking.pool_info pi From 1b5e223942802e1c93ac2a8ff6a5d6b0efcfa20c Mon Sep 17 00:00:00 2001 From: Ben Lyaunzon Date: Tue, 10 Aug 2021 14:01:15 -0700 Subject: [PATCH 02/15] individual pool fix ++ --- staking-api/src/queries.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/staking-api/src/queries.ts b/staking-api/src/queries.ts index 3b42d445..1c118609 100644 --- a/staking-api/src/queries.ts +++ b/staking-api/src/queries.ts @@ -701,10 +701,10 @@ export const nextEpochPoolStatsQuery = ` , cs.operator_zrx_staked , cs.member_zrx_staked , ts.total_staked - , cs.zrx_staked / NULLIF(ts.total_staked,0)) AS share_of_stake + , cs.zrx_staked / NULLIF(ts.total_staked,0) AS share_of_stake , 0.00 AS total_protocol_fees_generated_in_eth , 0 AS number_of_fills - , (cs.zrx_staked / NULLIF(ts.total_staked,0))) + , (cs.zrx_staked / NULLIF(ts.total_staked,0)) / (fbp.protocol_fees / tr.total_protocol_fees) AS approximate_stake_ratio FROM staking.pool_info pi From bed011b1bc6d2bb18b3f5fed5feefb0581c10aa1 Mon Sep 17 00:00:00 2001 From: Theo Gonella Date: Wed, 6 Oct 2021 11:17:05 -0700 Subject: [PATCH 03/15] fixed query for approximate stake ratio, taking into account instances with 0 protocol fees collected globally --- staking-api/src/queries.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staking-api/src/queries.ts b/staking-api/src/queries.ts index 1c118609..d6aaa5d0 100644 --- a/staking-api/src/queries.ts +++ b/staking-api/src/queries.ts @@ -705,7 +705,7 @@ export const nextEpochPoolStatsQuery = ` , 0.00 AS total_protocol_fees_generated_in_eth , 0 AS number_of_fills , (cs.zrx_staked / NULLIF(ts.total_staked,0)) - / (fbp.protocol_fees / tr.total_protocol_fees) + / NULLIF((fbp.protocol_fees / NULLIF(tr.total_protocol_fees,0)),0) AS approximate_stake_ratio FROM staking.pool_info pi LEFT JOIN operator_share os ON os.pool_id = pi.pool_id From 7e720a02189590a91460c38c88a6b42b88912da2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20P=C3=A9rez?= Date: Wed, 6 Oct 2021 14:11:03 -0500 Subject: [PATCH 04/15] Staking API - More fixes for 0 protocol fees --- staking-api/src/queries.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/staking-api/src/queries.ts b/staking-api/src/queries.ts index d6aaa5d0..79bb2f7d 100644 --- a/staking-api/src/queries.ts +++ b/staking-api/src/queries.ts @@ -458,11 +458,19 @@ WITH , cebs.zrx_delegated / NULLIF(ts.total_staked,0) AS share_of_stake , fbp.protocol_fees AS total_protocol_fees_generated_in_eth , fbp.num_fills AS number_of_fills - , fbp.protocol_fees / tf.total_protocol_fees AS share_of_fees + , CASE + WHEN tf.total_protocol_fees = 0 THEN + NULL + ELSE fbp.protocol_fees / tf.total_protocol_fees + END AS share_of_fees , fbp.num_fills::FLOAT / tf.total_fills::FLOAT AS share_of_fills - , (cebs.zrx_delegated / NULLIF(ts.total_staked,0)) - / NULLIF((COALESCE(fbp.protocol_fees,0) / tf.total_protocol_fees),0) - AS approximate_stake_ratio + , CASE + WHEN tf.total_protocol_fees = 0 THEN + NULL + ELSE + (cebs.zrx_delegated / NULLIF(ts.total_staked,0)) + / NULLIF((COALESCE(fbp.protocol_fees,0) / tf.total_protocol_fees),0) + END AS approximate_stake_ratio FROM events.staking_pool_created_events pce LEFT JOIN current_epoch_beginning_status cebs ON cebs.pool_id = pce.pool_id LEFT JOIN current_epoch_fills_by_pool fbp ON fbp.epoch_id = cebs.epoch_id AND fbp.pool_id = cebs.pool_id From 060e7b04bd73fe3d60d9646b7eb03ddabeb2641b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20P=C3=A9rez?= Date: Wed, 6 Oct 2021 15:00:32 -0500 Subject: [PATCH 05/15] Staking API More 0 Fee Fixes --- staking-api/src/queries.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/staking-api/src/queries.ts b/staking-api/src/queries.ts index 79bb2f7d..1fd8b834 100644 --- a/staking-api/src/queries.ts +++ b/staking-api/src/queries.ts @@ -548,7 +548,8 @@ export const currentEpochPoolStatsQuery = ` `; export const nextEpochPoolsStatsQuery = ` - WITH + + WITH current_stake AS ( SELECT pi.pool_id @@ -624,9 +625,13 @@ export const nextEpochPoolsStatsQuery = ` , cs.zrx_staked / NULLIF(ts.total_staked,0) AS share_of_stake , 0.00 AS total_protocol_fees_generated_in_eth , 0 AS number_of_fills - , (cs.zrx_staked / NULLIF(ts.total_staked,0)) + , CASE + WHEN tr.total_protocol_fees = 0 THEN + NULL + ELSE + (cs.zrx_staked / NULLIF(ts.total_staked,0)) / NULLIF((COALESCE(fbp.protocol_fees,0) / tr.total_protocol_fees),0) - AS approximate_stake_ratio + END AS approximate_stake_ratio FROM staking.pool_info pi LEFT JOIN operator_share os ON os.pool_id = pi.pool_id LEFT JOIN current_stake cs ON cs.pool_id = pi.pool_id From c18fdde645f0a460dd89aeda31f4bfdf9df7d135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20P=C3=A9rez?= Date: Fri, 22 Oct 2021 15:36:15 -0500 Subject: [PATCH 06/15] Staking API Division by 0 --- staking-api/src/queries.ts | 57 +++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/staking-api/src/queries.ts b/staking-api/src/queries.ts index 1fd8b834..c2f3d855 100644 --- a/staking-api/src/queries.ts +++ b/staking-api/src/queries.ts @@ -416,7 +416,7 @@ WITH ce.epoch_id , case when fe.protocol_version='v4' then fe.pool else cebs.pool_id end - as pool_id + as pool_id , COUNT(*) AS num_fills , SUM(fe.protocol_fee_paid) / 1e18 AS protocol_fees FROM events.native_fills fe @@ -439,7 +439,7 @@ WITH ELSE 0.00 END) AS total_staked FROM current_epoch_beginning_status cebs - LEFT JOIN current_epoch_fills_by_pool fbp ON fbp.epoch_id = cebs.epoch_id AND fbp.pool_id = cebs.pool_id + LEFT JOIN current_epoch_fills_by_pool fbp ON fbp.epoch_id = cebs.epoch_id AND fbp.pool_id = cebs.pool_id ) , total_fees AS ( SELECT @@ -452,24 +452,30 @@ WITH , cebs.maker_addresses AS maker_addresses , cebs.operator_share AS operator_share , cebs.zrx_delegated AS zrx_staked - , ts.total_staked , cebs.operator_zrx_delegated AS operator_zrx_staked , cebs.member_zrx_delegated AS member_zrx_staked - , cebs.zrx_delegated / NULLIF(ts.total_staked,0) AS share_of_stake + , ts.total_staked + , CASE + WHEN ts.total_staked = 0 THEN + NULL + ELSE + cebs.zrx_delegated / ts.total_staked + END AS share_of_stake , fbp.protocol_fees AS total_protocol_fees_generated_in_eth , fbp.num_fills AS number_of_fills , CASE - WHEN tf.total_protocol_fees = 0 THEN - NULL - ELSE fbp.protocol_fees / tf.total_protocol_fees - END AS share_of_fees + WHEN tf.total_protocol_fees = 0 THEN + NULL + ELSE + fbp.protocol_fees / tf.total_protocol_fees + END AS share_of_fees , fbp.num_fills::FLOAT / tf.total_fills::FLOAT AS share_of_fills , CASE - WHEN tf.total_protocol_fees = 0 THEN - NULL - ELSE - (cebs.zrx_delegated / NULLIF(ts.total_staked,0)) - / NULLIF((COALESCE(fbp.protocol_fees,0) / tf.total_protocol_fees),0) + WHEN tf.total_protocol_fees = 0 THEN + NULL + ELSE + (cebs.zrx_delegated / ts.total_staked) + / (fbp.protocol_fees / tf.total_protocol_fees) END AS approximate_stake_ratio FROM events.staking_pool_created_events pce LEFT JOIN current_epoch_beginning_status cebs ON cebs.pool_id = pce.pool_id @@ -530,14 +536,27 @@ export const currentEpochPoolStatsQuery = ` , cebs.operator_zrx_delegated AS operator_zrx_staked , cebs.member_zrx_delegated AS member_zrx_staked , ts.total_staked - , cebs.zrx_delegated / NULLIF(ts.total_staked,0) AS share_of_stake + , CASE + WHEN ts.total_staked = 0 THEN + NULL + ELSE + cebs.zrx_delegated / ts.total_staked + END AS share_of_stake , fbp.protocol_fees AS total_protocol_fees_generated_in_eth , fbp.num_fills AS number_of_fills - , fbp.protocol_fees / tf.total_protocol_fees AS share_of_fees - , fbp.num_fills::FLOAT / tf.total_fills::FLOAT AS share_of_fills - , (cebs.zrx_delegated / NULLIF(ts.total_staked,0)) - / (fbp.protocol_fees / tf.total_protocol_fees) - AS approximate_stake_ratio + , CASE + WHEN tf.total_protocol_fees = 0 THEN + NULL + ELSE + fbp.protocol_fees / tf.total_protocol_fees + END AS share_of_fees + , CASE + WHEN tf.total_protocol_fees = 0 THEN + NULL + ELSE + (cebs.zrx_delegated / ts.total_staked) + / (fbp.protocol_fees / tf.total_protocol_fees) + END AS approximate_stake_ratio FROM events.staking_pool_created_events pce LEFT JOIN current_epoch_beginning_status cebs ON cebs.pool_id = pce.pool_id LEFT JOIN current_epoch_fills_by_pool fbp ON fbp.epoch_id = cebs.epoch_id AND fbp.pool_id = cebs.pool_id From 030dff104826f87cd69d487493bf53b21c85d689 Mon Sep 17 00:00:00 2001 From: robpal1990 Date: Mon, 20 Dec 2021 22:42:18 +0100 Subject: [PATCH 07/15] fix maker/taker issue for univ3 --- event-pipeline/src/parsers/events/swap_events.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/event-pipeline/src/parsers/events/swap_events.ts b/event-pipeline/src/parsers/events/swap_events.ts index 97bc6f8a..70aa7f8b 100644 --- a/event-pipeline/src/parsers/events/swap_events.ts +++ b/event-pipeline/src/parsers/events/swap_events.ts @@ -12,15 +12,20 @@ export function parseUniswapV3SwapEvent(eventLog: RawLogEntry): ERC20BridgeTrans // decode the basic info directly into eRC20BridgeTransferEvent const decodedLog = abiCoder.decodeLog(SWAP_V3_ABI.inputs, eventLog.data, [eventLog.topics[1], eventLog.topics[2]]); - eRC20BridgeTransferEvent.fromToken = '1'; // taker_token - eRC20BridgeTransferEvent.toToken = '0'; // maker_token + const amount0 = new BigNumber(Math.abs(decodedLog.amount0)); + const amount1 = new BigNumber(Math.abs(decodedLog.amount1)); - eRC20BridgeTransferEvent.fromTokenAmount = new BigNumber(Math.abs(decodedLog.amount1)); // taker_token_amount - eRC20BridgeTransferEvent.toTokenAmount = new BigNumber(Math.abs(decodedLog.amount0)); // maker_token_amount + // amount0 and amount1 are of opposite signs + // neg value means token left the pool ie. maker + // pos value means token entered the pool ie. taker + eRC20BridgeTransferEvent.fromToken = (decodedLog.amount0 >= 0) ? '0': '1' ; // taker_token + eRC20BridgeTransferEvent.toToken = (decodedLog.amount0 < 0) ? '0': '1' ; // maker_token + eRC20BridgeTransferEvent.fromTokenAmount = decodedLog.amount0 >= 0 ? amount0 : amount1; // taker_token_amount + eRC20BridgeTransferEvent.toTokenAmount = decodedLog.amount0 < 0 ? amount0 : amount1; // maker_token_amount eRC20BridgeTransferEvent.from = 'UniswapV3'; // maker eRC20BridgeTransferEvent.to = decodedLog.recipient.toLowerCase(); // taker eRC20BridgeTransferEvent.directFlag = true; eRC20BridgeTransferEvent.directProtocol = 'UniswapV3'; - + return eRC20BridgeTransferEvent; } From 1d913f34f4f24249f22481b38ce450c16b7785ad Mon Sep 17 00:00:00 2001 From: robpal1990 Date: Mon, 20 Dec 2021 22:55:47 +0100 Subject: [PATCH 08/15] prettier --- event-pipeline/src/parsers/events/swap_events.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/event-pipeline/src/parsers/events/swap_events.ts b/event-pipeline/src/parsers/events/swap_events.ts index 70aa7f8b..de07eeb8 100644 --- a/event-pipeline/src/parsers/events/swap_events.ts +++ b/event-pipeline/src/parsers/events/swap_events.ts @@ -18,14 +18,14 @@ export function parseUniswapV3SwapEvent(eventLog: RawLogEntry): ERC20BridgeTrans // amount0 and amount1 are of opposite signs // neg value means token left the pool ie. maker // pos value means token entered the pool ie. taker - eRC20BridgeTransferEvent.fromToken = (decodedLog.amount0 >= 0) ? '0': '1' ; // taker_token - eRC20BridgeTransferEvent.toToken = (decodedLog.amount0 < 0) ? '0': '1' ; // maker_token + eRC20BridgeTransferEvent.fromToken = decodedLog.amount0 >= 0 ? '0' : '1'; // taker_token + eRC20BridgeTransferEvent.toToken = decodedLog.amount0 < 0 ? '0' : '1'; // maker_token eRC20BridgeTransferEvent.fromTokenAmount = decodedLog.amount0 >= 0 ? amount0 : amount1; // taker_token_amount eRC20BridgeTransferEvent.toTokenAmount = decodedLog.amount0 < 0 ? amount0 : amount1; // maker_token_amount eRC20BridgeTransferEvent.from = 'UniswapV3'; // maker eRC20BridgeTransferEvent.to = decodedLog.recipient.toLowerCase(); // taker eRC20BridgeTransferEvent.directFlag = true; eRC20BridgeTransferEvent.directProtocol = 'UniswapV3'; - + return eRC20BridgeTransferEvent; } From cf5d3609a03e4fb1c5e7876c2276aec3c78fa54a Mon Sep 17 00:00:00 2001 From: robpal1990 Date: Mon, 20 Dec 2021 23:11:46 +0100 Subject: [PATCH 09/15] fix python version --- event-pipeline-evm/Dockerfile | 2 +- event-pipeline/Dockerfile | 6 +++--- staking-api/Dockerfile | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/event-pipeline-evm/Dockerfile b/event-pipeline-evm/Dockerfile index 7c60d2b1..d18ef0d0 100644 --- a/event-pipeline-evm/Dockerfile +++ b/event-pipeline-evm/Dockerfile @@ -8,7 +8,7 @@ COPY pipeline-utils pipeline-utils COPY package.json yarn.lock tsconfig.json lerna.json ./ RUN apk update && \ apk upgrade && \ - apk add --no-cache --virtual build-dependencies bash git openssh python make g++ && \ + apk add --no-cache --virtual build-dependencies bash git openssh python3 make g++ && \ yarn --frozen-lockfile --no-cache RUN yarn build diff --git a/event-pipeline/Dockerfile b/event-pipeline/Dockerfile index 40977ace..f513c11d 100644 --- a/event-pipeline/Dockerfile +++ b/event-pipeline/Dockerfile @@ -8,7 +8,7 @@ COPY pipeline-utils pipeline-utils COPY package.json yarn.lock tsconfig.json lerna.json ./ RUN apk update && \ apk upgrade && \ - apk add --no-cache --virtual build-dependencies bash git openssh python make g++ && \ + apk add --no-cache --virtual build-dependencies bash git openssh python3 make g++ && \ yarn --frozen-lockfile --no-cache RUN yarn build @@ -26,9 +26,9 @@ COPY --from=build /usr/src/app/pipeline-utils/lib pipeline-utils/lib # Install event-pipeline runtime dependencies COPY event-pipeline/package.json event-pipeline/ -RUN apk add git python make g++ && \ +RUN apk add git python3 make g++ && \ yarn install --frozen-lockfile --no-cache --production && \ - apk del git python make g++ + apk del git python3 make g++ # Copy built files COPY --from=build /usr/src/app/event-pipeline/lib event-pipeline/lib/ diff --git a/staking-api/Dockerfile b/staking-api/Dockerfile index 3eb41d11..83092e74 100644 --- a/staking-api/Dockerfile +++ b/staking-api/Dockerfile @@ -6,7 +6,7 @@ COPY staking-api staking-api COPY package.json yarn.lock tsconfig.json lerna.json ./ RUN apk update && \ apk upgrade && \ - apk add --no-cache --virtual build-dependencies bash git openssh python make g++ && \ + apk add --no-cache --virtual build-dependencies bash git openssh python3 make g++ && \ yarn --frozen-lockfile --no-cache RUN yarn build From d1a3761faeaea91838292dcfdc131057c8628d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20P=C3=A9rez?= Date: Fri, 7 Jan 2022 15:00:10 -0600 Subject: [PATCH 10/15] feat/Add OTC event Scraping --- .dockerignore | 1 + docker-compose.yml | 2 +- event-pipeline/Dockerfile | 8 +-- .../1641418834000-AddOtcOrderFilledEvents.ts | 53 +++++++++++++++++++ event-pipeline/src/constants.ts | 52 ++++++++++++++++++ event-pipeline/src/entities/index.ts | 1 + .../src/entities/otc_order_filled_event.ts | 34 ++++++++++++ event-pipeline/src/ormconfig.ts | 2 + .../parsers/events/otc_order_filled_events.ts | 37 +++++++++++++ .../scripts/pull_and_save_events_by_topic.ts | 16 ++++++ 10 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 event-pipeline/migrations/1641418834000-AddOtcOrderFilledEvents.ts create mode 100644 event-pipeline/src/entities/otc_order_filled_event.ts create mode 100644 event-pipeline/src/parsers/events/otc_order_filled_events.ts diff --git a/.dockerignore b/.dockerignore index 7de31d78..68326534 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ postgres/ +node_modules diff --git a/docker-compose.yml b/docker-compose.yml index 654e1203..28c7aca0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: postgres: - image: postgres:9.6 + image: postgres:10.14 environment: - POSTGRES_USER=api - POSTGRES_PASSWORD=api diff --git a/event-pipeline/Dockerfile b/event-pipeline/Dockerfile index 40977ace..9d732a2a 100644 --- a/event-pipeline/Dockerfile +++ b/event-pipeline/Dockerfile @@ -8,9 +8,9 @@ COPY pipeline-utils pipeline-utils COPY package.json yarn.lock tsconfig.json lerna.json ./ RUN apk update && \ apk upgrade && \ - apk add --no-cache --virtual build-dependencies bash git openssh python make g++ && \ + apk add --no-cache --virtual build-dependencies bash git openssh python3 make g++ && \ yarn --frozen-lockfile --no-cache - + RUN yarn build # Stage 2 @@ -26,9 +26,9 @@ COPY --from=build /usr/src/app/pipeline-utils/lib pipeline-utils/lib # Install event-pipeline runtime dependencies COPY event-pipeline/package.json event-pipeline/ -RUN apk add git python make g++ && \ +RUN apk add git python3 make g++ && \ yarn install --frozen-lockfile --no-cache --production && \ - apk del git python make g++ + apk del git python3 make g++ # Copy built files COPY --from=build /usr/src/app/event-pipeline/lib event-pipeline/lib/ diff --git a/event-pipeline/migrations/1641418834000-AddOtcOrderFilledEvents.ts b/event-pipeline/migrations/1641418834000-AddOtcOrderFilledEvents.ts new file mode 100644 index 00000000..9b4483e2 --- /dev/null +++ b/event-pipeline/migrations/1641418834000-AddOtcOrderFilledEvents.ts @@ -0,0 +1,53 @@ +import { MigrationInterface, QueryRunner, Table } from 'typeorm'; + +const eventsOtcOrderFilledEvents = new Table({ + name: 'events.otc_order_filled_events', + columns: [ + { name: 'observed_timestamp', type: 'bigint' }, + { name: 'contract_address', type: 'varchar' }, + { name: 'transaction_hash', type: 'varchar', isPrimary: true }, + { name: 'transaction_index', type: 'bigint' }, + { name: 'log_index', type: 'bigint', isPrimary: true }, + { name: 'block_hash', type: 'varchar' }, + { name: 'block_number', type: 'bigint' }, + + { name: 'order_hash', type: 'varchar' }, + { name: 'maker_address', type: 'varchar' }, + { name: 'taker_address', type: 'varchar' }, + { name: 'maker_token_address', type: 'varchar' }, + { name: 'taker_token_address', type: 'varchar' }, + { name: 'maker_token_filled_amount', type: 'numeric' }, + { name: 'taker_token_filled_amount', type: 'numeric' }, + ], +}); + +const indexQuery = ` + CREATE INDEX otc_order_filled_events_transaction_hash_index + ON events.otc_order_filled_events (transaction_hash); + CREATE INDEX otc_order_filled_events_block_number_index + ON events.otc_order_filled_events (block_number); + CREATE INDEX otc_order_filled_events_maker_address_index + ON events.otc_order_filled_events (maker_address); + CREATE INDEX otc_order_filled_events_order_hash_index + ON events.otc_order_filled_events (order_hash); + +`; + +const dropIndexQuery = ` + DROP INDEX events.otc_order_filled_events_transaction_hash_index; + DROP INDEX events.otc_order_filled_events_block_number_index; + DROP INDEX events.otc_order_filled_events_maker_address_index; + DROP INDEX events.otc_order_filled_events_order_hash_index; +`; + +export class AddOtcOrderFilledEvents1641418834000 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable(eventsOtcOrderFilledEvents); + await queryRunner.query(indexQuery); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(dropIndexQuery); + await queryRunner.dropTable(eventsOtcOrderFilledEvents); + } +} diff --git a/event-pipeline/src/constants.ts b/event-pipeline/src/constants.ts index 41daf5b7..e038fb22 100644 --- a/event-pipeline/src/constants.ts +++ b/event-pipeline/src/constants.ts @@ -38,6 +38,8 @@ export const SWAP_V3_EVENT_TOPIC = [ '0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67', '0x000000000000000000000000def1c0ded9bec7f1a1670819833240f027b25eff', ]; +export const OTC_ORDER_FILLED_EVENT_TOPIC = ['0xac75f773e3a92f1a02b12134d65e1f47f8a14eabe4eaf1e24624918e6a8b269f']; +export const OTC_ORDERS_FEATURE_START_BLOCK = 13143075; export { EXPIRED_RFQ_ORDER_ABI, @@ -309,3 +311,53 @@ export const V3_FILL_ABI = { name: 'Fill', type: 'event', }; + +export const OTC_ORDER_FILLED_ABI = { + anonymous: false, + inputs: [ + { + indexed: false, + internalType: 'bytes32', + name: 'orderHash', + type: 'bytes32', + }, + { + indexed: false, + internalType: 'address', + name: 'maker', + type: 'address', + }, + { + indexed: false, + internalType: 'address', + name: 'taker', + type: 'address', + }, + { + indexed: false, + internalType: 'address', + name: 'makerToken', + type: 'address', + }, + { + indexed: false, + internalType: 'address', + name: 'takerToken', + type: 'address', + }, + { + indexed: false, + internalType: 'uint128', + name: 'makerTokenFilledAmount', + type: 'uint128', + }, + { + indexed: false, + internalType: 'uint128', + name: 'takerTokenFilledAmount', + type: 'uint128', + }, + ], + name: 'OtcOrderFilled', + type: 'event', +}; diff --git a/event-pipeline/src/entities/index.ts b/event-pipeline/src/entities/index.ts index 1cecba09..30fec4e2 100644 --- a/event-pipeline/src/entities/index.ts +++ b/event-pipeline/src/entities/index.ts @@ -32,6 +32,7 @@ export { EpochFinalizedEvent } from './epoch_finalized_event'; export { RewardsPaidEvent } from './rewards_paid_event'; export { StakingProxyDeployment } from './staking_proxy_deployment'; export { TransactionExecutionEvent } from './transaction_execution_event'; +export { OtcOrderFilledEvent } from './otc_order_filled_event'; @Entity({ name: 'blocks', schema: 'events' }) export class Block extends BlockTemplate {} diff --git a/event-pipeline/src/entities/otc_order_filled_event.ts b/event-pipeline/src/entities/otc_order_filled_event.ts new file mode 100644 index 00000000..ad961e31 --- /dev/null +++ b/event-pipeline/src/entities/otc_order_filled_event.ts @@ -0,0 +1,34 @@ +import { bigNumberTransformer } from '@0x/pipeline-utils'; +import { BigNumber } from '@0x/utils'; +import { Column, Entity } from 'typeorm'; + +import { Event } from '@0x/pipeline-utils'; + +import { ProxyType } from '../utils'; + +// These events come directly from the Exchange contract and are fired whenever +// someone fills an order. +@Entity({ name: 'otc_order_filled_events', schema: 'events' }) +export class OtcOrderFilledEvent extends Event { + // The order hash. + @Column({ name: 'order_hash' }) + public orderHash!: string; + // The address of the maker. + @Column({ name: 'maker_address' }) + public makerAddress!: string; + // The address of the taker (may be null). + @Column({ name: 'taker_address' }) + public takerAddress!: string; + // The address of the maker token. + @Column({ name: 'maker_token_address', type: 'varchar', nullable: true }) + public makerTokenAddress!: string | null; + // The address of the taker token. + @Column({ name: 'taker_token_address', type: 'varchar', nullable: true }) + public takerTokenAddress!: string | null; + // The amount of the maker asset which was filled. + @Column({ name: 'maker_token_filled_amount', type: 'numeric', transformer: bigNumberTransformer }) + public makerTokenFilledAmount!: BigNumber; + // The amount of the taker asset which was filled. + @Column({ name: 'taker_token_filled_amount', type: 'numeric', transformer: bigNumberTransformer }) + public takerTokenFilledAmount!: BigNumber; +} diff --git a/event-pipeline/src/ormconfig.ts b/event-pipeline/src/ormconfig.ts index 268ad0a7..4822ab5e 100644 --- a/event-pipeline/src/ormconfig.ts +++ b/event-pipeline/src/ormconfig.ts @@ -25,6 +25,7 @@ import { EpochFinalizedEvent, RewardsPaidEvent, StakingProxyDeployment, + OtcOrderFilledEvent, ERC20BridgeTransferEvent, TransformedERC20Event, TransactionExecutionEvent, @@ -58,6 +59,7 @@ const entities = [ EpochFinalizedEvent, RewardsPaidEvent, StakingProxyDeployment, + OtcOrderFilledEvent, ERC20BridgeTransferEvent, TransformedERC20Event, TransactionExecutionEvent, diff --git a/event-pipeline/src/parsers/events/otc_order_filled_events.ts b/event-pipeline/src/parsers/events/otc_order_filled_events.ts new file mode 100644 index 00000000..9b825989 --- /dev/null +++ b/event-pipeline/src/parsers/events/otc_order_filled_events.ts @@ -0,0 +1,37 @@ +import { assetDataUtils } from '@0x/order-utils'; +import { AssetProxyId } from '@0x/types'; +import { RawLogEntry } from 'ethereum-types'; + +import { OtcOrderFilledEvent } from '../../entities'; +import { convertAssetProxyIdToType } from '../../utils'; +import { parseEvent } from './parse_event'; +import { OTC_ORDER_FILLED_ABI } from '../../constants'; + +const abiCoder = require('web3-eth-abi'); + +/** + * Parses raw event logs for a OTC Fill Event and returns an array of + * OtcOrderFilledEvent entities. + * @param eventLog Raw event log (e.g. returned from contract-wrappers). + */ + +export function parseOtcOrderFilledEvent(eventLog: RawLogEntry): OtcOrderFilledEvent { + const otcOrderFillEvent = new OtcOrderFilledEvent(); + parseEvent(eventLog, otcOrderFillEvent); + + const decodedLog = abiCoder.decodeLog( + OTC_ORDER_FILLED_ABI.inputs, + eventLog.data, + eventLog.topics.slice(1, eventLog.topics.length), + ); + + otcOrderFillEvent.orderHash = decodedLog.orderHash.toLowerCase(); + otcOrderFillEvent.makerAddress = decodedLog.maker.toLowerCase(); + otcOrderFillEvent.takerAddress = decodedLog.taker.toLowerCase(); + otcOrderFillEvent.makerTokenAddress = decodedLog.makerToken.toLowerCase(); + otcOrderFillEvent.takerTokenAddress = decodedLog.takerToken.toLowerCase(); + otcOrderFillEvent.makerTokenFilledAmount = decodedLog.makerTokenFilledAmount; + otcOrderFillEvent.takerTokenFilledAmount = decodedLog.takerTokenFilledAmount; + + return otcOrderFillEvent; +} diff --git a/event-pipeline/src/scripts/pull_and_save_events_by_topic.ts b/event-pipeline/src/scripts/pull_and_save_events_by_topic.ts index 3ce9d1a9..6a9f8428 100644 --- a/event-pipeline/src/scripts/pull_and_save_events_by_topic.ts +++ b/event-pipeline/src/scripts/pull_and_save_events_by_topic.ts @@ -14,6 +14,7 @@ import { FillEvent, V4CancelEvent, ExpiredRfqOrderEvent, + OtcOrderFilledEvent, } from '../entities'; import { ETHEREUM_RPC_URL, FIRST_SEARCH_BLOCK } from '../config'; @@ -36,6 +37,8 @@ import { NEW_BRIDGEFILL_BLOCK, FLASHWALLET_ADDRESS, SWAP_V3_EVENT_TOPIC, + OTC_ORDER_FILLED_EVENT_TOPIC, + OTC_ORDERS_FEATURE_START_BLOCK, } from '../constants'; import { parseTransformedERC20Event } from '../parsers/events/transformed_erc20_events'; @@ -54,6 +57,7 @@ import { parseFillEvent } from '../parsers/events/fill_events'; import { parseNativeFillFromFillEvent } from '../parsers/events/fill_events'; import { parseV4CancelEvent } from '../parsers/events/v4_cancel_events'; import { parseExpiredRfqOrderEvent } from '../parsers/events/expired_rfq_order_events'; +import { parseOtcOrderFilledEvent } from '../parsers/events/otc_order_filled_events'; import { PullAndSaveEventsByTopic } from './utils/event_abi_utils'; @@ -216,6 +220,18 @@ export class EventsByTopicScraper { parseExpiredRfqOrderEvent, {}, ), + pullAndSaveEventsByTopic.getParseSaveEventsByTopic( + connection, + web3Source, + latestBlockWithOffset, + 'OtcOrderFilledEvent', + 'otc_order_filled_events', + OTC_ORDER_FILLED_EVENT_TOPIC, + EXCHANGE_PROXY_ADDRESS, + OTC_ORDERS_FEATURE_START_BLOCK, + parseOtcOrderFilledEvent, + {}, + ), ]); const endTime = new Date().getTime(); From d5676272bf118b8f1d5bdc079ceda432eb357344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20P=C3=A9rez?= Date: Fri, 21 Jan 2022 10:35:11 -0600 Subject: [PATCH 11/15] feat/OTC native event scraping --- .../parsers/events/otc_order_filled_events.ts | 33 +++++++++++++++++++ .../scripts/pull_and_save_events_by_topic.ts | 17 +++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/event-pipeline/src/parsers/events/otc_order_filled_events.ts b/event-pipeline/src/parsers/events/otc_order_filled_events.ts index 9b825989..1c13ef5c 100644 --- a/event-pipeline/src/parsers/events/otc_order_filled_events.ts +++ b/event-pipeline/src/parsers/events/otc_order_filled_events.ts @@ -1,11 +1,13 @@ import { assetDataUtils } from '@0x/order-utils'; import { AssetProxyId } from '@0x/types'; import { RawLogEntry } from 'ethereum-types'; +import { BigNumber } from '@0x/utils'; import { OtcOrderFilledEvent } from '../../entities'; import { convertAssetProxyIdToType } from '../../utils'; import { parseEvent } from './parse_event'; import { OTC_ORDER_FILLED_ABI } from '../../constants'; +import { NativeFill } from '../../entities'; const abiCoder = require('web3-eth-abi'); @@ -35,3 +37,34 @@ export function parseOtcOrderFilledEvent(eventLog: RawLogEntry): OtcOrderFilledE return otcOrderFillEvent; } + +export function parseNativeFillFromV4OtcOrderFilledEvent(eventLog: RawLogEntry): NativeFill { + const nativeFill = new NativeFill(); + parseEvent(eventLog, nativeFill); + // decode the basic info directly into nativeFill + + const decodedLog = abiCoder.decodeLog(OTC_ORDER_FILLED_ABI.inputs, eventLog.data); + + nativeFill.orderHash = decodedLog.orderHash.toLowerCase(); + nativeFill.maker = decodedLog.maker.toLowerCase(); + nativeFill.taker = decodedLog.taker.toLowerCase(); + nativeFill.feeRecipient = null; + + nativeFill.makerToken = decodedLog.makerToken.toLowerCase(); + nativeFill.takerToken = decodedLog.takerToken.toLowerCase(); + nativeFill.takerTokenFilledAmount = new BigNumber(decodedLog.takerTokenFilledAmount); + nativeFill.makerTokenFilledAmount = new BigNumber(decodedLog.makerTokenFilledAmount); + nativeFill.takerFeePaid = null; + nativeFill.makerFeePaid = null; + nativeFill.takerProxyType = null; + nativeFill.makerProxyType = null; + nativeFill.takerFeeToken = null; + nativeFill.makerFeeToken = null; + nativeFill.protocolFeePaid = null; + nativeFill.pool = null; + + nativeFill.nativeOrderType = 'OTC Order'; + nativeFill.protocolVersion = 'v4'; + + return nativeFill; +} diff --git a/event-pipeline/src/scripts/pull_and_save_events_by_topic.ts b/event-pipeline/src/scripts/pull_and_save_events_by_topic.ts index 6a9f8428..6ded2e6b 100644 --- a/event-pipeline/src/scripts/pull_and_save_events_by_topic.ts +++ b/event-pipeline/src/scripts/pull_and_save_events_by_topic.ts @@ -57,7 +57,10 @@ import { parseFillEvent } from '../parsers/events/fill_events'; import { parseNativeFillFromFillEvent } from '../parsers/events/fill_events'; import { parseV4CancelEvent } from '../parsers/events/v4_cancel_events'; import { parseExpiredRfqOrderEvent } from '../parsers/events/expired_rfq_order_events'; -import { parseOtcOrderFilledEvent } from '../parsers/events/otc_order_filled_events'; +import { + parseOtcOrderFilledEvent, + parseNativeFillFromV4OtcOrderFilledEvent, +} from '../parsers/events/otc_order_filled_events'; import { PullAndSaveEventsByTopic } from './utils/event_abi_utils'; @@ -136,6 +139,18 @@ export class EventsByTopicScraper { parseV4RfqOrderFilledEvent, {}, ), + pullAndSaveEventsByTopic.getParseSaveEventsByTopic( + connection, + web3Source, + latestBlockWithOffset, + 'NativeFillFromOTC', + 'native_fills', + OTC_ORDER_FILLED_EVENT_TOPIC, + EXCHANGE_PROXY_ADDRESS, + OTC_ORDERS_FEATURE_START_BLOCK, + parseNativeFillFromV4OtcOrderFilledEvent, + { protocolVersion: 'v4', nativeOrderType: 'OTC Order' }, + ), pullAndSaveEventsByTopic.getParseSaveEventsByTopic( connection, web3Source, From 2079f2711dbf9ca29a06e5752f6a62a7b8561000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20P=C3=A9rez?= Date: Mon, 22 Aug 2022 10:05:32 -0500 Subject: [PATCH 12/15] Avoid division by 0 --- staking-api/src/queries.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/staking-api/src/queries.ts b/staking-api/src/queries.ts index c2f3d855..168268b7 100644 --- a/staking-api/src/queries.ts +++ b/staking-api/src/queries.ts @@ -469,9 +469,18 @@ WITH ELSE fbp.protocol_fees / tf.total_protocol_fees END AS share_of_fees - , fbp.num_fills::FLOAT / tf.total_fills::FLOAT AS share_of_fills , CASE - WHEN tf.total_protocol_fees = 0 THEN + WHEN tf.total_fills = 0 THEN + NULL + ELSE + fbp.num_fills::FLOAT / tf.total_fills::FLOAT + END AS share_of_fills + , CASE + WHEN + tf.total_protocol_fees = 0 OR + fbp.protocol_fees = 0 OR + ts.total_staked = 0 + THEN NULL ELSE (cebs.zrx_delegated / ts.total_staked) From 1b4cc1b6489f913cdbb7147187cada1bc82f5ed3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20P=C3=A9rez?= Date: Mon, 22 Aug 2022 14:38:35 -0500 Subject: [PATCH 13/15] Avoid another division by 0 --- staking-api/src/queries.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/staking-api/src/queries.ts b/staking-api/src/queries.ts index 168268b7..6fd8ad9c 100644 --- a/staking-api/src/queries.ts +++ b/staking-api/src/queries.ts @@ -560,9 +560,11 @@ export const currentEpochPoolStatsQuery = ` fbp.protocol_fees / tf.total_protocol_fees END AS share_of_fees , CASE - WHEN tf.total_protocol_fees = 0 THEN - NULL - ELSE + WHEN + tf.total_protocol_fees = 0 OR + fbp.protocol_fees = 0 OR + ts.total_staked = 0 + THEN (cebs.zrx_delegated / ts.total_staked) / (fbp.protocol_fees / tf.total_protocol_fees) END AS approximate_stake_ratio From 0ae45f9fd282b37bf21a8ab9d7096d9f887d5d1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20P=C3=A9rez?= Date: Mon, 22 Aug 2022 15:50:38 -0500 Subject: [PATCH 14/15] Avoid another division by 0 - typo --- staking-api/src/queries.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/staking-api/src/queries.ts b/staking-api/src/queries.ts index 6fd8ad9c..762feb27 100644 --- a/staking-api/src/queries.ts +++ b/staking-api/src/queries.ts @@ -565,6 +565,8 @@ export const currentEpochPoolStatsQuery = ` fbp.protocol_fees = 0 OR ts.total_staked = 0 THEN + NULL + ELSE (cebs.zrx_delegated / ts.total_staked) / (fbp.protocol_fees / tf.total_protocol_fees) END AS approximate_stake_ratio From f8a279bf527e253a3df43a41386ea53a700cb73e Mon Sep 17 00:00:00 2001 From: Marcin Wolny Date: Wed, 14 Sep 2022 12:11:46 +0200 Subject: [PATCH 15/15] Stop use Drone instead use Github actions --- .drone.yml | 176 ---------------------------------- .github/workflows/docker.yml | 56 +++++++++++ event-pipeline-evm/Dockerfile | 28 ++++-- event-pipeline/Dockerfile | 20 ++-- staking-api/Dockerfile | 18 +++- 5 files changed, 101 insertions(+), 197 deletions(-) delete mode 100644 .drone.yml create mode 100644 .github/workflows/docker.yml diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index e0e84498..00000000 --- a/.drone.yml +++ /dev/null @@ -1,176 +0,0 @@ -name: event-pipeline-sha -kind: pipeline -type: kubernetes - -platform: - os: linux - arch: amd64 - -steps: - - name: publish-sha - image: plugins/ecr - settings: - context: . - dockerfile: event-pipeline/Dockerfile - repo: - from_secret: ecr_repo_event-pipeline - registry: - from_secret: ecr_registry - region: us-east-1 - tags: - - ${DRONE_COMMIT_SHA} -trigger: - event: - exclude: - - pull_request -node_selector: - drone-builds: true ---- -name: event-pipeline -kind: pipeline -type: kubernetes - -platform: - os: linux - arch: amd64 - -steps: - - name: publish - image: plugins/ecr - settings: - context: . - dockerfile: event-pipeline/Dockerfile - repo: - from_secret: ecr_repo_event-pipeline - registry: - from_secret: ecr_registry - region: us-east-1 - tags: - - latest - -trigger: - branch: - - master - event: - include: - - push -node_selector: - drone-builds: true ---- -name: event-pipeline-evm-sha -kind: pipeline -type: kubernetes - -platform: - os: linux - arch: amd64 - -steps: - - name: publish-sha - image: plugins/ecr - settings: - context: . - dockerfile: event-pipeline-evm/Dockerfile - repo: - from_secret: ecr_repo_event-pipeline-evm - registry: - from_secret: ecr_registry - region: us-east-1 - tags: - - ${DRONE_COMMIT_SHA} -trigger: - event: - exclude: - - pull_request -node_selector: - drone-builds: true ---- -name: event-pipeline-evm -kind: pipeline -type: kubernetes - -platform: - os: linux - arch: amd64 - -steps: - - name: publish - image: plugins/ecr - settings: - context: . - dockerfile: event-pipeline-evm/Dockerfile - repo: - from_secret: ecr_repo_event-pipeline-evm - registry: - from_secret: ecr_registry - region: us-east-1 - tags: - - latest - -trigger: - branch: - - master - event: - include: - - push -node_selector: - drone-builds: true ---- -name: staking-api-sha -kind: pipeline -type: kubernetes - -platform: - os: linux - arch: amd64 - -steps: - - name: publish-sha - image: plugins/ecr - settings: - context: . - dockerfile: staking-api/Dockerfile - repo: - from_secret: ecr_repo_staking-api - registry: - from_secret: ecr_registry - region: us-east-1 - tags: - - ${DRONE_COMMIT_SHA} -trigger: - event: - exclude: - - pull_request -node_selector: - drone-builds: true ---- -name: staking-api -kind: pipeline -type: kubernetes - -platform: - os: linux - arch: amd64 - -steps: - - name: publish - image: plugins/ecr - settings: - context: . - dockerfile: staking-api/Dockerfile - repo: - from_secret: ecr_repo_staking-api - registry: - from_secret: ecr_registry - region: us-east-1 - tags: - - latest - -trigger: - branch: - - master - event: - include: - - push -node_selector: - drone-builds: true diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..ba289ef8 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,56 @@ +name: docker + +on: + pull_request: + branches: + - master + push: + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# This is required by aws-actions. +# +permissions: + id-token: write + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: hadolint/hadolint-action@v2.1.0 + with: + recursive: true + ignore: DL3018 + + build: + runs-on: ubuntu-latest + strategy: + matrix: + build: + - dockerfile: event-pipeline-evm/Dockerfile + name: event-pipeline-evm + - dockerfile: event-pipeline/Dockerfile + name: event-pipeline + - dockerfile: staking-api/Dockerfile + name: staking-api + steps: + - uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.AWSROLE }} + aws-region: us-east-1 + - uses: aws-actions/amazon-ecr-login@v1 + - uses: actions/checkout@v3 + - uses: docker/setup-buildx-action@v2 + - uses: docker/build-push-action@v3 + with: + push: ${{ github.ref == 'refs/heads/master' }} + context: . + file: ${{ matrix.build.dockerfile }} + platforms: linux/amd64 + tags: ${{ secrets.REGISTRY }}/0x/${{ matrix.build.name }}:${{ github.sha }} diff --git a/event-pipeline-evm/Dockerfile b/event-pipeline-evm/Dockerfile index d18ef0d0..f3d84d48 100644 --- a/event-pipeline-evm/Dockerfile +++ b/event-pipeline-evm/Dockerfile @@ -1,38 +1,50 @@ # Stage 1 - Build -FROM node:12-alpine as build +# +FROM node:12-alpine AS build WORKDIR /usr/src/app # Install app dependencies COPY event-pipeline-evm event-pipeline-evm COPY pipeline-utils pipeline-utils COPY package.json yarn.lock tsconfig.json lerna.json ./ -RUN apk update && \ - apk upgrade && \ - apk add --no-cache --virtual build-dependencies bash git openssh python3 make g++ && \ - yarn --frozen-lockfile --no-cache -RUN yarn build +RUN apk add --update --no-cache --virtual build-dependencies \ + bash \ + git \ + openssh \ + python3 \ + make \ + g++ && \ + yarn --frozen-lockfile --no-cache && \ + yarn build # Stage 2 +# FROM node:12-alpine WORKDIR /usr/src/app # Setup monorepo +# COPY package.json yarn.lock tsconfig.json lerna.json ./ # Setup pipeline-utils +# COPY pipeline-utils/package.json pipeline-utils/ COPY --from=build /usr/src/app/pipeline-utils/lib pipeline-utils/lib # Install event-pipeline-evm runtime dependencies +# COPY event-pipeline-evm/package.json event-pipeline-evm/ -RUN apk add git && \ +RUN apk add --update --no-cache git && \ yarn install --frozen-lockfile --no-cache --production && \ + yarn cache clean && \ apk del git # Copy built files +# COPY --from=build /usr/src/app/event-pipeline-evm/lib event-pipeline-evm/lib/ -#Start +# Start +# WORKDIR /usr/src/app/event-pipeline-evm CMD [ "yarn", "migrate_and_start" ] diff --git a/event-pipeline/Dockerfile b/event-pipeline/Dockerfile index 9d732a2a..973f9fcd 100644 --- a/event-pipeline/Dockerfile +++ b/event-pipeline/Dockerfile @@ -1,17 +1,16 @@ # Stage 1 - Build -FROM node:12-alpine as build +# +FROM node:12-alpine AS build WORKDIR /usr/src/app # Install app dependencies COPY event-pipeline event-pipeline COPY pipeline-utils pipeline-utils COPY package.json yarn.lock tsconfig.json lerna.json ./ -RUN apk update && \ - apk upgrade && \ - apk add --no-cache --virtual build-dependencies bash git openssh python3 make g++ && \ - yarn --frozen-lockfile --no-cache -RUN yarn build +RUN apk add --update --no-cache --virtual build-dependencies bash git openssh python3 make g++ && \ + yarn --frozen-lockfile --no-cache && \ + yarn build # Stage 2 FROM node:12-alpine @@ -26,9 +25,14 @@ COPY --from=build /usr/src/app/pipeline-utils/lib pipeline-utils/lib # Install event-pipeline runtime dependencies COPY event-pipeline/package.json event-pipeline/ -RUN apk add git python3 make g++ && \ +RUN apk add --update --no-cache --virtual build-dependencies \ + git \ + python3 \ + make \ + g++ && \ yarn install --frozen-lockfile --no-cache --production && \ - apk del git python3 make g++ + apk del build-dependencies && \ + yarn cache clean # Copy built files COPY --from=build /usr/src/app/event-pipeline/lib event-pipeline/lib/ diff --git a/staking-api/Dockerfile b/staking-api/Dockerfile index 83092e74..cd573ece 100644 --- a/staking-api/Dockerfile +++ b/staking-api/Dockerfile @@ -1,14 +1,20 @@ # Stage 1 FROM node:12-alpine as build + WORKDIR /usr/src/app # Install app dependencies COPY staking-api staking-api COPY package.json yarn.lock tsconfig.json lerna.json ./ -RUN apk update && \ - apk upgrade && \ - apk add --no-cache --virtual build-dependencies bash git openssh python3 make g++ && \ + +RUN apk add --update --no-cache --virtual build-dependencies \ + bash \ + git \ + openssh \ + python3 \ + make \ + g++ && \ yarn --frozen-lockfile --no-cache - + RUN yarn build # Stage 2 @@ -20,7 +26,9 @@ ENV NODE_ENV=production # Install runtime dependencies COPY staking-api/package.json yarn.lock ./ -RUN yarn install --frozen-lockfile --no-cache + +RUN yarn install --frozen-lockfile && \ + yarn cache clean # Copy built files COPY --from=build /usr/src/app/staking-api/lib /usr/src/app/staking-api/lib