Merge pull request #1 from dexaai/feature/initial-impl

.editorconfig

@@ -0,0 +1,10 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+tab_width = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true

.env.example

@@ -0,0 +1,44 @@
+# ------------------------------------------------------------------------------
+# This is an example .env file.
+#
+# All of these environment vars must be defined either in your environment or in
+# a local .env file in order to run this app.
+# ------------------------------------------------------------------------------
+
+# Nango simplifies handling Twitter API OAuth (https://nango.dev)
+NANGO_CONNECTION_ID=
+NANGO_SECRET_KEY=
+NANGO_CALLBACK_URL='https://api.nango.dev/oauth/callback'
+
+# Twitter has different API rate limits and quotas per plan, so in order to 
+# rate-limit effectively, we need to know which plan you're using.
+# Must be one of: 'free' | 'basic' | 'pro' | 'enterprise'
+TWITTER_API_PLAN=
+
+# Answer engine settings
+# If you're using openai (the default), you must define OPENAI_API_KEY
+#ANSWER_ENGINE='openai' | 'dexa'
+#OPENAI_API_KEY=
+
+# Optional database settings
+# If REDIS_URL isn't defined and REQUIRE_REDIS=true, the app will abort.
+# If REDIS_URL isn't defined and REQUIRE_REDIS!=true, the app will use an 
+# in-memory store.
+#REQUIRE_REDIS=
+#REDIS_URL=
+#REDIS_NAMESPACE_TWEETS=
+#REDIS_NAMESPACE_USERS=
+#REDIS_NAMESPACE_MESSAGES=
+#REDIS_NAMESPACE_STATE=
+#REDIS_NAMESPACE_MENTIONS_PREFIX=
+
+# Optional behavior settings generally specified locally during debugging.
+# Think of these as quick & dirty CLI switches.
+#DEBUG=true
+#DRY_RUN=true
+#NO_CACHE=true
+#EARLY_EXIT=true
+#FORCE_REPLY=true
+#RESOLVE_ALL_MENTIONS=true
+#DEBUG_TWEET='tweetId1,tweetId2,...' # Comma-separated list of tweet IDs to process
+#SINCE_MENTION_ID='tweetId' # Overrides the default tweet sinceMentionId

.github/workflows/main.yml

@@ -0,0 +1,48 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    name: Test Node.js ${{ matrix.node-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        node-version:
+          - 20
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v3
+        id: pnpm-install
+        with:
+          version: 8
+          run_install: false
+
+      - name: Get pnpm store directory
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+
+      - name: Setup pnpm cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run test
+        run: pnpm run test

.gitignore

@@ -1,8 +1,8 @@
 dist
 node_modules
 .env
-.env.*
 *.log
 .idea
 .vscode
 .DS_Store
+.tsimp

.prettierignore

@@ -0,0 +1,10 @@
+*.env
+*.log
+*.jsonl
+**/node_modules
+**/dist
+**/build
+**/dbschema
+**/public
+.next
+.tsimp

.prettierrc.cjs

@@ -0,0 +1,17 @@
+module.exports = {
+  plugins: [require('@trivago/prettier-plugin-sort-imports')],
+  singleQuote: true,
+  jsxSingleQuote: true,
+  semi: false,
+  useTabs: false,
+  tabWidth: 2,
+  bracketSpacing: true,
+  bracketSameLine: false,
+  arrowParens: 'always',
+  trailingComma: 'none',
+  importOrderParserPlugins: ['typescript'],
+  importOrder: ['^node:.*', '<THIRD_PARTY_MODULES>', '^(@/(.*)$)', '^[./]'],
+  importOrderSeparation: true,
+  importOrderSortSpecifiers: true,
+  importOrderGroupNamespaceSpecifiers: true
+}

bin/debug-get-twitter-user.ts

@@ -0,0 +1,18 @@
+import { getTwitterClient } from '../src/twitter-client.js'
+
+async function main() {
+  const twitterClient = await getTwitterClient()
+  const { data: user } = await twitterClient.users.findUserByUsername(
+    'dustyplaylist'
+  )
+  console.log(user)
+}
+
+main()
+  .then(() => {
+    process.exit(0)
+  })
+  .catch((err) => {
+    console.error(err)
+    process.exit(1)
+  })

bin/index.ts

@@ -0,0 +1,146 @@
+import delay from 'delay'
+
+import * as config from '../src/config.js'
+import * as db from '../src/db.js'
+import type * as types from '../src/types.js'
+import { validateAnswerEngine } from '../src/answer-engine.js'
+import { respondToNewMentions } from '../src/respond-to-new-mentions.js'
+import { getTwitterClient } from '../src/twitter-client.js'
+import { maxTwitterId } from '../src/twitter-utils.js'
+
+async function main() {
+  const debug = !!process.env.DEBUG
+  const dryRun = !!process.env.DRY_RUN
+  const noCache = !!process.env.NO_CACHE
+  const earlyExit = !!process.env.EARLY_EXIT
+  const forceReply = !!process.env.FORCE_REPLY
+  const resolveAllMentions = !!process.env.RESOLVE_ALL_MENTIONS
+  const debugTweetIds = process.env.DEBUG_TWEET_IDS?.split(',').map((id) =>
+    id.trim()
+  )
+  const overrideSinceMentionId = process.env.SINCE_MENTION_ID
+  const overrideMaxNumMentionsToProcess = parseInt(
+    process.env.MAX_NUM_MENTIONS_TO_PROCESS ?? '',
+    10
+  )
+  const answerEngine: types.AnswerEngineType =
+    (process.env.ANSWER_ENGINE as types.AnswerEngineType) ?? 'openai'
+  validateAnswerEngine(answerEngine)
+
+  let twitterClient = await getTwitterClient()
+  const { data: user } = await twitterClient.users.findMyUser()
+
+  if (!user?.id) {
+    throw new Error('twitter error unable to fetch current user')
+  }
+
+  async function refreshTwitterAuth() {
+    twitterClient = await getTwitterClient()
+  }
+
+  console.log('automating user', user.username)
+
+  const maxNumMentionsToProcess = isNaN(overrideMaxNumMentionsToProcess)
+    ? config.defaultMaxNumMentionsToProcessPerBatch
+    : overrideMaxNumMentionsToProcess
+
+  let initialSinceMentionId =
+    (resolveAllMentions
+      ? undefined
+      : overrideSinceMentionId || (await db.getSinceMentionId())) ?? '0'
+
+  const ctx: types.Context = {
+    // Dynamic a state which gets persisted to the db
+    sinceMentionId: initialSinceMentionId,
+
+    // Services
+    twitterClient,
+
+    // Constant app runtime config
+    debug,
+    dryRun,
+    noCache,
+    earlyExit,
+    forceReply,
+    resolveAllMentions,
+    maxNumMentionsToProcess,
+    debugTweetIds,
+    twitterBotHandle: `@${user.username}`,
+    twitterBotHandleL: `@${user.username.toLowerCase()}`,
+    twitterBotUserId: user.id,
+    answerEngine
+  }
+
+  const batches: types.TweetMentionBatch[] = []
+
+  do {
+    try {
+      console.log()
+      const batch = await respondToNewMentions(ctx)
+      batches.push(batch)
+
+      if (batch.sinceMentionId && !ctx.debugTweetIds?.length) {
+        ctx.sinceMentionId = maxTwitterId(
+          ctx.sinceMentionId,
+          batch.sinceMentionId
+        )
+
+        if (!overrideMaxNumMentionsToProcess && !resolveAllMentions) {
+          // Make sure it's in sync in case other processes are writing to the store
+          // as well. Note: this still has the potential for a race condition, but
+          // it's not enough to worry about for our use case.
+          const recentSinceMentionId = await db.getSinceMentionId()
+          ctx.sinceMentionId = maxTwitterId(
+            ctx.sinceMentionId,
+            recentSinceMentionId
+          )
+
+          if (ctx.sinceMentionId && !dryRun) {
+            await db.setSinceMentionId(ctx.sinceMentionId)
+          }
+        }
+      }
+
+      if (ctx.earlyExit) {
+        break
+      }
+
+      console.log(
+        `processed ${batch.messages?.length ?? 0} messages`,
+        batch.messages
+      )
+
+      if (debugTweetIds?.length) {
+        break
+      }
+
+      if (batch.hasNetworkError) {
+        console.warn('network error; sleeping...')
+        await delay(10_000)
+      }
+
+      if (batch.hasTwitterRateLimitError) {
+        console.warn('twitter rate limit error; sleeping...')
+        await delay(30_000)
+      }
+
+      if (batch.hasTwitterAuthError) {
+        console.warn('twitter auth error; refreshing...')
+        await refreshTwitterAuth()
+      }
+    } catch (err) {
+      console.error('top-level error', err)
+      await delay(5000)
+      await refreshTwitterAuth()
+    }
+  } while (true)
+}
+
+main()
+  .then(() => {
+    process.exit(0)
+  })
+  .catch((err) => {
+    console.error(err)
+    process.exit(1)
+  })

license

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Dexa
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.