Merge pull request #294 from holaplex/dev

deploy

.env

@@ -2,5 +2,5 @@ ARWEAVE_URL=https://arweave.net/
 IPFS_CDN=https://ipfs.cache.holaplex.com/
 ARWEAVE_CDN=https://arweave.net/
 HTTP_INDEXER_TIMEOUT=10
-ASSET_PROXY_ENDPOINT=https://assets[n].holaplex.com/
+ASSET_PROXY_ENDPOINT=https://assets[n].holaplex.tools/
 ASSET_PROXY_COUNT=5

crates/core/migrations/2022-03-22-214148_add_slot_col_and_trigger_to_token_account/down.sql

@@ -0,0 +1,4 @@
+alter table token_accounts
+drop column slot;
+drop trigger set_token_account_updated_at on token_accounts;
+drop function trigger_set_updated_at_timestamp();

crates/core/migrations/2022-03-22-214148_add_slot_col_and_trigger_to_token_account/up.sql

@@ -0,0 +1,18 @@
+alter table token_accounts
+add column slot bigint null;
+
+alter table token_accounts alter updated_at set default now();
+
+create  function trigger_set_updated_at_timestamp()
+returns trigger as $$
+begin
+    new.updated_at = now();
+    return new;
+end;
+$$ language 'plpgsql';
+
+create trigger set_token_account_updated_at
+before update on token_accounts
+for each row
+execute procedure trigger_set_updated_at_timestamp();
+

crates/core/src/assets.rs

@@ -1,4 +1,7 @@
 //! ``AssetIdentifier`` utils - Parse and capture tx and cid
+
+use std::borrow::Cow;
+
 use cid::Cid;
 use url::Url;
 
@@ -7,14 +10,23 @@ use url::Url;
 pub struct ArTxid(pub [u8; 32]);
 
 /// Struct to hold tx ids
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone)]
 pub struct AssetIdentifier {
     /// ipfs cid
-    pub ipfs: Option<Cid>,
+    pub ipfs: Option<(Cid, String)>,
     /// Arweave tx id
     pub arweave: Option<ArTxid>,
 }
 
+/// An unambiguous asset-type hint
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum AssetHint {
+    /// The asset is expected to be an IPFS CID
+    Ipfs,
+    /// The asset is expected to be an Arweave transaction
+    Arweave,
+}
+
 /// Supported width sizes for asset proxy
 #[derive(Debug, Clone, Copy, strum::FromRepr)]
 #[repr(i32)]
@@ -40,21 +52,63 @@ impl From<i32> for ImageSize {
 }
 
 impl AssetIdentifier {
-    fn visit_url(url: &Url, mut f: impl FnMut(&str)) {
+    /// Attempt to parse IPFS or Arweave asset IDs from a URL.
+    ///
+    /// Parsing occurs as follows:
+    ///  - If the URL contains a CID in any segment, it is considered to be an
+    ///    IPFS URL.
+    ///  - If the URL contains a base64-encoded 256-bit digest, it is
+    ///    considered to be an Arweave transaction.
+    ///  - If both of the above are found, the URL is considered ambiguous but
+    ///    usable and both Arweave and IPFS parse results are stored.
+    ///  - If more than one valid IPFS parse result is found, the IPFS result is
+    ///    considered ambiguous and unusable and no IPFS data is returned.  The
+    ///    same holds for the Arweave parse result.
+    #[must_use]
+    pub fn new(url: &Url) -> Self {
+        let mut ipfs = Ok(None);
+        let mut arweave = Ok(None);
+
+        Self::visit_url(url, |s, i| {
+            if let Some(c) = Self::try_ipfs(s) {
+                let path = i
+                    .and_then(|i| url.path_segments().map(|s| (i, s)))
+                    .map_or_else(String::new, |(i, s)| s.skip(i).intersperse("/").collect());
+
+                Self::advance_heuristic(&mut ipfs, (c, path));
+            }
+
+            if let Some(t) = Self::try_arweave(s) {
+                Self::advance_heuristic(&mut arweave, t);
+            }
+        });
+
+        Self {
+            ipfs: ipfs.ok().flatten(),
+            arweave: arweave.ok().flatten(),
+        }
+    }
+
+    fn visit_url(url: &Url, mut f: impl FnMut(&str, Option<usize>)) {
         Some(url.scheme())
             .into_iter()
             .chain(url.domain().into_iter().flat_map(|s| s.split('.')))
             .chain(Some(url.username()))
             .chain(url.password())
-            .chain(Some(url.path()))
-            .chain(url.path_segments().into_iter().flatten())
-            .chain(url.query())
-            .chain(url.fragment().into_iter().flat_map(|s| s.split('/')))
-            .for_each(&mut f);
+            .map(|s| (s, Some(0)))
+            .chain(Some((url.path(), None)))
+            .chain(
+                url.path_segments()
+                    .into_iter()
+                    .flat_map(|s| s.into_iter().enumerate().map(|(i, s)| (s, Some(i + 1)))),
+            )
+            .chain(url.query().map(|q| (q, Some(0))))
+            .chain(url.fragment().map(|f| (f, Some(0))))
+            .for_each(|(s, i)| f(s, i));
 
         url.query_pairs().for_each(|(k, v)| {
-            f(k.as_ref());
-            f(v.as_ref());
+            f(k.as_ref(), Some(0));
+            f(v.as_ref(), Some(0));
         });
     }
 
@@ -80,31 +134,59 @@ impl AssetIdentifier {
 
     fn advance_heuristic<T>(state: &mut Result<Option<T>, ()>, value: T) {
         match state {
+            // We found a match
             Ok(None) => *state = Ok(Some(value)),
+            // We found two matches, convert to error due to ambiguity
             Ok(Some(_)) => *state = Err(()),
             Err(()) => (),
         }
     }
 
-    /// parse cid from url
+    /// Generate a binary fingerprint for this asset ID.
+    ///
+    /// For ambiguous cases, a type hint must be provided for disambiguation
+    /// otherwise no result is returned.
     #[must_use]
-    pub fn new(url: &Url) -> Self {
-        let mut ipfs = Ok(None);
-        let mut arweave = Ok(None);
+    pub fn fingerprint(&self, hint: Option<AssetHint>) -> Option<Cow<[u8]>> {
+        match (self.ipfs.as_ref(), self.arweave.as_ref(), hint) {
+            (Some((cid, path)), Some(_), Some(AssetHint::Ipfs)) | (Some((cid, path)), None, _) => {
+                Some(Cow::Owned(Self::fingerprint_ipfs(cid, path)))
+            },
+            (Some(_), Some(txid), Some(AssetHint::Arweave)) | (None, Some(txid), _) => {
+                Some(Cow::Borrowed(Self::fingerprint_arweave(txid)))
+            },
+            (Some(_), Some(_), None) | (None, None, _) => None,
+        }
+    }
 
-        Self::visit_url(url, |s| {
-            if let Some(c) = Self::try_ipfs(s) {
-                Self::advance_heuristic(&mut ipfs, c);
-            }
+    /// Return all possible fingerprints for this asset ID.
+    pub fn fingerprints(&self) -> impl Iterator<Item = Cow<[u8]>> {
+        self.ipfs
+            .iter()
+            .map(|(c, p)| Cow::Owned(Self::fingerprint_ipfs(c, p)))
+            .chain(
+                self.arweave
+                    .iter()
+                    .map(|t| Cow::Borrowed(Self::fingerprint_arweave(t))),
+            )
+    }
 
-            if let Some(t) = Self::try_arweave(s) {
-                Self::advance_heuristic(&mut arweave, t);
-            }
-        });
+    fn fingerprint_ipfs(cid: &Cid, path: &str) -> Vec<u8> {
+        if path.is_empty() {
+            use cid::multihash::StatefulHasher;
 
-        Self {
-            ipfs: ipfs.ok().flatten(),
-            arweave: arweave.ok().flatten(),
+            let mut h = cid::multihash::Sha2_256::default();
+
+            cid.write_bytes(&mut h).unwrap_or_else(|_| unreachable!());
+            h.update(path.as_bytes());
+
+            h.finalize().as_ref().to_vec()
+        } else {
+            cid.to_bytes()
         }
     }
+
+    fn fingerprint_arweave(txid: &ArTxid) -> &[u8] {
+        &txid.0
+    }
 }

crates/core/src/db/models.rs

@@ -168,8 +168,9 @@ pub struct TokenAccount<'a> {
     pub owner_address: Cow<'a, str>,
     /// The amount of the token, often 1
     pub amount: i64,
-    /// updated_at
-    pub updated_at: NaiveDateTime,
+    /// Solana slot number
+    /// The period of time for which each leader ingests transactions and produces a block.
+    pub slot: Option<i64>,
 }
 
 /// A row in the `metadatas` table

crates/core/src/db/schema.rs

@@ -513,6 +513,7 @@ table! {
         owner_address -> Varchar,
         amount -> Int8,
         updated_at -> Timestamp,
+        slot -> Nullable<Int8>,
     }
 }
 

crates/core/src/lib.rs

@@ -7,6 +7,7 @@
     missing_copy_implementations
 )]
 #![warn(clippy::pedantic, clippy::cargo, missing_docs)]
+#![feature(iter_intersperse)]
 
 // TODO: #[macro_use] is somewhat deprecated, but diesel still relies on it
 #[macro_use]
@@ -16,6 +17,7 @@ extern crate diesel_migrations;
 
 pub extern crate chrono;
 pub extern crate clap;
+pub extern crate url;
 
 pub mod assets;
 pub mod db;

crates/graphql/Cargo.toml

@@ -27,7 +27,6 @@ serde_json = "1.0.70"
 thiserror = "1.0.30"
 base64 = "0.13.0"
 md5 = "0.7.0"
-regex = "1.4.2"
 
 [dependencies.indexer-core]
 package = "metaplex-indexer-core"

crates/graphql/src/main.rs

@@ -32,9 +32,6 @@ struct Opts {
 
     #[clap(long, env)]
     asset_proxy_count: u8,
-
-    #[clap(long, env, default_value = "0")]
-    api_version: String,
 }
 
 fn graphiql(uri: String) -> impl Fn() -> HttpResponse + Clone {
@@ -47,6 +44,20 @@ fn graphiql(uri: String) -> impl Fn() -> HttpResponse + Clone {
     }
 }
 
+fn redirect_version(
+    route: &'static str,
+    version: &'static str,
+) -> impl Fn() -> HttpResponse + Clone {
+    move || {
+        HttpResponse::MovedPermanently()
+            .insert_header(("Location", version))
+            .body(format!(
+                "API route {} deprecated, please use {}",
+                route, version
+            ))
+    }
+}
+
 fn graphql(
     db_pool: Arc<Pool>,
     shared: Arc<SharedData>,
@@ -85,7 +96,6 @@ fn main() {
             twitter_bearer_token,
             asset_proxy_endpoint,
             asset_proxy_count,
-            api_version,
         } = Opts::parse();
 
         let (addr,) = server.into_parts();
@@ -103,13 +113,10 @@ fn main() {
             db::connect(db::ConnectMode::Read).context("Failed to connect to Postgres")?;
         let db_pool = Arc::new(db_pool);
 
-        let version_extension = format!(
-            "/v{}",
-            percent_encoding::utf8_percent_encode(&api_version, percent_encoding::NON_ALPHANUMERIC,)
-        );
+        let version_extension = "/v1";
 
         // Should look something like "/..."
-        let graphiql_uri = version_extension.clone();
+        let graphiql_uri = version_extension.to_owned();
         assert!(graphiql_uri.starts_with('/'));
 
         let schema = Arc::new(schema::create());
@@ -132,9 +139,12 @@ fn main() {
                                 .max_age(3600),
                         )
                         .service(
-                            web::resource(&version_extension)
+                            web::resource(version_extension)
                                 .route(web::post().to(graphql(db_pool.clone(), shared.clone()))),
                         )
+                        .service(
+                            web::resource("/v0").to(redirect_version("/v0", version_extension)),
+                        )
                         .service(
                             web::resource("/graphiql")
                                 .route(web::get().to(graphiql(graphiql_uri.clone()))),

crates/graphql/src/schema/context.rs

@@ -5,6 +5,7 @@ use objects::{
     listing::{Bid, Listing},
     listing_receipt::ListingReceipt,
     nft::{Nft, NftAttribute, NftCreator, NftOwner},
+    purchase_receipt::PurchaseReceipt,
     stats::{MarketStats, MintStats},
     store_creator::StoreCreator,
     storefront::Storefront,
@@ -30,6 +31,7 @@ pub struct AppContext {
     pub nft_owner_loader: Loader<PublicKey<Nft>, Option<NftOwner>>,
     pub storefront_loader: Loader<PublicKey<Storefront>, Option<Storefront>>,
     pub listing_receipts_loader: Loader<PublicKey<Nft>, Vec<ListingReceipt>>,
+    pub purchase_receipts_loader: Loader<PublicKey<Nft>, Vec<PurchaseReceipt>>,
     pub bid_receipts_loader: Loader<PublicKey<Nft>, Vec<BidReceipt>>,
     pub store_creator_loader: Loader<PublicKey<StoreConfig>, Vec<StoreCreator>>,
     pub collection_loader: Loader<PublicKey<StoreCreator>, Vec<Nft>>,
@@ -53,6 +55,7 @@ impl AppContext {
             nft_owner_loader: Loader::new(batcher.clone()),
             storefront_loader: Loader::new(batcher.clone()),
             listing_receipts_loader: Loader::new(batcher.clone()),
+            purchase_receipts_loader: Loader::new(batcher.clone()),
             bid_receipts_loader: Loader::new(batcher.clone()),
             store_creator_loader: Loader::new(batcher.clone()),
             collection_loader: Loader::new(batcher),

crates/graphql/src/schema/dataloaders/mod.rs

@@ -2,7 +2,6 @@ pub mod auction_house;
 pub mod bid_receipt;
 pub mod collection;
 pub mod listing;
-pub mod listing_receipt;
 pub mod nft;
 pub mod stats;
 pub mod store_creator;