From a254079b0472edcf5617eefe03fdccc613d28816 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 30 Aug 2024 10:30:27 +0100 Subject: [PATCH 01/19] Revert "removed poly references" 1e85db47f6d99d983e7198a9c55f04c11eb316cb --- config/dev.exs | 1 + config/test.exs | 1 + habitat/config/config.toml | 3 ++ lib/ret/media_resolver.ex | 45 ++++++++++++++++ lib/ret/media_search.ex | 52 +++++++++++++++++++ lib/ret/meta.ex | 1 + .../api/v1/media_search_controller.ex | 3 +- 7 files changed, 105 insertions(+), 1 deletion(-) diff --git a/config/dev.exs b/config/dev.exs index 2da1f4fc3..702575eb3 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -128,6 +128,7 @@ config :ret, Ret.MediaResolver, deviantart_client_secret: nil, imgur_mashape_api_key: nil, imgur_client_id: nil, + google_poly_api_key: nil, youtube_api_key: nil, sketchfab_api_key: nil, ytdl_host: nil, diff --git a/config/test.exs b/config/test.exs index 9c1cf1b9c..d4d2ee4bc 100644 --- a/config/test.exs +++ b/config/test.exs @@ -66,6 +66,7 @@ config :ret, Ret.MediaResolver, deviantart_client_secret: nil, imgur_mashape_api_key: nil, imgur_client_id: nil, + google_poly_api_key: nil, youtube_api_key: nil, sketchfab_api_key: nil, ytdl_host: nil, diff --git a/habitat/config/config.toml b/habitat/config/config.toml index 0b36a5f62..c1fad4c91 100644 --- a/habitat/config/config.toml +++ b/habitat/config/config.toml @@ -197,6 +197,9 @@ deviantart_client_secret = {{ toToml cfg.resolver.deviantart_client_secret }} imgur_mashape_api_key = {{ toToml cfg.resolver.imgur_mashape_api_key }} imgur_client_id = {{ toToml cfg.resolver.imgur_client_id }} {{/if}} +{{ #if cfg.resolver.google_poly_api_key }} +google_poly_api_key = {{ toToml cfg.resolver.google_poly_api_key }} +{{/if}} {{ #if cfg.resolver.youtube_api_key }} youtube_api_key = {{ toToml cfg.resolver.youtube_api_key }} {{/if}} diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 31e91f606..f7326e9e8 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -20,6 +20,7 @@ defmodule Ret.MediaResolver do @youtube_rate_limit %{scale: 8_000, limit: 1} @sketchfab_rate_limit %{scale: 60_000, limit: 15} + @poly_rate_limit %{scale: 60_000, limit: 1000} @max_await_for_rate_limit_s 120 @non_video_root_hosts [ @@ -79,6 +80,13 @@ defmodule Ret.MediaResolver do |> resolved()} end + # Necessary short circuit around google.com root_host to skip YT-DL check for Poly + def resolve(%MediaResolverQuery{url: %URI{host: "poly.google.com"}} = query, root_host) do + rate_limited_resolve(query, root_host, @poly_rate_limit, fn -> + resolve_non_video(query, root_host) + end) + end + def resolve(%MediaResolverQuery{} = query, root_host) when root_host in @non_video_root_hosts do resolve_non_video(query, root_host) end @@ -327,6 +335,43 @@ defmodule Ret.MediaResolver do {:commit, (resolved_url || uri) |> resolved(meta)} end + defp resolve_non_video( + %MediaResolverQuery{url: %URI{host: "poly.google.com", path: "/view/" <> asset_id} = uri}, + "google.com" + ) do + [uri, meta] = + with api_key when is_binary(api_key) <- module_config(:google_poly_api_key) do + Statix.increment("ret.media_resolver.poly.requests") + + payload = + "https://poly.googleapis.com/v1/assets/#{asset_id}?key=#{api_key}" + |> retry_get_until_success + |> Map.get(:body) + |> Poison.decode!() + + meta = + %{expected_content_type: "model/gltf"} + |> Map.put(:name, payload["displayName"]) + |> Map.put(:author, payload["authorName"]) + |> Map.put(:license, payload["license"]) + + formats = payload |> Map.get("formats") + + uri = + (Enum.find(formats, &(&1["formatType"] == "GLTF2")) || Enum.find(formats, &(&1["formatType"] == "GLTF"))) + |> Kernel.get_in(["root", "url"]) + |> URI.parse() + + Statix.increment("ret.media_resolver.poly.ok") + + [uri, meta] + else + _err -> [uri, nil] + end + + {:commit, uri |> resolved(meta)} + end + defp resolve_non_video( %MediaResolverQuery{url: %URI{path: "/models/" <> model_id}} = query, "sketchfab.com" = root_host diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 03a4f615c..5e4f8cf57 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -249,6 +249,46 @@ defmodule Ret.MediaSearch do sketchfab_search(query) end + def search(%Ret.MediaSearchQuery{source: "poly", cursor: cursor, filter: filter, q: q}) do + with api_key when is_binary(api_key) <- resolver_config(:google_poly_api_key) do + query = + URI.encode_query( + pageSize: @page_size, + maxComplexity: :MEDIUM, + format: :GLTF2, + pageToken: cursor, + category: filter, + keywords: q, + key: api_key + ) + + res = + "https://poly.googleapis.com/v1/assets?#{query}" + |> retry_get_until_success() + + case res do + :error -> + :error + + res -> + decoded_res = res |> Map.get(:body) |> Poison.decode!() + entries = decoded_res |> Map.get("assets") |> Enum.map(&poly_api_result_to_entry/1) + next_cursor = decoded_res |> Map.get("nextPageToken") + + {:commit, + %Ret.MediaSearchResult{ + meta: %Ret.MediaSearchResultMeta{ + next_cursor: next_cursor, + source: :poly + }, + entries: entries + }} + end + else + _ -> nil + end + end + def search(%Ret.MediaSearchQuery{source: "youtube_videos", cursor: cursor, filter: filter, q: q}) do with api_key when is_binary(api_key) <- resolver_config(:youtube_api_key) do query = @@ -377,6 +417,7 @@ defmodule Ret.MediaSearch do end end + def available?(:poly), do: has_resolver_config?(:google_poly_api_key) def available?(:bing_images), do: has_resolver_config?(:bing_search_api_key) def available?(:bing_videos), do: has_resolver_config?(:bing_search_api_key) def available?(:youtube_videos), do: has_resolver_config?(:youtube_api_key) @@ -957,6 +998,17 @@ defmodule Ret.MediaSearch do } end + defp poly_api_result_to_entry(result) do + %{ + id: result["name"], + type: "poly_model", + name: result["displayName"], + attributions: %{creator: %{name: result["authorName"]}}, + url: "https://poly.google.com/view/#{result["name"] |> String.replace("assets/", "")}", + images: %{preview: %{url: result["thumbnail"]["url"]}} + } + end + defp youtube_api_result_to_entry(result) do %{ id: result["id"]["videoId"], diff --git a/lib/ret/meta.ex b/lib/ret/meta.ex index adc310317..07b474513 100644 --- a/lib/ret/meta.ex +++ b/lib/ret/meta.ex @@ -44,6 +44,7 @@ defmodule Ret.Meta do def available_integrations_meta do %{ twitter: Ret.TwitterClient.available?(), + poly: Ret.MediaSearch.available?(:poly), bing_images: Ret.MediaSearch.available?(:bing_images), bing_videos: Ret.MediaSearch.available?(:bing_videos), youtube_videos: Ret.MediaSearch.available?(:youtube_videos), diff --git a/lib/ret_web/controllers/api/v1/media_search_controller.ex b/lib/ret_web/controllers/api/v1/media_search_controller.ex index f21ca1460..516f5d5fc 100644 --- a/lib/ret_web/controllers/api/v1/media_search_controller.ex +++ b/lib/ret_web/controllers/api/v1/media_search_controller.ex @@ -98,6 +98,7 @@ defmodule RetWeb.Api.V1.MediaSearchController do def index(conn, %{"source" => source} = params) when source in [ "sketchfab", + "poly", "tenor", "youtube_videos", "bing_videos", @@ -127,7 +128,7 @@ defmodule RetWeb.Api.V1.MediaSearchController do # For Google services, increase cache duration for landing pages by using long-lived cache, due to quotas. defp cache_for_query(%Ret.MediaSearchQuery{source: source, q: nil}) - when source == "youtube_videos", + when source == "youtube_videos" or source == "poly", do: :media_search_results_long defp cache_for_query(_query), do: :media_search_results From a435ae2cccc945615fb5dccf753dc530852c563f Mon Sep 17 00:00:00 2001 From: Craig Moore Date: Fri, 30 Aug 2024 10:47:43 +0100 Subject: [PATCH 02/19] Replacing hardcoded references to poly.googleapis.com with icosa-api.ixxy.co.uk --- lib/ret/media_resolver.ex | 2 +- lib/ret/media_search.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index f7326e9e8..bce63c9ac 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -344,7 +344,7 @@ defmodule Ret.MediaResolver do Statix.increment("ret.media_resolver.poly.requests") payload = - "https://poly.googleapis.com/v1/assets/#{asset_id}?key=#{api_key}" + "https://icosa-api.ixxy.co.uk/v1/assets/#{asset_id}?key=#{api_key}" |> retry_get_until_success |> Map.get(:body) |> Poison.decode!() diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 5e4f8cf57..bd5c2eaf2 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -263,7 +263,7 @@ defmodule Ret.MediaSearch do ) res = - "https://poly.googleapis.com/v1/assets?#{query}" + "https://icosa-api.ixxy.co.uk/v1/assets?#{query}" |> retry_get_until_success() case res do From a496cfdef11202938861844a58216077791d4399 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 30 Aug 2024 10:58:10 +0100 Subject: [PATCH 03/19] Poly to Icosa --- lib/ret/media_resolver.ex | 11 +++++++++-- lib/ret/media_search.ex | 8 ++++---- lib/ret/meta.ex | 2 +- .../controllers/api/v1/media_search_controller.ex | 4 ++-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index bce63c9ac..fddd25834 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -81,7 +81,14 @@ defmodule Ret.MediaResolver do end # Necessary short circuit around google.com root_host to skip YT-DL check for Poly - def resolve(%MediaResolverQuery{url: %URI{host: "poly.google.com"}} = query, root_host) do + def resolve(%MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk"}} = query, root_host) do + rate_limited_resolve(query, root_host, @poly_rate_limit, fn -> + resolve_non_video(query, root_host) + end) + end + + # Necessary short circuit around google.com root_host to skip YT-DL check for Poly + def resolve(%MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk"}} = query, root_host) do rate_limited_resolve(query, root_host, @poly_rate_limit, fn -> resolve_non_video(query, root_host) end) @@ -336,7 +343,7 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "poly.google.com", path: "/view/" <> asset_id} = uri}, + %MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk", path: "/view/" <> asset_id} = uri}, "google.com" ) do [uri, meta] = diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index bd5c2eaf2..8af3771d5 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -249,7 +249,7 @@ defmodule Ret.MediaSearch do sketchfab_search(query) end - def search(%Ret.MediaSearchQuery{source: "poly", cursor: cursor, filter: filter, q: q}) do + def search(%Ret.MediaSearchQuery{source: "icosa", cursor: cursor, filter: filter, q: q}) do with api_key when is_binary(api_key) <- resolver_config(:google_poly_api_key) do query = URI.encode_query( @@ -279,7 +279,7 @@ defmodule Ret.MediaSearch do %Ret.MediaSearchResult{ meta: %Ret.MediaSearchResultMeta{ next_cursor: next_cursor, - source: :poly + source: :icosa }, entries: entries }} @@ -417,7 +417,7 @@ defmodule Ret.MediaSearch do end end - def available?(:poly), do: has_resolver_config?(:google_poly_api_key) + def available?(:icosa), do: has_resolver_config?(:google_poly_api_key) def available?(:bing_images), do: has_resolver_config?(:bing_search_api_key) def available?(:bing_videos), do: has_resolver_config?(:bing_search_api_key) def available?(:youtube_videos), do: has_resolver_config?(:youtube_api_key) @@ -1004,7 +1004,7 @@ defmodule Ret.MediaSearch do type: "poly_model", name: result["displayName"], attributions: %{creator: %{name: result["authorName"]}}, - url: "https://poly.google.com/view/#{result["name"] |> String.replace("assets/", "")}", + url: "https://icosa.ixxy.co.uk/view/#{result["name"] |> String.replace("assets/", "")}", images: %{preview: %{url: result["thumbnail"]["url"]}} } end diff --git a/lib/ret/meta.ex b/lib/ret/meta.ex index 07b474513..be3df373e 100644 --- a/lib/ret/meta.ex +++ b/lib/ret/meta.ex @@ -44,7 +44,7 @@ defmodule Ret.Meta do def available_integrations_meta do %{ twitter: Ret.TwitterClient.available?(), - poly: Ret.MediaSearch.available?(:poly), + icosa: Ret.MediaSearch.available?(:icosa), bing_images: Ret.MediaSearch.available?(:bing_images), bing_videos: Ret.MediaSearch.available?(:bing_videos), youtube_videos: Ret.MediaSearch.available?(:youtube_videos), diff --git a/lib/ret_web/controllers/api/v1/media_search_controller.ex b/lib/ret_web/controllers/api/v1/media_search_controller.ex index 516f5d5fc..946eaad61 100644 --- a/lib/ret_web/controllers/api/v1/media_search_controller.ex +++ b/lib/ret_web/controllers/api/v1/media_search_controller.ex @@ -98,7 +98,7 @@ defmodule RetWeb.Api.V1.MediaSearchController do def index(conn, %{"source" => source} = params) when source in [ "sketchfab", - "poly", + "icosa", "tenor", "youtube_videos", "bing_videos", @@ -128,7 +128,7 @@ defmodule RetWeb.Api.V1.MediaSearchController do # For Google services, increase cache duration for landing pages by using long-lived cache, due to quotas. defp cache_for_query(%Ret.MediaSearchQuery{source: source, q: nil}) - when source == "youtube_videos" or source == "poly", + when source == "youtube_videos" or source == "icosa", do: :media_search_results_long defp cache_for_query(_query), do: :media_search_results From 85bb2d0b7b4257748d1bc2d6e4a5ca8adf5dfd24 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 30 Aug 2024 11:34:27 +0100 Subject: [PATCH 04/19] Update media_resolver.ex --- lib/ret/media_resolver.ex | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index fddd25834..57e14157c 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -80,13 +80,6 @@ defmodule Ret.MediaResolver do |> resolved()} end - # Necessary short circuit around google.com root_host to skip YT-DL check for Poly - def resolve(%MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk"}} = query, root_host) do - rate_limited_resolve(query, root_host, @poly_rate_limit, fn -> - resolve_non_video(query, root_host) - end) - end - # Necessary short circuit around google.com root_host to skip YT-DL check for Poly def resolve(%MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk"}} = query, root_host) do rate_limited_resolve(query, root_host, @poly_rate_limit, fn -> From aa396187774c94298d099b71fdbfef253d76164a Mon Sep 17 00:00:00 2001 From: Craig Moore Date: Fri, 6 Sep 2024 16:18:42 +0100 Subject: [PATCH 05/19] Replacing MediaSearchQuery to not check that the google_poly api key was available --- lib/ret/media_search.ex | 65 +++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 8af3771d5..040346e71 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -250,42 +250,37 @@ defmodule Ret.MediaSearch do end def search(%Ret.MediaSearchQuery{source: "icosa", cursor: cursor, filter: filter, q: q}) do - with api_key when is_binary(api_key) <- resolver_config(:google_poly_api_key) do - query = - URI.encode_query( - pageSize: @page_size, - maxComplexity: :MEDIUM, - format: :GLTF2, - pageToken: cursor, - category: filter, - keywords: q, - key: api_key - ) - - res = - "https://icosa-api.ixxy.co.uk/v1/assets?#{query}" - |> retry_get_until_success() - - case res do - :error -> - :error - - res -> - decoded_res = res |> Map.get(:body) |> Poison.decode!() - entries = decoded_res |> Map.get("assets") |> Enum.map(&poly_api_result_to_entry/1) - next_cursor = decoded_res |> Map.get("nextPageToken") + query = + URI.encode_query( + pageSize: @page_size, + maxComplexity: :MEDIUM, + format: :GLTF2, + pageToken: cursor, + category: filter, + keywords: q + ) - {:commit, - %Ret.MediaSearchResult{ - meta: %Ret.MediaSearchResultMeta{ - next_cursor: next_cursor, - source: :icosa - }, - entries: entries - }} - end - else - _ -> nil + res = + "https://icosa-api.ixxy.co.uk/v1/assets?#{query}" + |> retry_get_until_success() + + case res do + :error -> + :error + + res -> + decoded_res = res |> Map.get(:body) |> Poison.decode!() + entries = decoded_res |> Map.get("assets") |> Enum.map(&poly_api_result_to_entry/1) + next_cursor = decoded_res |> Map.get("nextPageToken") + + {:commit, + %Ret.MediaSearchResult{ + meta: %Ret.MediaSearchResultMeta{ + next_cursor: next_cursor, + source: :icosa + }, + entries: entries + }} end end From ee8ce21eb325f0adf9cc4008d33fd98b72ef0812 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 6 Sep 2024 16:23:58 +0100 Subject: [PATCH 06/19] remove api key from single asset api call --- lib/ret/media_resolver.ex | 49 ++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 57e14157c..6ed140d0b 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -336,37 +336,34 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk", path: "/view/" <> asset_id} = uri}, - "google.com" - ) do - [uri, meta] = - with api_key when is_binary(api_key) <- module_config(:google_poly_api_key) do - Statix.increment("ret.media_resolver.poly.requests") - - payload = - "https://icosa-api.ixxy.co.uk/v1/assets/#{asset_id}?key=#{api_key}" - |> retry_get_until_success - |> Map.get(:body) - |> Poison.decode!() + %MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk", path: "/view/" <> asset_id} = uri}, + "google.com" + ) do + [uri, meta] = + Statix.increment("ret.media_resolver.poly.requests") + + payload = + "https://icosa-api.ixxy.co.uk/v1/assets/#{asset_id}?" + |> retry_get_until_success + |> Map.get(:body) + |> Poison.decode!() - meta = - %{expected_content_type: "model/gltf"} - |> Map.put(:name, payload["displayName"]) - |> Map.put(:author, payload["authorName"]) - |> Map.put(:license, payload["license"]) + meta = + %{expected_content_type: "model/gltf"} + |> Map.put(:name, payload["displayName"]) + |> Map.put(:author, payload["authorName"]) + |> Map.put(:license, payload["license"]) - formats = payload |> Map.get("formats") + formats = payload |> Map.get("formats") - uri = - (Enum.find(formats, &(&1["formatType"] == "GLTF2")) || Enum.find(formats, &(&1["formatType"] == "GLTF"))) - |> Kernel.get_in(["root", "url"]) - |> URI.parse() + uri = + (Enum.find(formats, &(&1["formatType"] == "GLTF2")) || Enum.find(formats, &(&1["formatType"] == "GLTF"))) + |> Kernel.get_in(["root", "url"]) + |> URI.parse() - Statix.increment("ret.media_resolver.poly.ok") + Statix.increment("ret.media_resolver.poly.ok") - [uri, meta] - else - _err -> [uri, nil] + [uri, meta] end {:commit, uri |> resolved(meta)} From 08f741ec2c4d85dca6c5ff6bb4920cd107441437 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 13 Sep 2024 17:10:23 +0100 Subject: [PATCH 07/19] switch resolver to api instead of front end urls --- lib/ret/media_resolver.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 6ed140d0b..294015b40 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -336,8 +336,8 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk", path: "/view/" <> asset_id} = uri}, - "google.com" + %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "/assets/" <> asset_id} = uri}, + "ixxy.co.uk" ) do [uri, meta] = Statix.increment("ret.media_resolver.poly.requests") From dd7eed2a5e7b508cf0e25e459e1e78a21d331d8b Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 13 Sep 2024 17:10:41 +0100 Subject: [PATCH 08/19] only ask for GLTF2 --- lib/ret/media_resolver.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 294015b40..bf803b31e 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -357,7 +357,7 @@ defmodule Ret.MediaResolver do formats = payload |> Map.get("formats") uri = - (Enum.find(formats, &(&1["formatType"] == "GLTF2")) || Enum.find(formats, &(&1["formatType"] == "GLTF"))) + (Enum.find(formats, &(&1["formatType"] == "GLTF2"))) |> Kernel.get_in(["root", "url"]) |> URI.parse() From 8069c31a52c668bd00d123170b6efbc8faccd4b5 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 13 Sep 2024 17:30:01 +0100 Subject: [PATCH 09/19] Back to front end url but add root to non_video_root_hosts --- lib/ret/media_resolver.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index bf803b31e..2cb751bcd 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -26,7 +26,8 @@ defmodule Ret.MediaResolver do @non_video_root_hosts [ "sketchfab.com", "giphy.com", - "tenor.com" + "tenor.com", + "ixxy.co.uk" ] @deviant_id_regex ~r/\"DeviantArt:\/\/deviation\/([^"]+)/ @@ -336,7 +337,7 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "/assets/" <> asset_id} = uri}, + %MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk", path: "/views/" <> asset_id} = uri}, "ixxy.co.uk" ) do [uri, meta] = From 3cea8428d0c05233968ba7664e9d4eacfecc4d97 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 13 Sep 2024 17:46:25 +0100 Subject: [PATCH 10/19] Sigh. Back to api urls --- lib/ret/media_resolver.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 2cb751bcd..1f224a059 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -82,7 +82,7 @@ defmodule Ret.MediaResolver do end # Necessary short circuit around google.com root_host to skip YT-DL check for Poly - def resolve(%MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk"}} = query, root_host) do + def resolve(%MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk"}} = query, root_host) do rate_limited_resolve(query, root_host, @poly_rate_limit, fn -> resolve_non_video(query, root_host) end) @@ -337,7 +337,7 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk", path: "/views/" <> asset_id} = uri}, + %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "/views/" <> asset_id} = uri}, "ixxy.co.uk" ) do [uri, meta] = From 2e82b1f061bfa7b5076053de78a931e2dfeb321f Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 13 Sep 2024 17:52:17 +0100 Subject: [PATCH 11/19] Try moving all urls to be api urls --- lib/ret/media_resolver.ex | 2 +- lib/ret/media_search.ex | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 1f224a059..a0a676a7c 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -337,7 +337,7 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "/views/" <> asset_id} = uri}, + %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "v1/assets/" <> asset_id} = uri}, "ixxy.co.uk" ) do [uri, meta] = diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 040346e71..57a88ed1e 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -995,11 +995,11 @@ defmodule Ret.MediaSearch do defp poly_api_result_to_entry(result) do %{ - id: result["name"], + id: result["assetId"], type: "poly_model", name: result["displayName"], attributions: %{creator: %{name: result["authorName"]}}, - url: "https://icosa.ixxy.co.uk/view/#{result["name"] |> String.replace("assets/", "")}", + url: result["url"], images: %{preview: %{url: result["thumbnail"]["url"]}} } end From 8bae0a626d8dce26ff08c9c25f4fbd6a24876cba Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 13 Sep 2024 17:53:54 +0100 Subject: [PATCH 12/19] quick fix for ".co.uk" --- lib/ret/media_resolver.ex | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index a0a676a7c..b3d1c90aa 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -665,10 +665,19 @@ defmodule Ret.MediaResolver do nil end - defp get_root_host(host) do - # Drop subdomains - host |> String.split(".") |> Enum.slice(-2..-1) |> Enum.join(".") +defp get_root_host(host) do + if String.ends_with?(host, ".co.uk") do + host + |> String.split(".") + |> Enum.slice(-3..-1) + |> Enum.join(".") + else + host + |> String.split(".") + |> Enum.slice(-2..-1) + |> Enum.join(".") end +end defp get_imgur_headers() do with client_id when is_binary(client_id) <- module_config(:imgur_client_id), From 525a1a065312ce7085d84ee514167f1c541d0861 Mon Sep 17 00:00:00 2001 From: Craig Moore Date: Fri, 27 Sep 2024 13:45:47 +0100 Subject: [PATCH 13/19] Minor fixes for resolving icosa assets --- lib/ret/media_resolver.ex | 66 +++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index b3d1c90aa..9df633a6d 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -337,39 +337,43 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "v1/assets/" <> asset_id} = uri}, - "ixxy.co.uk" - ) do - [uri, meta] = - Statix.increment("ret.media_resolver.poly.requests") - - payload = - "https://icosa-api.ixxy.co.uk/v1/assets/#{asset_id}?" - |> retry_get_until_success - |> Map.get(:body) - |> Poison.decode!() - - meta = - %{expected_content_type: "model/gltf"} - |> Map.put(:name, payload["displayName"]) - |> Map.put(:author, payload["authorName"]) - |> Map.put(:license, payload["license"]) - - formats = payload |> Map.get("formats") - - uri = - (Enum.find(formats, &(&1["formatType"] == "GLTF2"))) - |> Kernel.get_in(["root", "url"]) - |> URI.parse() - - Statix.increment("ret.media_resolver.poly.ok") - - [uri, meta] - end - - {:commit, uri |> resolved(meta)} + %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "/v1/assets/" <> asset_id} = uri}, + "ixxy.co.uk" + ) do + # Increment stat for the request + Statix.increment("ret.media_resolver.poly.requests") + + # Make the API call to get the asset data + payload = + "https://icosa-api.ixxy.co.uk/v1/assets/#{asset_id}" + |> retry_get_until_success() # Assuming this function sends the request and handles retries + |> Map.get(:body) + |> Poison.decode!() + + # Create the meta information based on the payload + meta = + %{ + expected_content_type: "model/gltf", + name: payload["displayName"], + author: payload["authorName"], + license: payload["license"] + } + + # Extract the GLTF2 format URL from the payload + uri = + payload["formats"] + |> Enum.find(&(&1["formatType"] == "GLTF2")) + |> Kernel.get_in(["root", "url"]) + |> URI.parse() + + # Increment stat for successful resolution + Statix.increment("ret.media_resolver.poly.ok") + + # Return the URI and meta data for further processing + {:commit, resolved(uri, meta)} end + defp resolve_non_video( %MediaResolverQuery{url: %URI{path: "/models/" <> model_id}} = query, "sketchfab.com" = root_host From c8b356ebc32b8b191e37c9cce99a2dad8527394d Mon Sep 17 00:00:00 2001 From: Craig Moore Date: Fri, 27 Sep 2024 13:53:33 +0100 Subject: [PATCH 14/19] Adding temporary fix for icosa asset url --- lib/ret/media_search.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 57a88ed1e..b687f009b 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -999,7 +999,8 @@ defmodule Ret.MediaSearch do type: "poly_model", name: result["displayName"], attributions: %{creator: %{name: result["authorName"]}}, - url: result["url"], + url: "http://icosa-api.ixxy.co.uk/v1/assets/" <> result["assetId"], + # url: result["url"], images: %{preview: %{url: result["thumbnail"]["url"]}} } end From 5a1b0a72827cbec57a33c2f084babb1b4f43907c Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 25 Oct 2024 10:25:14 +0100 Subject: [PATCH 15/19] Remove references to google_poly_api_key This reverts commit 3cea8428d0c05233968ba7664e9d4eacfecc4d97. --- config/dev.exs | 1 - config/test.exs | 1 - habitat/config/config.toml | 3 --- lib/ret/media_search.ex | 2 +- 4 files changed, 1 insertion(+), 6 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index 702575eb3..2da1f4fc3 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -128,7 +128,6 @@ config :ret, Ret.MediaResolver, deviantart_client_secret: nil, imgur_mashape_api_key: nil, imgur_client_id: nil, - google_poly_api_key: nil, youtube_api_key: nil, sketchfab_api_key: nil, ytdl_host: nil, diff --git a/config/test.exs b/config/test.exs index d4d2ee4bc..9c1cf1b9c 100644 --- a/config/test.exs +++ b/config/test.exs @@ -66,7 +66,6 @@ config :ret, Ret.MediaResolver, deviantart_client_secret: nil, imgur_mashape_api_key: nil, imgur_client_id: nil, - google_poly_api_key: nil, youtube_api_key: nil, sketchfab_api_key: nil, ytdl_host: nil, diff --git a/habitat/config/config.toml b/habitat/config/config.toml index c1fad4c91..0b36a5f62 100644 --- a/habitat/config/config.toml +++ b/habitat/config/config.toml @@ -197,9 +197,6 @@ deviantart_client_secret = {{ toToml cfg.resolver.deviantart_client_secret }} imgur_mashape_api_key = {{ toToml cfg.resolver.imgur_mashape_api_key }} imgur_client_id = {{ toToml cfg.resolver.imgur_client_id }} {{/if}} -{{ #if cfg.resolver.google_poly_api_key }} -google_poly_api_key = {{ toToml cfg.resolver.google_poly_api_key }} -{{/if}} {{ #if cfg.resolver.youtube_api_key }} youtube_api_key = {{ toToml cfg.resolver.youtube_api_key }} {{/if}} diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index b687f009b..d226a1ae7 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -412,7 +412,7 @@ defmodule Ret.MediaSearch do end end - def available?(:icosa), do: has_resolver_config?(:google_poly_api_key) + def available?(:icosa), do: true # Icosa does not currently require an API key def available?(:bing_images), do: has_resolver_config?(:bing_search_api_key) def available?(:bing_videos), do: has_resolver_config?(:bing_search_api_key) def available?(:youtube_videos), do: has_resolver_config?(:youtube_api_key) From d6fe31e7991190355805639a091f2b4c487cb4c5 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 25 Oct 2024 10:40:27 +0100 Subject: [PATCH 16/19] rename another poly to icosa --- lib/ret/media_resolver.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 9df633a6d..ed244694c 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -20,7 +20,7 @@ defmodule Ret.MediaResolver do @youtube_rate_limit %{scale: 8_000, limit: 1} @sketchfab_rate_limit %{scale: 60_000, limit: 15} - @poly_rate_limit %{scale: 60_000, limit: 1000} + @icosa_rate_limit %{scale: 60_000, limit: 1000} @max_await_for_rate_limit_s 120 @non_video_root_hosts [ @@ -83,7 +83,7 @@ defmodule Ret.MediaResolver do # Necessary short circuit around google.com root_host to skip YT-DL check for Poly def resolve(%MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk"}} = query, root_host) do - rate_limited_resolve(query, root_host, @poly_rate_limit, fn -> + rate_limited_resolve(query, root_host, @icosa_rate_limit, fn -> resolve_non_video(query, root_host) end) end From 59e8127f6fdaf687945aaecc8e690095c25eabc7 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 25 Oct 2024 10:43:28 +0100 Subject: [PATCH 17/19] More poly > icosa renaming --- lib/ret/media_resolver.ex | 4 ++-- lib/ret/media_search.ex | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index ed244694c..873b93916 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -341,7 +341,7 @@ defmodule Ret.MediaResolver do "ixxy.co.uk" ) do # Increment stat for the request - Statix.increment("ret.media_resolver.poly.requests") + Statix.increment("ret.media_resolver.icosa.requests") # Make the API call to get the asset data payload = @@ -367,7 +367,7 @@ defmodule Ret.MediaResolver do |> URI.parse() # Increment stat for successful resolution - Statix.increment("ret.media_resolver.poly.ok") + Statix.increment("ret.media_resolver.icosa.ok") # Return the URI and meta data for further processing {:commit, resolved(uri, meta)} diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index d226a1ae7..3f396d7c6 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -270,7 +270,7 @@ defmodule Ret.MediaSearch do res -> decoded_res = res |> Map.get(:body) |> Poison.decode!() - entries = decoded_res |> Map.get("assets") |> Enum.map(&poly_api_result_to_entry/1) + entries = decoded_res |> Map.get("assets") |> Enum.map(&icosa_api_result_to_entry/1) next_cursor = decoded_res |> Map.get("nextPageToken") {:commit, @@ -993,10 +993,10 @@ defmodule Ret.MediaSearch do } end - defp poly_api_result_to_entry(result) do + defp icosa_api_result_to_entry(result) do %{ id: result["assetId"], - type: "poly_model", + type: "icosa_model", name: result["displayName"], attributions: %{creator: %{name: result["authorName"]}}, url: "http://icosa-api.ixxy.co.uk/v1/assets/" <> result["assetId"], From cb896bb0d334c04ce378e2f3487cbba4fec0c345 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Wed, 13 Nov 2024 10:47:15 +0000 Subject: [PATCH 18/19] Switch from staging to production urls --- lib/ret/media_resolver.ex | 10 +++++----- lib/ret/media_search.ex | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 873b93916..a6892f0d3 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -27,7 +27,7 @@ defmodule Ret.MediaResolver do "sketchfab.com", "giphy.com", "tenor.com", - "ixxy.co.uk" + "icosa.gallery" ] @deviant_id_regex ~r/\"DeviantArt:\/\/deviation\/([^"]+)/ @@ -82,7 +82,7 @@ defmodule Ret.MediaResolver do end # Necessary short circuit around google.com root_host to skip YT-DL check for Poly - def resolve(%MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk"}} = query, root_host) do + def resolve(%MediaResolverQuery{url: %URI{host: "api.icosa.gallery"}} = query, root_host) do rate_limited_resolve(query, root_host, @icosa_rate_limit, fn -> resolve_non_video(query, root_host) end) @@ -337,15 +337,15 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "/v1/assets/" <> asset_id} = uri}, - "ixxy.co.uk" + %MediaResolverQuery{url: %URI{host: "api.icosa.gallery", path: "/v1/assets/" <> asset_id} = uri}, + "icosa.gallery" ) do # Increment stat for the request Statix.increment("ret.media_resolver.icosa.requests") # Make the API call to get the asset data payload = - "https://icosa-api.ixxy.co.uk/v1/assets/#{asset_id}" + "https://api.icosa.gallery/v1/assets/#{asset_id}" |> retry_get_until_success() # Assuming this function sends the request and handles retries |> Map.get(:body) |> Poison.decode!() diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 3f396d7c6..54de43ce9 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -261,7 +261,7 @@ defmodule Ret.MediaSearch do ) res = - "https://icosa-api.ixxy.co.uk/v1/assets?#{query}" + "https://api.icosa.gallery/v1/assets?#{query}" |> retry_get_until_success() case res do @@ -999,7 +999,7 @@ defmodule Ret.MediaSearch do type: "icosa_model", name: result["displayName"], attributions: %{creator: %{name: result["authorName"]}}, - url: "http://icosa-api.ixxy.co.uk/v1/assets/" <> result["assetId"], + url: "http://api.icosa.gallery/v1/assets/" <> result["assetId"], # url: result["url"], images: %{preview: %{url: result["thumbnail"]["url"]}} } From 4343a30ae3122569de8ba35cfb31f5ac1a112b82 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Wed, 13 Nov 2024 10:47:45 +0000 Subject: [PATCH 19/19] Revert "quick fix for ".co.uk"" This reverts commit 8bae0a626d8dce26ff08c9c25f4fbd6a24876cba. --- lib/ret/media_resolver.ex | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index a6892f0d3..015cb554f 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -669,19 +669,10 @@ defmodule Ret.MediaResolver do nil end -defp get_root_host(host) do - if String.ends_with?(host, ".co.uk") do - host - |> String.split(".") - |> Enum.slice(-3..-1) - |> Enum.join(".") - else - host - |> String.split(".") - |> Enum.slice(-2..-1) - |> Enum.join(".") + defp get_root_host(host) do + # Drop subdomains + host |> String.split(".") |> Enum.slice(-2..-1) |> Enum.join(".") end -end defp get_imgur_headers() do with client_id when is_binary(client_id) <- module_config(:imgur_client_id),