diff --git a/core/src/main/java/org/nzbhydra/downloading/downloaders/Downloader.java b/core/src/main/java/org/nzbhydra/downloading/downloaders/Downloader.java index 378f563c6..a61e73be5 100644 --- a/core/src/main/java/org/nzbhydra/downloading/downloaders/Downloader.java +++ b/core/src/main/java/org/nzbhydra/downloading/downloaders/Downloader.java @@ -149,7 +149,7 @@ public AddNzbsResponse addBySearchResultIds(List s if (addingType == NzbAddingType.UPLOAD && accessTypeForIndexer == FileDownloadAccessType.PROXY) { DownloadResult result = fileHandler.getFileByResult(FileDownloadAccessType.PROXY, SearchSource.INTERNAL, optionalResult.get()); //Uploading NZBs can only be done via proxying if (result.isSuccessful()) { - String externalId = addNzb(result.getContent(), result.getTitle(), categoryToSend); + String externalId = addContent(result.getContent(), result.getTitle(), searchResult.getDownloadType(), categoryToSend); result.getDownloadEntity().setExternalId(externalId); fileHandler.updateStatusByEntity(result.getDownloadEntity(), FileDownloadStatus.NZB_ADDED); addedNzbs.add(guid); @@ -158,7 +158,7 @@ public AddNzbsResponse addBySearchResultIds(List s } } else { String link = downloadUrlBuilder.getDownloadLinkForSendingToDownloader(searchResult, false); - String externalId = addLink(link, searchResultTitle, categoryToSend); + String externalId = addLink(link, searchResultTitle, searchResult.getDownloadType(), categoryToSend); guidExternalIds.put(guid, externalId); addedNzbs.add(guid); } @@ -203,22 +203,24 @@ protected NzbAddingType getNzbAddingType(DownloadType downloadType) { public abstract List getCategories(); /** - * @param link Link to the NZB - * @param title Title to tell the downloader - * @param category Category to file under + * @param link Link to the NZB + * @param title Title to tell the downloader + * @param downloadType + * @param category Category to file under * @return ID returned by the downloader * @throws DownloaderException Error while downloading */ - public abstract String addLink(String link, String title, String category) throws DownloaderException; + public abstract String addLink(String link, String title, DownloadType downloadType, String category) throws DownloaderException; /** - * @param content NZB content to upload - * @param title Title to tell the downloader - * @param category Category to file under + * @param content NZB content to upload + * @param title Title to tell the downloader + * @param downloadType + * @param category Category to file under * @return ID returned by the downloader * @throws DownloaderException Error while downloading */ - public abstract String addNzb(byte[] content, String title, String category) throws DownloaderException; + public abstract String addContent(byte[] content, String title, DownloadType downloadType, String category) throws DownloaderException; public abstract DownloaderStatus getStatus() throws DownloaderException; diff --git a/core/src/main/java/org/nzbhydra/downloading/downloaders/nzbget/NzbGet.java b/core/src/main/java/org/nzbhydra/downloading/downloaders/nzbget/NzbGet.java index c1a8e2dfe..71145df43 100644 --- a/core/src/main/java/org/nzbhydra/downloading/downloaders/nzbget/NzbGet.java +++ b/core/src/main/java/org/nzbhydra/downloading/downloaders/nzbget/NzbGet.java @@ -21,6 +21,7 @@ import com.googlecode.jsonrpc4j.JsonRpcHttpClient; import org.nzbhydra.GenericResponse; import org.nzbhydra.config.ConfigProvider; +import org.nzbhydra.config.downloading.DownloadType; import org.nzbhydra.config.downloading.DownloaderConfig; import org.nzbhydra.downloading.FileDownloadStatus; import org.nzbhydra.downloading.FileHandler; @@ -151,7 +152,7 @@ public List getCategories() { } @Override - public String addLink(String link, String title, String category) throws DownloaderException { + public String addLink(String link, String title, DownloadType downloadType, String category) throws DownloaderException { logger.debug("Adding link for {} to NZB with category {}", title, category); try { return callAppend(link, title, category); @@ -165,7 +166,7 @@ public String addLink(String link, String title, String category) throws Downloa } @Override - public String addNzb(byte[] content, String title, String category) throws DownloaderException { + public String addContent(byte[] content, String title, DownloadType downloadType, String category) throws DownloaderException { logger.debug("Adding NZB for {} to NZB with category {}", title, category); try { return callAppend(BaseEncoding.base64().encode(content), title, category); diff --git a/core/src/main/java/org/nzbhydra/downloading/downloaders/sabnzbd/Sabnzbd.java b/core/src/main/java/org/nzbhydra/downloading/downloaders/sabnzbd/Sabnzbd.java index aa9ede9f3..eae608ed7 100644 --- a/core/src/main/java/org/nzbhydra/downloading/downloaders/sabnzbd/Sabnzbd.java +++ b/core/src/main/java/org/nzbhydra/downloading/downloaders/sabnzbd/Sabnzbd.java @@ -12,6 +12,7 @@ import org.nzbhydra.GenericResponse; import org.nzbhydra.Jackson; import org.nzbhydra.config.ConfigProvider; +import org.nzbhydra.config.downloading.DownloadType; import org.nzbhydra.downloading.DownloaderType; import org.nzbhydra.downloading.FileDownloadStatus; import org.nzbhydra.downloading.FileHandler; @@ -99,7 +100,7 @@ private UriComponentsBuilder getBaseUrl() { } @Override - public String addLink(String url, String title, String category) throws DownloaderException { + public String addLink(String url, String title, DownloadType downloadType, String category) throws DownloaderException { logger.debug("Sending link for NZB {} to sabnzbd", title); title = suffixNzbToTitle(title); UriComponentsBuilder urlBuilder = getBaseUrl(); @@ -133,7 +134,7 @@ private String sendAddNzbLinkCommand(UriComponentsBuilder urlBuilder, HttpEntity } @Override - public String addNzb(byte[] fileContent, String title, String category) throws DownloaderException { + public String addContent(byte[] fileContent, String title, DownloadType downloadType, String category) throws DownloaderException { //Using OKHTTP here because RestTemplate wouldn't work logger.debug("Uploading NZB {} to sabnzbd", title); UriComponentsBuilder urlBuilder = getBaseUrl(); diff --git a/core/src/main/java/org/nzbhydra/downloading/downloaders/torbox/Torbox.java b/core/src/main/java/org/nzbhydra/downloading/downloaders/torbox/Torbox.java index 88a29bdba..be3a4bd87 100644 --- a/core/src/main/java/org/nzbhydra/downloading/downloaders/torbox/Torbox.java +++ b/core/src/main/java/org/nzbhydra/downloading/downloaders/torbox/Torbox.java @@ -61,6 +61,14 @@ @Component("torboxdownloader") public class Torbox extends Downloader { + private enum ResultType { + TORBOX_TORRENT, + TORRENT, + MAGNET, + TORBOX_USENET, + NZB + } + // TODO sist 14.12.2024: Try and parse error responses // TODO sist 14.12.2024: Exclude nzbs.in // TODO sist 14.12.2024: Check if bypass_cache for status check is OK and which limit should be used @@ -116,23 +124,31 @@ public List getCategories() { } @Override - public String addLink(String link, String title, String category) throws DownloaderException { - return sendAddRequest(link.getBytes(StandardCharsets.UTF_8), title, "link", "link"); + public String addLink(String link, String title, DownloadType downloadType, String category) throws DownloaderException { + return sendAddRequest(link.getBytes(StandardCharsets.UTF_8), title, "link", "link", downloadType); } @Override - public String addNzb(byte[] content, String title, String category) throws DownloaderException { - return sendAddRequest(content, title, "file", "data"); + public String addContent(byte[] content, String title, DownloadType downloadType, String category) throws DownloaderException { + return sendAddRequest(content, title, "file", "data", downloadType); } - private String sendAddRequest(byte[] value, String title, String addType, String descInLog) throws DownloaderException { - log.debug("Sending {} for NZB {} to torbox", descInLog, title); - UriComponentsBuilder url = getBaseUrl().path("/usenet/createusenetdownload"); + private String sendAddRequest(byte[] value, String title, String addType, String descInLog, DownloadType downloadType) throws DownloaderException { + log.debug("Sending {} for \"{}\" to torbox", descInLog, title); + UriComponentsBuilder url; + // We have no way of knowing if the original search result was a torrent or usenet result. Kinda hacky, I know... + final ResultType resultType = determineResultType(value, title, downloadType); + + switch (resultType) { + case TORBOX_USENET, NZB -> url = getBaseUrl().path("/usenet/createusenetdownload"); + case TORBOX_TORRENT, TORRENT, MAGNET -> url = getBaseUrl().path("/torrents/createtorrent"); + default -> throw new IllegalStateException("Unexpected result type " + resultType); + } HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.MULTIPART_FORM_DATA); MultiValueMap map = new LinkedMultiValueMap<>(); - if (addType.equals("file")) { + if (addType.equals("file") && resultType != ResultType.MAGNET) { ByteArrayResource fileResource = new ByteArrayResource(value) { @Override public String getFilename() { @@ -141,7 +157,11 @@ public String getFilename() { }; map.add("file", fileResource); } else { - map.add(addType, value); + if (resultType == ResultType.MAGNET) { + map.add("magnet", value); + } else { + map.add(addType, value); + } map.add("name", title); } HttpEntity> request = new HttpEntity<>(map, headers); @@ -149,16 +169,36 @@ public String getFilename() { ResponseEntity entity = restTemplate.postForEntity(url.toUriString(), request, AddUDlResponse.class); AddUDlResponse dlResponse = entity.getBody(); if (dlResponse.isSuccess()) { + log.info("Successfully added \"{}\" to torbox", title); return dlResponse.getData().getUsenetdownload_id(); } log.error("Error adding {} for NZB {} to torbox. Error: {}\nDetail:{}", descInLog, title, dlResponse.getError(), dlResponse.getDetail()); throw new DownloaderException("Torbox returned error: " + dlResponse.getError()); } catch (Exception e) { + log.error("Unexpected response when sending add request for {} to torbox", title, e); throw new DownloaderException("Error sending " + descInLog + " to torbox", e); } } + private static ResultType determineResultType(byte[] value, String title, DownloadType downloadType) throws DownloaderException { + String valueString = new String(value); + final ResultType resultType; + + if (downloadType == DownloadType.TORRENT) { + resultType = ResultType.TORRENT; + } else if (downloadType == DownloadType.NZB) { + resultType = ResultType.NZB; + } else if (valueString.contains("usenet/download")) { + resultType = ResultType.TORBOX_USENET; + } else if (valueString.startsWith("magnet:")) { + resultType = ResultType.MAGNET; + } else { + throw new DownloaderException("Unable to determine type of download for " + title + " from type " + downloadType + " and content " + valueString); + } + return resultType; + } + @Override public DownloaderStatus getStatus() throws DownloaderException { List downloadingEntries = getLastTorboxDownloads().stream().filter(x -> x.getDownloadState().equals("downloading")).toList(); @@ -269,6 +309,7 @@ protected FileDownloadStatus getDownloadStatusFromDownloaderEntry(DownloaderEntr } + private UriComponentsBuilder getBaseUrl() { UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(BASE_API_URL); return builder; diff --git a/core/src/main/java/org/nzbhydra/indexers/Anizb.java b/core/src/main/java/org/nzbhydra/indexers/Anizb.java index 9207ae029..ec7167ecb 100644 --- a/core/src/main/java/org/nzbhydra/indexers/Anizb.java +++ b/core/src/main/java/org/nzbhydra/indexers/Anizb.java @@ -27,7 +27,7 @@ import java.util.ArrayList; import java.util.List; -@Component("aninzb") +@Component("anizb") @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) public class Anizb extends Indexer { diff --git a/core/src/main/java/org/nzbhydra/indexers/torbox/Torbox.java b/core/src/main/java/org/nzbhydra/indexers/torbox/Torbox.java index c94f6aee9..90a6bdae9 100644 --- a/core/src/main/java/org/nzbhydra/indexers/torbox/Torbox.java +++ b/core/src/main/java/org/nzbhydra/indexers/torbox/Torbox.java @@ -17,6 +17,7 @@ package org.nzbhydra.indexers.torbox; import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.NotImplementedException; @@ -31,8 +32,10 @@ import org.nzbhydra.indexers.IndexerHandlingStrategy; import org.nzbhydra.indexers.NewznabCategoryComputer; import org.nzbhydra.indexers.NfoResult; +import org.nzbhydra.indexers.QueryGenerator; import org.nzbhydra.indexers.SearchRequestIdConverter; import org.nzbhydra.indexers.exceptions.IndexerAccessException; +import org.nzbhydra.indexers.exceptions.IndexerNoIdConversionPossibleException; import org.nzbhydra.indexers.exceptions.IndexerParsingException; import org.nzbhydra.indexers.exceptions.IndexerSearchAbortedException; import org.nzbhydra.indexers.torbox.mapping.TorboxResult; @@ -44,6 +47,7 @@ import org.nzbhydra.searching.dtoseventsenums.SearchResultItem; import org.nzbhydra.searching.searchrequests.SearchRequest; import org.slf4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; import org.springframework.core.annotation.Order; @@ -55,17 +59,26 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; @Slf4j @Component("torbox") @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) public class Torbox extends Indexer { + private static final Map ID_TYPE_MAP = new HashMap<>(); public static final Set SUPPORTED_MEDIA_ID_TYPES = Set.of(MediaIdType.IMDB, MediaIdType.TVDB); + @Autowired + private SearchRequestIdConverter searchRequestIdConverter; + // TODO sist 04.01.2025: Search and add torrent results // TODO sist 04.01.2025: Show disabled download icon for direct download of torbox results // TODO sist 04.01.2025: Make sure whatever the setting for the torbox downloader is send the link to the result @@ -78,6 +91,11 @@ public class Torbox extends Indexer { } private final ExecutorService executorService = Executors.newFixedThreadPool(2); + private final QueryGenerator queryGenerator; + + public Torbox(QueryGenerator queryGenerator) { + this.queryGenerator = queryGenerator; + } @Override @@ -113,9 +131,12 @@ protected List getSearchResultItems(UsenetAndTorrentResponse s searchResultItem.setIndexerGuid(String.valueOf(result.getHash())); if (result.getNzb() != null) { searchResultItem.setLink(result.getNzb()); - } else if (result.getMagnet() != null && result.getTorrent() != null) { - searchResultItem.setSource(result.getTracker()); - searchResultItem.setLink(result.getMagnet()); + } else if (result.getMagnet() != null || result.getTorrent() != null) { + if (result.getMagnet() != null) { + searchResultItem.setLink(result.getMagnet()); + } else { + searchResultItem.setLink(result.getTorrent()); + } } else { error("Result " + result.getRawTitle() + " has neither nzb nor magnet or torrent"); continue; @@ -131,12 +152,19 @@ protected UriComponentsBuilder buildSearchUrl(SearchRequest searchRequest, Integ } @Override - protected IndexerSearchResult buildSearchUrlAndCall(SearchRequest searchRequest, int offset, Integer limit) throws IndexerSearchAbortedException, IndexerAccessException { - NzbHydra.getApplicationContext().getAutowireCapableBeanFactory().getBean(SearchRequestIdConverter.class).convertSearchIdsIfNeeded(searchRequest, config); + protected IndexerSearchResult buildSearchUrlAndCall(SearchRequest searchRequest, int offset, Integer limit) throws IndexerAccessException { + searchRequestIdConverter.convertSearchIdsIfNeeded(searchRequest, config); Stopwatch stopwatch = Stopwatch.createStarted(); - TorboxSearchResponse usenetResponse = buildAndCall(searchRequest, TorboxResultType.USENET); - UsenetAndTorrentResponse response = new UsenetAndTorrentResponse(usenetResponse, new TorboxSearchResponse()); + Future usenetFuture = executorService.submit(() -> buildAndCall(searchRequest, TorboxResultType.USENET)); + Future torrentFuture = executorService.submit(() -> buildAndCall(searchRequest, TorboxResultType.TORRENT)); + + Optional usenetResponse = getResultWithTimeout(usenetFuture); + Optional torrentResponse = getResultWithTimeout(torrentFuture); + if (usenetResponse.isEmpty() && torrentResponse.isEmpty()) { + throw new IndexerAccessException("Usenet and torrent search failed"); + } + UsenetAndTorrentResponse response = new UsenetAndTorrentResponse(usenetResponse.orElse(new TorboxSearchResponse()), torrentResponse.orElse(new TorboxSearchResponse())); return processSearchResponse(searchRequest, offset, limit, stopwatch, response); } @@ -153,9 +181,9 @@ private TorboxSearchResponse buildAndCall(SearchRequest searchRequest, TorboxRes private UriComponentsBuilder buildSearchUrl(SearchRequest searchRequest, TorboxResultType type) throws IndexerSearchAbortedException { UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl("https://search-api.torbox.app"); if (type == TorboxResultType.TORRENT) { - builder.path("/torrents/"); + builder.pathSegment("torrents"); } else { - builder.path("/usenet/"); + builder.pathSegment("usenet"); } boolean idSearch = false; for (MediaIdType idType : SUPPORTED_MEDIA_ID_TYPES) { @@ -165,15 +193,17 @@ private UriComponentsBuilder buildSearchUrl(SearchRequest searchRequest, TorboxR if (idType == MediaIdType.IMDB && !idValue.startsWith("tt")) { idValue = "tt" + idValue; } - builder.path(ID_TYPE_MAP.get(idType) + ":" + idValue); + builder.pathSegment(ID_TYPE_MAP.get(idType) + ":" + idValue); break; } } if (!idSearch) { - builder.path("/search"); - // TODO sist 04.01.2025: query generation, clean up - builder.path(searchRequest.getQuery().orElseThrow(() -> new IndexerSearchAbortedException("No query or supported IDs found"))); + builder.pathSegment("search"); + String query = queryGenerator.generateQueryIfApplicable(searchRequest, "", this); + verifyIdentifiersNotUnhandled(searchRequest, builder, query); + + builder.path(query); } builder.queryParam("metadata", "false"); @@ -181,6 +211,15 @@ private UriComponentsBuilder buildSearchUrl(SearchRequest searchRequest, TorboxR return builder; } + private void verifyIdentifiersNotUnhandled(SearchRequest searchRequest, UriComponentsBuilder componentsBuilder, String query) throws IndexerNoIdConversionPossibleException { + //Make sure we didn't for some reason neither find any usable search IDs nor generate a query + String currentUriString = componentsBuilder.toUriString(); + boolean noIdsOrIdWithNull = ID_TYPE_MAP.values().stream().noneMatch(currentUriString::contains); + if (Strings.isNullOrEmpty(query) && !searchRequest.getIdentifiers().isEmpty() && noIdsOrIdWithNull) { + throw new IndexerNoIdConversionPossibleException("Aborting searching for indexer because no usable search IDs could be found and no query was generated"); + } + } + @Override public NfoResult getNfo(String guid) { return null; @@ -196,6 +235,16 @@ protected Logger getLogger() { return log; } + private Optional getResultWithTimeout(Future future) { + try { + Integer timeoutSeconds = config.getTimeout().orElse(configProvider.getBaseConfig().getSearching().getTimeout()); + return Optional.of(future.get(timeoutSeconds, TimeUnit.SECONDS)); + } catch (ExecutionException | InterruptedException | TimeoutException e) { + log.error("Error searching torbox", e); + return Optional.empty(); + } + } + @PreDestroy public void tearDown() { executorService.shutdownNow(); diff --git a/core/src/test/java/org/nzbhydra/downloading/downloaders/torbox/TorboxTest.java b/core/src/test/java/org/nzbhydra/downloading/downloaders/torbox/TorboxTest.java index 16409b6df..b14a001ba 100644 --- a/core/src/test/java/org/nzbhydra/downloading/downloaders/torbox/TorboxTest.java +++ b/core/src/test/java/org/nzbhydra/downloading/downloaders/torbox/TorboxTest.java @@ -24,6 +24,7 @@ import org.nzbhydra.GenericResponse; import org.nzbhydra.NzbHydra; import org.nzbhydra.backup.BackupTask; +import org.nzbhydra.config.downloading.DownloadType; import org.nzbhydra.config.downloading.DownloaderConfig; import org.nzbhydra.downloading.exceptions.DownloaderException; import org.nzbhydra.webaccess.HydraOkHttp3ClientHttpRequestFactory; @@ -71,7 +72,7 @@ void shouldAddLink() throws DownloaderException { String title = "Test NZB"; String category = null; - String downloadId = torbox.addLink(nzbLink, title, category); + String downloadId = torbox.addLink(nzbLink, title, DownloadType.NZB, category); assertThat(downloadId).isNotEmpty(); } @@ -79,11 +80,11 @@ void shouldAddLink() throws DownloaderException { @Test @SneakyThrows void shouldAddData() throws DownloaderException { - byte[] data = Files.readAllBytes(Path.of("c:\\temp\\American.Dad.S20E07.480p.x264-mSD.nzb")); + byte[] data = Files.readAllBytes(Path.of("c:\\temp\\Northern.Papa.S20E07.480p.x264-mSD.nzb")); String title = "Test NZB"; String category = null; - String downloadId = torbox.addNzb(data, title, category); + String downloadId = torbox.addContent(data, title, DownloadType.NZB, category); assertThat(downloadId).isNotEmpty(); }