Skip to content

Commit

Permalink
chore: support fallback names in the download_deps util
Browse files Browse the repository at this point in the history
  • Loading branch information
@Im-Beast
Im-Beast committed Jan 3, 2025
1 parent 804f1a8 commit 7539984
Show file tree
Hide file tree
Showing 3 changed files with 109 additions and 66 deletions.
Binary file added tests/.DS_Store
View file
Binary file not shown.
171 changes: 107 additions & 64 deletions tests/download_deps.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@ import { Coin, getMoneroCTags } from "./utils.ts";
export type Target = `${typeof Deno["build"]["os"]}_${typeof Deno["build"]["arch"]}`;

interface FileInfo {
overrideMirrors?: string[];
name: string;
// List of mirrors that override DownloadInfo mirrors for this FileInfo
overrideMirrors?: [mainFilePath: string, ...fallbacks: string[]];
// List of file names that fallback if previous failed
// If first name failed and got fallbacked by another, it will get renamed to the first entry
names: string[];
sha256?: string;
}

Expand All @@ -21,7 +24,7 @@ export function getFileInfo(
downloadInfo: DownloadInfo,
target: Target = `${Deno.build.os}_${Deno.build.arch}`,
): FileInfo {
const fileInfo = "name" in downloadInfo.file ? downloadInfo.file : downloadInfo.file[target];
const fileInfo = "names" in downloadInfo.file ? downloadInfo.file : downloadInfo.file[target];
if (!fileInfo) {
throw new Error(`No fileInfo set for target: ${target}`);
}
Expand All @@ -37,6 +40,61 @@ export async function downloadDependencies(...infos: DownloadInfo[]): Promise<vo
return await downloadFiles("./tests/dependencies", `${Deno.build.os}_${Deno.build.arch}`, ...infos);
}

async function tryToDownloadFile(
mirror: string,
fileInfo: FileInfo,
fileName: string,
): Promise<Uint8Array | undefined> {
const url = `${mirror}/${fileName}`;

const response = await fetch(url);
if (!response.ok) {
console.warn(`Could not reach file ${fileName} on mirror: ${mirror}`);
await response.body?.cancel();
return;
}

const responseBuffer = await response.bytes();

if (fileInfo.sha256) {
const responseChecksum = await sha256(responseBuffer);
if (responseChecksum !== fileInfo.sha256) {
console.warn(
`Checksum mismatch on file ${fileName} on mirror: ${mirror} (${responseChecksum} != ${fileInfo.sha256})`,
);
return;
}
}

return responseBuffer;
}

async function validFileExists(filePath: string, fileInfo: FileInfo): Promise<boolean> {
const [mainFileName] = fileInfo.names;

let fileBuffer: Uint8Array;
try {
fileBuffer = await Deno.readFile(filePath);
} catch {
return false;
}

// File exists, make sure checksum matches
if (fileInfo.sha256) {
const fileChecksum = await sha256(fileBuffer);
if (fileChecksum !== fileInfo.sha256) {
console.log(
`File ${mainFileName} already exists, but checksum is mismatched (${fileChecksum} != ${fileInfo.sha256}), redownloading`,
);
await Deno.remove(filePath);
return false;
}
}

console.log(`File ${mainFileName} already exists, skipping`);
return true;
}

export async function downloadFiles(outDir: string, target: Target, ...infos: DownloadInfo[]): Promise<void> {
try {
await Deno.mkdir(outDir, { recursive: true });
Expand All @@ -48,65 +106,40 @@ export async function downloadFiles(outDir: string, target: Target, ...infos: Do

for (const info of infos) {
const fileInfo = getFileInfo(info, target);
const fileName = fileInfo.name;
const filePath = join(outDir, info.outDir ?? "", fileName);

file_might_exist: try {
const fileBuffer = await Deno.readFile(filePath);

// File exists, make sure checksum matches
if (fileInfo.sha256) {
const fileChecksum = await sha256(fileBuffer);
if (fileChecksum !== fileInfo.sha256) {
console.log(
`File ${fileName} already exists, but checksum is mismatched (${fileChecksum} != ${fileInfo.sha256}), redownloading`,
);
await Deno.remove(filePath);
break file_might_exist;
}
}

console.log(`File ${fileName} already exists, skipping`);
const [mainFileName] = fileInfo.names;

if (
await validFileExists(
join(outDir, info.outDir || ""),
fileInfo,
)
) {
continue;
} catch { /**/ }
}

let buffer: Uint8Array | undefined;
outer: for (const mirror of fileInfo.overrideMirrors ?? info.mirrors) {
for (const fileName of fileInfo.names) {
buffer = await tryToDownloadFile(mirror, fileInfo, fileName);

for (const mirror of fileInfo.overrideMirrors ?? info.mirrors) {
const url = `${mirror}/${fileName}`;

const response = await fetch(url);
if (!response.ok) {
console.warn(`Could not reach file ${fileName} on mirror: ${mirror}`);
await response.body?.cancel();
continue;
}

const responseBuffer = await response.bytes();

if (fileInfo.sha256) {
const responseChecksum = await sha256(responseBuffer);
if (responseChecksum !== fileInfo.sha256) {
console.warn(
`Checksum mismatch on file ${fileName} on mirror: ${mirror} (${responseChecksum} != ${fileInfo.sha256})`,
);
continue;
if (buffer) {
break outer;
}
}

buffer = responseBuffer;
}

if (!buffer) {
throw new Error(`None of the mirrors for ${fileName} are available`);
throw new Error(`None of the mirrors for ${fileInfo.names} are available`);
}

const filePath = join(outDir, info.outDir ?? "", mainFileName);

await Deno.mkdir(resolve(filePath, ".."), {
recursive: true,
}).catch(() => {});

await Deno.writeFile(filePath, buffer);
console.info("Downloaded file", fileInfo.name);
console.info("Downloaded file", filePath);
}
}

Expand All @@ -117,25 +150,25 @@ export const wowneroCliInfo: DownloadInfo = {
],
file: {
linux_aarch64: {
name: "wownero-aarch64-linux-gnu-59db3fe8d.tar.bz2",
names: ["wownero-aarch64-linux-gnu-59db3fe8d.tar.bz2"],
sha256: "07ce678302c07a6e79d90be65cbda243d843d414fbadb30f972d6c226575cfa7",
},
linux_x86_64: {
name: "wownero-x86_64-linux-gnu-59db3fe8d.tar.bz2",
names: ["wownero-x86_64-linux-gnu-59db3fe8d.tar.bz2"],
sha256: "03880967c70cc86558d962b8a281868c3934238ea457a36174ba72b99d70107e",
},

darwin_aarch64: {
name: "wownero-aarch64-apple-darwin11-59db3fe8d.tar.bz2",
names: ["wownero-aarch64-apple-darwin11-59db3fe8d.tar.bz2"],
sha256: "25ff454a92b1cf036df5f28cdd2c63dcaf4b03da7da9403087371f868827c957",
},
darwin_x86_64: {
name: "wownero-x86_64-apple-darwin11-59db3fe8d.tar.bz2",
names: ["wownero-x86_64-apple-darwin11-59db3fe8d.tar.bz2"],
sha256: "7e9b6a84a560ed7a9ed7117c6f07fb228d77a06afac863d0ea1dbf833c4eddf6",
},

windows_x86_64: {
name: "wownero-x86_64-w64-mingw32-59db3fe8d.zip",
names: ["wownero-x86_64-w64-mingw32-59db3fe8d.zip"],
sha256: "7e0ed84afa51e3b403d635c706042859094eb6850de21c9e82cb0a104425510e",
},

Expand All @@ -144,7 +177,7 @@ export const wowneroCliInfo: DownloadInfo = {
"https://static.mrcyjanek.net/download_mirror/",
"https://codeberg.org/wownero/wownero/releases/download/v0.11.1.0/",
],
name: "wownero-aarch64-linux-android-v0.11.1.0.tar.bz2",
names: ["wownero-aarch64-linux-android-v0.11.1.0.tar.bz2"],
sha256: "236188f8d8e7fad2ff35973f8c2417afffa8855d1a57b4c682fff5b199ea40f5",
},
},
Expand All @@ -157,30 +190,30 @@ export const moneroCliInfo: DownloadInfo = {
],
file: {
linux_aarch64: {
name: "monero-linux-armv8-v0.18.3.4.tar.bz2",
names: ["monero-linux-armv8-v0.18.3.4.tar.bz2"],
sha256: "33ca2f0055529d225b61314c56370e35606b40edad61c91c859f873ed67a1ea7",
},
linux_x86_64: {
name: "monero-linux-x64-v0.18.3.4.tar.bz2",
names: ["monero-linux-x64-v0.18.3.4.tar.bz2"],
sha256: "51ba03928d189c1c11b5379cab17dd9ae8d2230056dc05c872d0f8dba4a87f1d",
},

darwin_aarch64: {
name: "monero-mac-armv8-v0.18.3.4.tar.bz2",
names: ["monero-mac-armv8-v0.18.3.4.tar.bz2"],
sha256: "44520cb3a05c2518ca9aeae1b2e3080fe2bba1e3596d014ceff1090dfcba8ab4",
},
darwin_x86_64: {
name: "monero-mac-x64-v0.18.3.4.tar.bz2",
names: ["monero-mac-x64-v0.18.3.4.tar.bz2"],
sha256: "32c449f562216d3d83154e708471236d07db7477d6b67f1936a0a85a5005f2b8",
},

windows_x86_64: {
name: "monero-win-x64-v0.18.3.4.zip",
names: ["monero-win-x64-v0.18.3.4.zip"],
sha256: "54a66db6c892b2a0999754841f4ca68511741b88ea3ab20c7cd504a027f465f5",
},

android_aarch64: {
name: "monero-android-armv8-v0.18.3.4.tar.bz2",
names: ["monero-android-armv8-v0.18.3.4.tar.bz2"],
sha256: "d9c9249d1408822ce36b346c6b9fb6b896cda16714d62117fb1c588a5201763c",
},
},
Expand All @@ -199,12 +232,22 @@ for (const tag of await getMoneroCTags()) {
`https://github.com/MrCyjaneK/monero_c/releases/download/${tag}/`,
],
file: {
linux_aarch64: { name: `${coin}_aarch64-linux-gnu_libwallet2_api_c.so.xz` },
linux_x86_64: { name: `${coin}_x86_64-linux-gnu_libwallet2_api_c.so.xz` },
darwin_aarch64: { name: `${coin}_aarch64-apple-darwin11_libwallet2_api_c.dylib.xz` },
darwin_x86_64: { name: `${coin}_x86_64-apple-darwin11_libwallet2_api_c.dylib.xz` },
windows_x86_64: { name: `${coin}_x86_64-w64-mingw32_libwallet2_api_c.dll.xz` },
android_aarch64: { name: `${coin}_aarch64-linux-android_libwallet2_api_c.so.xz` },
linux_aarch64: { names: [`${coin}_aarch64-linux-gnu_libwallet2_api_c.so.xz`] },
linux_x86_64: { names: [`${coin}_x86_64-linux-gnu_libwallet2_api_c.so.xz`] },
darwin_aarch64: {
names: [
`${coin}_aarch64-apple-darwin11_libwallet2_api_c.dylib.xz`,
`${coin}_aarch64-apple-darwin_libwallet2_api_c.dylib.xz`,
],
},
darwin_x86_64: {
names: [
`${coin}_x86_64-apple-darwin11_libwallet2_api_c.dylib.xz`,
`${coin}_x86_64-apple-darwin_libwallet2_api_c.dylib.xz`,
],
},
windows_x86_64: { names: [`${coin}_x86_64-w64-mingw32_libwallet2_api_c.dll.xz`] },
android_aarch64: { names: [`${coin}_aarch64-linux-android_libwallet2_api_c.so.xz`] },
},
outDir: `libs/${tag}`,
});
Expand Down
4 changes: 2 additions & 2 deletions tests/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,13 +84,13 @@ export async function extract(path: string, out: string) {

export async function prepareMoneroCli() {
await downloadDependencies(moneroCliInfo);
const path = join("./tests/dependencies", moneroCliInfo.outDir ?? "", getFileInfo(moneroCliInfo).name);
const path = join("./tests/dependencies", moneroCliInfo.outDir ?? "", getFileInfo(moneroCliInfo).names[0]);
await extract(path, "./tests/dependencies/monero-cli/");
}

export async function prepareWowneroCli() {
await downloadDependencies(wowneroCliInfo);
const path = join("./tests/dependencies", wowneroCliInfo.outDir ?? "", getFileInfo(wowneroCliInfo).name);
const path = join("./tests/dependencies", wowneroCliInfo.outDir ?? "", getFileInfo(wowneroCliInfo).names[0]);
await extract(path, "./tests/dependencies/wownero-cli/");
}

Expand Down

0 comments on commit 7539984

Please sign in to comment.