mirror of
https://github.com/Zagrios/bs-manager.git
synced 2026-07-03 14:08:25 +02:00
[bugfix] implement rety strategie for ip families
This commit is contained in:
MathieuG-P
+29
-29
@@ -54,65 +54,65 @@
|
||||
"type": "diamond",
|
||||
"link": "https://www.youtube.com/@lumberjack462"
|
||||
},
|
||||
{
|
||||
{
|
||||
"username": "Xero"
|
||||
},
|
||||
{
|
||||
{
|
||||
"username": "Minescence"
|
||||
},
|
||||
{
|
||||
{
|
||||
"username": "mereknom"
|
||||
},
|
||||
{
|
||||
{
|
||||
"username": "Jascha"
|
||||
},
|
||||
{
|
||||
{
|
||||
"username": "Celldweller",
|
||||
"type": "gold"
|
||||
},
|
||||
{
|
||||
{
|
||||
"username": "rhythmshade"
|
||||
},
|
||||
{
|
||||
{
|
||||
"username": "liborsaf"
|
||||
},
|
||||
{
|
||||
{
|
||||
"username": "aatame3",
|
||||
"type": "gold"
|
||||
},
|
||||
{
|
||||
{
|
||||
"username": "Joshua"
|
||||
},
|
||||
{
|
||||
{
|
||||
"username": "Stuijvi",
|
||||
"type": "gold"
|
||||
},
|
||||
{
|
||||
"username": "_ monaka"
|
||||
},
|
||||
{
|
||||
"username": "Better_Axel"
|
||||
},
|
||||
{
|
||||
{
|
||||
"username": "_ monaka"
|
||||
},
|
||||
{
|
||||
"username": "Better_Axel"
|
||||
},
|
||||
{
|
||||
"username": "Riley",
|
||||
"type": "gold"
|
||||
},
|
||||
{
|
||||
"username": "Austin"
|
||||
},
|
||||
{
|
||||
{
|
||||
"username": "Austin"
|
||||
},
|
||||
{
|
||||
"username": "Fatalution",
|
||||
"type": "diamond",
|
||||
"link": "https://x.com/fatalution"
|
||||
},
|
||||
{
|
||||
{
|
||||
"username": "Taurus Arcade",
|
||||
"type": "gold"
|
||||
},
|
||||
{
|
||||
"username": "Mozz_Zm"
|
||||
},
|
||||
{
|
||||
"username": "clapxz"
|
||||
}
|
||||
]
|
||||
{
|
||||
"username": "Mozz_Zm"
|
||||
},
|
||||
{
|
||||
"username": "clapxz"
|
||||
}
|
||||
]
|
||||
Generated
+35
-21
@@ -29,7 +29,7 @@
|
||||
"format-duration": "^3.0.2",
|
||||
"framer-motion": "^11.2.6",
|
||||
"fs-extra": "^11.2.0",
|
||||
"got": "^14.4.3",
|
||||
"got": "^14.4.4",
|
||||
"history": "^5.3.0",
|
||||
"is-elevated": "^4.0.0",
|
||||
"jszip": "^3.10.1",
|
||||
@@ -5713,7 +5713,8 @@
|
||||
"node_modules/@sec-ant/readable-stream": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz",
|
||||
"integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg=="
|
||||
"integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@sinclair/typebox": {
|
||||
"version": "0.27.8",
|
||||
@@ -5760,6 +5761,7 @@
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz",
|
||||
"integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"defer-to-connect": "^2.0.1"
|
||||
},
|
||||
@@ -8769,6 +8771,7 @@
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz",
|
||||
"integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=14.16"
|
||||
}
|
||||
@@ -8777,6 +8780,7 @@
|
||||
"version": "12.0.1",
|
||||
"resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-12.0.1.tgz",
|
||||
"integrity": "sha512-Yo9wGIQUaAfIbk+qY0X4cDQgCosecfBe3V9NSyeY4qPC2SAkbCS4Xj79VP8WOzitpJUZKc/wsRCYF5ariDIwkg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/http-cache-semantics": "^4.0.4",
|
||||
"get-stream": "^9.0.1",
|
||||
@@ -8794,6 +8798,7 @@
|
||||
"version": "9.0.1",
|
||||
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-9.0.1.tgz",
|
||||
"integrity": "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@sec-ant/readable-stream": "^0.4.1",
|
||||
"is-stream": "^4.0.1"
|
||||
@@ -8809,6 +8814,7 @@
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz",
|
||||
"integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
@@ -8816,6 +8822,18 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/cacheable-request/node_modules/mimic-response": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz",
|
||||
"integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/call-bind": {
|
||||
"version": "1.0.5",
|
||||
"dev": true,
|
||||
@@ -9134,15 +9152,6 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/clone-response/node_modules/mimic-response": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz",
|
||||
"integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/clsx": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
|
||||
@@ -12919,6 +12928,7 @@
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-4.0.2.tgz",
|
||||
"integrity": "sha512-KQVhvhK8ZkWzxKxOr56CPulAhH3dobtuQ4+hNQ+HekH/Wp5gSOafqRAeTphQUJAIk0GBvHZgJ2ZGRWd5kphMuw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 18"
|
||||
}
|
||||
@@ -13341,9 +13351,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/got": {
|
||||
"version": "14.4.3",
|
||||
"resolved": "https://registry.npmjs.org/got/-/got-14.4.3.tgz",
|
||||
"integrity": "sha512-iTC0Z87yxSijWTh/IpvGpwOhIQK7+GgWkYrMRoN/hB9qeRj9RPuLGODwevs0p5idUf7nrxCVa5IlOmK3b8z+KA==",
|
||||
"version": "14.4.4",
|
||||
"resolved": "https://registry.npmjs.org/got/-/got-14.4.4.tgz",
|
||||
"integrity": "sha512-tqiF7eSgTBwQkxb1LxsEpva8TaMYVisbhplrFVmw9GQE3855Z+MH/mnsXLLOkDxR6hZJRFMj5VTAZ8lmTF8ZOA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@sindresorhus/is": "^7.0.1",
|
||||
@@ -13767,6 +13777,7 @@
|
||||
"version": "2.2.1",
|
||||
"resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.1.tgz",
|
||||
"integrity": "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"quick-lru": "^5.1.1",
|
||||
"resolve-alpn": "^1.2.0"
|
||||
@@ -16585,6 +16596,7 @@
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",
|
||||
"integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
|
||||
},
|
||||
@@ -16813,14 +16825,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/mimic-response": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz",
|
||||
"integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==",
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz",
|
||||
"integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/min-document": {
|
||||
@@ -17250,6 +17261,7 @@
|
||||
"version": "8.0.1",
|
||||
"resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.0.1.tgz",
|
||||
"integrity": "sha512-IO9QvjUMWxPQQhs60oOu10CRkWCiZzSUkzbXGGV9pviYl1fXYcvkzQ5jV9z8Y6un8ARoVRl4EtC6v6jNqbaJ/w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=14.16"
|
||||
},
|
||||
@@ -17555,6 +17567,7 @@
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-4.0.1.tgz",
|
||||
"integrity": "sha512-wBowNApzd45EIKdO1LaU+LrMBwAcjfPaYtVzV3lmfM3gf8Z4CHZsiIqlM8TZZ8okYvh5A1cP6gTfCRQtwUpaUg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=14.16"
|
||||
}
|
||||
@@ -19487,6 +19500,7 @@
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz",
|
||||
"integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"lowercase-keys": "^3.0.0"
|
||||
},
|
||||
|
||||
+1
-1
@@ -249,7 +249,7 @@
|
||||
"format-duration": "^3.0.2",
|
||||
"framer-motion": "^11.2.6",
|
||||
"fs-extra": "^11.2.0",
|
||||
"got": "^14.4.3",
|
||||
"got": "^14.4.4",
|
||||
"history": "^5.3.0",
|
||||
"is-elevated": "^4.0.0",
|
||||
"jszip": "^3.10.1",
|
||||
|
||||
@@ -141,6 +141,7 @@ export class SongDetailsCacheService {
|
||||
|
||||
if(error) {
|
||||
lastError = error;
|
||||
log.error("Failed to download SongDetailCache file", etag, error);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -152,7 +153,6 @@ export class SongDetailsCacheService {
|
||||
}
|
||||
}
|
||||
|
||||
log.error("Failed to download SongDetailCache file", etag, lastError);
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,19 +1,20 @@
|
||||
import { createWriteStream } from "fs";
|
||||
import { Progression } from "main/helpers/fs.helpers";
|
||||
import { Observable, shareReplay, tap } from "rxjs";
|
||||
import log from "electron-log";
|
||||
import got, { Options } from "got";
|
||||
import { IncomingHttpHeaders, IncomingMessage } from "http";
|
||||
import { unlinkSync } from "fs-extra";
|
||||
import { tryit } from "shared/helpers/error.helpers";
|
||||
import path from "path";
|
||||
import { pipeline } from "stream/promises";
|
||||
import sanitize from "sanitize-filename";
|
||||
import { LookupAddress, LookupAllOptions, LookupOptions } from "dns";
|
||||
import dns from "node:dns";
|
||||
import { createWriteStream, WriteStream } from 'fs';
|
||||
import { Progression } from 'main/helpers/fs.helpers';
|
||||
import { Observable } from 'rxjs';
|
||||
import { shareReplay, tap } from 'rxjs/operators';
|
||||
import log from 'electron-log';
|
||||
import got from 'got';
|
||||
import { IncomingHttpHeaders, IncomingMessage } from 'http';
|
||||
import { unlinkSync } from 'fs-extra';
|
||||
import { tryit } from 'shared/helpers/error.helpers';
|
||||
import path from 'path';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import sanitize from 'sanitize-filename';
|
||||
import internal from 'stream';
|
||||
|
||||
export class RequestService {
|
||||
private static instance: RequestService;
|
||||
private preferredFamily: number | undefined = undefined;
|
||||
|
||||
public static getInstance(): RequestService {
|
||||
if (!RequestService.instance) {
|
||||
@@ -24,12 +25,169 @@ export class RequestService {
|
||||
|
||||
private constructor() {}
|
||||
|
||||
private dnslookup( hostname: string, options: LookupOptions, callback: (err: NodeJS.ErrnoException | null, address: string | LookupAddress[], family?: number) => void,): void {
|
||||
console.log("OPTINOS", options);
|
||||
const checkAllOptions: LookupAllOptions = {...(options ?? {}), all: true};
|
||||
dns.lookup(hostname, checkAllOptions, (err: Error, adresses: LookupAddress[]) => {
|
||||
callback(err, adresses);
|
||||
});
|
||||
public async getJSON<T = unknown>(url: string): Promise<{ data: T; headers: IncomingHttpHeaders }> {
|
||||
|
||||
const familiesToTry = this.preferredFamily ? [this.preferredFamily, this.preferredFamily === 4 ? 6 : 4] : [4, 6];
|
||||
|
||||
for (const family of familiesToTry) {
|
||||
try {
|
||||
|
||||
// @ts-ignore (ESM is not well supported in this project, We need to move out electron-react-boilerplate, and use Vite)
|
||||
const res = await got(url, { dnsLookupIpVersion: family, responseType: 'json' });
|
||||
this.preferredFamily = family;
|
||||
return { data: res.body as T, headers: res.headers };
|
||||
} catch (err) {
|
||||
log.warn(`IPv${family} request failed, trying next one... URL: ${url}`, err);
|
||||
}
|
||||
}
|
||||
|
||||
log.error(`IPv4 and IPv6 requests failed for URL: ${url}`);
|
||||
this.preferredFamily = undefined;
|
||||
throw new Error(`IPv4 and IPv6 requests failed for URL: ${url}`);
|
||||
}
|
||||
|
||||
public downloadFile(
|
||||
url: string,
|
||||
dest: string,
|
||||
opt?: { preferContentDisposition?: boolean }
|
||||
): Observable<Progression<string>> {
|
||||
return new Observable<Progression<string>>((subscriber) => {
|
||||
const progress: Progression<string> = { current: 0, total: 0 };
|
||||
const familiesToTry = this.preferredFamily ? [this.preferredFamily, this.preferredFamily === 4 ? 6 : 4] : [4, 6];
|
||||
|
||||
let attempt = 0;
|
||||
let stream: got.GotEmitter & internal.Duplex;
|
||||
|
||||
const tryNextFamily = () => {
|
||||
if (attempt >= familiesToTry.length) {
|
||||
subscriber.error(new Error(`Download failed over IPv4 and IPv6 for URL: ${url}`));
|
||||
return;
|
||||
}
|
||||
|
||||
const family = familiesToTry[attempt++];
|
||||
let file: WriteStream | undefined;
|
||||
|
||||
// @ts-ignore (ESM is not well supported in this project, We need to move out electron-react-boilerplate, and use Vite)
|
||||
stream = got.stream(url, { dnsLookupIpVersion: family });
|
||||
|
||||
stream.on('response', (response) => {
|
||||
this.preferredFamily = family;
|
||||
|
||||
const filename = opt?.preferContentDisposition ? this.getFilenameFromContentDisposition(response.headers['content-disposition']) : null;
|
||||
|
||||
if (filename) {
|
||||
dest = path.join(path.dirname(dest), sanitize(filename));
|
||||
}
|
||||
|
||||
progress.data = dest;
|
||||
file = createWriteStream(dest);
|
||||
|
||||
pipeline(stream, file).catch(err => {
|
||||
file?.destroy();
|
||||
tryit(() => unlinkSync(dest));
|
||||
subscriber.error(err);
|
||||
});
|
||||
});
|
||||
|
||||
stream.on('downloadProgress', ({ transferred, total }) => {
|
||||
progress.current = transferred;
|
||||
progress.total = total;
|
||||
subscriber.next(progress);
|
||||
});
|
||||
|
||||
stream.on('error', err => {
|
||||
log.warn(`Download failed over IPv${family} for URL: ${url}`, err);
|
||||
stream.destroy();
|
||||
file?.destroy();
|
||||
tryNextFamily();
|
||||
});
|
||||
|
||||
stream.on('end', () => {
|
||||
file?.end();
|
||||
subscriber.next(progress);
|
||||
subscriber.complete();
|
||||
});
|
||||
};
|
||||
|
||||
tryNextFamily();
|
||||
|
||||
return () => {
|
||||
stream?.destroy();
|
||||
};
|
||||
}).pipe(
|
||||
tap({ error: (e) => log.error(e, url, dest) }),
|
||||
shareReplay(1)
|
||||
);
|
||||
}
|
||||
|
||||
public downloadBuffer(
|
||||
url: string,
|
||||
options?: got.GotOptions<null>
|
||||
): Observable<Progression<Buffer, IncomingMessage>> {
|
||||
return new Observable<Progression<Buffer, IncomingMessage>>((subscriber) => {
|
||||
const progress: Progression<Buffer, IncomingMessage> = {
|
||||
current: 0,
|
||||
total: 0,
|
||||
data: null,
|
||||
};
|
||||
|
||||
const familiesToTry = this.preferredFamily ? [this.preferredFamily, this.preferredFamily === 4 ? 6 : 4] : [4, 6];
|
||||
|
||||
let attempt = 0;
|
||||
let stream: got.GotEmitter & internal.Duplex;
|
||||
|
||||
const tryNextFamily = () => {
|
||||
if (attempt >= familiesToTry.length) {
|
||||
subscriber.error(new Error(`Download failed over IPv4 and IPv6 for URL: ${url}`));
|
||||
return;
|
||||
}
|
||||
|
||||
const family = familiesToTry[attempt++];
|
||||
|
||||
// @ts-ignore (ESM is not well supported in this project, We need to move out electron-react-boilerplate, and use Vite)
|
||||
stream = got.stream(url, { dnsLookupIpVersion: family, ...(options ?? {}) });
|
||||
|
||||
let data = Buffer.alloc(0);
|
||||
let response: IncomingMessage;
|
||||
|
||||
stream.once('response', (res) => {
|
||||
this.preferredFamily = family;
|
||||
response = res;
|
||||
});
|
||||
|
||||
stream.on('data', (chunk: Buffer) => {
|
||||
data = Buffer.concat([data, chunk]);
|
||||
});
|
||||
|
||||
stream.on('downloadProgress', ({ transferred, total }) => {
|
||||
progress.current = transferred;
|
||||
progress.total = total;
|
||||
subscriber.next(progress);
|
||||
});
|
||||
|
||||
stream.on('error', err => {
|
||||
log.warn(`Download failed over IPv${family} for URL: ${url}`, err);
|
||||
stream.destroy();
|
||||
tryNextFamily();
|
||||
});
|
||||
|
||||
stream.on('end', () => {
|
||||
progress.data = data;
|
||||
progress.extra = response;
|
||||
subscriber.next(progress);
|
||||
subscriber.complete();
|
||||
});
|
||||
};
|
||||
|
||||
tryNextFamily();
|
||||
|
||||
return () => {
|
||||
stream?.destroy();
|
||||
};
|
||||
}).pipe(
|
||||
tap({ error: (e) => log.error(e, url) }),
|
||||
shareReplay(1)
|
||||
);
|
||||
}
|
||||
|
||||
public getFilenameFromContentDisposition(disposition: string): string | undefined {
|
||||
@@ -55,109 +213,4 @@ export class RequestService {
|
||||
const partialDisposition = disposition.slice(filenameStart);
|
||||
return asciiFilenameRegex.exec(partialDisposition)?.[2];
|
||||
}
|
||||
|
||||
public async getJSON<T = unknown>(url: string): Promise<{ data: T, headers: IncomingHttpHeaders }> {
|
||||
|
||||
try{
|
||||
const res = await got(url, { dnsLookup: this.dnslookup });
|
||||
return { data: JSON.parse(res.body), headers: res.headers };
|
||||
} catch (err) {
|
||||
log.error(err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
public downloadFile(url: string, dest: string, opt?:{ preferContentDisposition?: boolean }): Observable<Progression<string>> {
|
||||
return new Observable<Progression<string>>(subscriber => {
|
||||
const progress: Progression<string> = { current: 0, total: 0 };
|
||||
|
||||
const stream = got.stream(url, { dnsLookup: this.dnslookup })
|
||||
|
||||
stream.on("response", response => {
|
||||
const filename = opt?.preferContentDisposition ? this.getFilenameFromContentDisposition(response.headers["content-disposition"]) : null;
|
||||
|
||||
if (filename) {
|
||||
dest = path.join(path.dirname(dest), sanitize(filename));
|
||||
}
|
||||
|
||||
progress.data = dest;
|
||||
|
||||
const file = createWriteStream(dest);
|
||||
|
||||
pipeline(stream, file).catch(err => {
|
||||
subscriber.error(err);
|
||||
});
|
||||
});
|
||||
|
||||
stream.on("downloadProgress", ({ transferred, total }) => {
|
||||
progress.current = transferred;
|
||||
progress.total = total;
|
||||
subscriber.next(progress);
|
||||
});
|
||||
|
||||
stream.on("error", err => {
|
||||
tryit(() => unlinkSync(dest));
|
||||
subscriber.error(err);
|
||||
});
|
||||
|
||||
stream.on("end", () => {
|
||||
subscriber.next(progress);
|
||||
subscriber.complete();
|
||||
});
|
||||
|
||||
return () => {
|
||||
stream.destroy();
|
||||
}
|
||||
|
||||
}).pipe(tap({ error: e => log.error(e, url, dest) }), shareReplay(1));
|
||||
}
|
||||
|
||||
public downloadBuffer(url: string, options?: Options & { isStream?: true }): Observable<Progression<Buffer, IncomingMessage>> {
|
||||
return new Observable<Progression<Buffer, IncomingMessage>>(subscriber => {
|
||||
const progress: Progression<Buffer, IncomingMessage> = {
|
||||
current: 0,
|
||||
total: 0,
|
||||
data: null,
|
||||
};
|
||||
|
||||
const _options = {...(options ?? {}), dnsLookup: this.dnslookup}
|
||||
|
||||
const req = got.stream(url, _options);
|
||||
|
||||
let data = Buffer.alloc(0);
|
||||
let response: IncomingMessage;
|
||||
|
||||
req.once("response", res => {
|
||||
response = res;
|
||||
});
|
||||
|
||||
req.on("data", (chunk: Buffer) => {
|
||||
data = Buffer.concat([data, chunk]);
|
||||
})
|
||||
|
||||
req.on("downloadProgress", ({ transferred, total }) => {
|
||||
progress.current = transferred;
|
||||
progress.total = total;
|
||||
subscriber.next(progress);
|
||||
});
|
||||
|
||||
req.once("error", err => {
|
||||
subscriber.error(err);
|
||||
});
|
||||
|
||||
req.once("end", () => {
|
||||
progress.data = data;
|
||||
progress.extra = response;
|
||||
subscriber.next(progress);
|
||||
subscriber.complete();
|
||||
});
|
||||
|
||||
req.resume();
|
||||
|
||||
return () => {
|
||||
req.destroy();
|
||||
}
|
||||
|
||||
}).pipe(tap({ error: log.error }), shareReplay(1))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user