Merge pull request #649 from silentrald/bugfix/non-standard-zip-extraction

[bugfix] handle extracting non-conformant zip files

(cherry picked from commit f7f5f768fc)
This commit is contained in:
MathieuG-P
2024-11-25 20:30:29 +01:00
parent 6dde2fa5d0
commit 6f332f66bc
23 changed files with 4392 additions and 4848 deletions
+13
View File
@@ -52,6 +52,19 @@ module.exports = {
Function: false,
}
}]
"import/extensions": "off",
"lines-between-class-members": "off",
"no-throw-literal": "warn",
"no-use-before-define": "off",
"no-useless-constructor": "off",
"no-unused-vars": "off",
"@typescript-eslint/no-unused-vars": "error",
'react/jsx-filename-extension': [2, { 'extensions': ['.js', '.jsx', '.ts', '.tsx'] }],
"no-shadow": "off",
"react/function-component-definition": "off",
"jsx-a11y/control-has-associated-label": "off",
"react/button-has-type": "off",
"max-classes-per-file": "off",
},
parserOptions: {
ecmaVersion: 2020,
+2
View File
@@ -15,3 +15,5 @@
*.ttf binary
*.woff binary
*.woff2 binary
assets/scripts/* binary
*.zip binary
+5 -4
View File
@@ -13,7 +13,7 @@ jobs:
build:
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
os: [ubuntu-latest, windows-latest]
runs-on: ${{ matrix.os }}
@@ -26,6 +26,7 @@ jobs:
cache: "npm"
- run: npm ci
- run: npm run build
- run: npm test
# Currently typescript, eslint, and prettier are unhappy
continue-on-error: true
- name: Unit tests
run: npm run test:unit
+17
View File
@@ -785,6 +785,23 @@
"ReleaseURL": "https://steamcommunity.com/games/620980/announcements/detail/4669760442595134838",
"ReleaseImg": "https://clan.akamai.steamstatic.com/images/32055887/57cfa043407c091e67b7e0a8ae86fe5590deae33.png",
"ReleaseDate": "1728399815",
"year": "2024"
},
{
"BSVersion": "1.39.0",
"BSManifest": "5365094543246905054",
"OculusBinaryId": "7978663688900007",
"ReleaseURL": "https://steamcommunity.com/games/620980/announcements/detail/4451339035198095638",
"ReleaseImg": "https://clan.cloudflare.steamstatic.com/images/32055887/fd8e2c783e46decdf5e10fed1e16f124616d9321.png",
"ReleaseDate": "1731426932",
"year": "2024"
},
{
"BSVersion": "1.39.1",
"BSManifest": "2179614296547473280",
"OculusBinaryId": "8189133517853022",
"ReleaseURL": "https://steamcommunity.com/games/620980/announcements/detail/4451340304252928024",
"ReleaseDate": "1732107748",
"year": "2024",
"recommended": true
}
+19
View File
@@ -0,0 +1,19 @@
import type { Config } from "jest";
const config: Config = {
// testURL: "http://localhost/",
// testEnvironment: "jsdom",
transform: {
"\\.(ts|tsx|js|jsx)$": "ts-jest",
},
moduleNameMapper: {
"\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "<rootDir>/.erb/mocks/fileMock.js",
"\\.(css|less|sass|scss)$": "identity-obj-proxy",
},
moduleFileExtensions: ["js", "jsx", "ts", "tsx", "json"],
moduleDirectories: ["node_modules", "src"],
testPathIgnorePatterns: ["release/app/dist"],
setupFiles: ["./.erb/scripts/check-build-exists.ts"],
};
export default config;
+3705 -4715
View File
File diff suppressed because it is too large Load Diff
+10 -38
View File
@@ -14,7 +14,8 @@
"start:preload": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.preload.dev.ts",
"start:renderer": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack serve --config ./.erb/configs/webpack.config.renderer.dev.ts",
"test": "jest",
"publish": "npm run build && electron-builder -c.win.certificateSha1=206941d969c4fa8a0e04d9427def361e13b02fd0 --publish always --win --x64"
"test:unit": "jest ./src/__tests__/unit",
"publish": "npm run build && electron-builder -c.win.certificateSha1=2164d6a7d641ecf6ad57852f665a518ca2bf960f --publish always --win --x64"
},
"lint-staged": {
"*.{js,jsx,ts,tsx}": [
@@ -122,45 +123,17 @@
"beat-saber"
],
"homepage": "https://github.com/Zagrios/bs-manager#readme",
"jest": {
"testURL": "http://localhost/",
"testEnvironment": "jsdom",
"transform": {
"\\.(ts|tsx|js|jsx)$": "ts-jest"
},
"moduleNameMapper": {
"\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "<rootDir>/.erb/mocks/fileMock.js",
"\\.(css|less|sass|scss)$": "identity-obj-proxy"
},
"moduleFileExtensions": [
"js",
"jsx",
"ts",
"tsx",
"json"
],
"moduleDirectories": [
"node_modules",
"src"
],
"testPathIgnorePatterns": [
"release/app/dist"
],
"setupFiles": [
"./.erb/scripts/check-build-exists.ts"
]
},
"devDependencies": {
"@electron/rebuild": "^3.2.13",
"@pmmmwh/react-refresh-webpack-plugin": "0.5.5",
"@teamsupercell/typings-for-css-modules-loader": "^2.5.1",
"@testing-library/jest-dom": "^5.16.5",
"@testing-library/jest-dom": "^6.4.1",
"@testing-library/react": "^13.3.0",
"@types/archiver": "^5.3.1",
"@types/color": "^3.0.3",
"@types/crypto-js": "^4.2.1",
"@types/dateformat": "^5.0.0",
"@types/jest": "^27.5.2",
"@types/jest": "^29.5.11",
"@types/node": "^20",
"@types/node-fetch": "^2.6.3",
"@types/pako": "^2.0.1",
@@ -199,7 +172,7 @@
"eslint-import-resolver-webpack": "^0.13.2",
"eslint-plugin-compat": "^4.0.2",
"eslint-plugin-import": "^2.25.4",
"eslint-plugin-jest": "^26.8.7",
"eslint-plugin-jest": "^27.6.3",
"eslint-plugin-jsx-a11y": "^6.6.1",
"eslint-plugin-promise": "^6.0.0",
"eslint-plugin-react": "^7.30.0",
@@ -207,7 +180,7 @@
"file-loader": "^6.2.0",
"html-webpack-plugin": "^5.5.0",
"identity-obj-proxy": "^3.0.0",
"jest": "^27.5.1",
"jest": "^29.7.0",
"lint-staged": "^12.5.0",
"mini-css-extract-plugin": "^2.6.1",
"opencollective-postinstall": "^2.0.3",
@@ -225,7 +198,7 @@
"tailwind-scrollbar": "^2.0.1",
"tailwindcss": "^3.3.1",
"terser-webpack-plugin": "^5.3.5",
"ts-jest": "^27.1.5",
"ts-jest": "^29.1.2",
"ts-loader": "^9.3.0",
"ts-node": "^10.8.2",
"typescript": "^4.7.4",
@@ -254,11 +227,9 @@
"fs-extra": "^11.2.0",
"history": "^5.3.0",
"is-elevated": "^3.0.0",
"jszip": "^3.10.1",
"md5-file": "^5.0.0",
"node-abi": "^3.47.0",
"node-abi": "^3.65.0",
"node-fetch": "^2.6.7",
"node-stream-zip": "^1.15.0",
"pako": "^2.1.0",
"ps-list": "^7.2.0",
"qrcode.react": "^3.1.0",
@@ -281,7 +252,8 @@
"tippy.js": "^6.3.7",
"to-ico": "^1.1.5",
"use-double-click": "^1.0.5",
"use-fit-text": "^2.4.0"
"use-fit-text": "^2.4.0",
"yauzl": "^3.2.0"
},
"devEngines": {
"node": ">=14.x",
-9
View File
@@ -1,9 +0,0 @@
import "@testing-library/jest-dom";
import { render } from "@testing-library/react";
import App from "../renderer/windows/App";
describe("App", () => {
it("should render", () => {
expect(render(<App />)).toBeTruthy();
});
});
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+329
View File
@@ -0,0 +1,329 @@
import path from "path";
import { mkdir, pathExistsSync, readFile, rm } from "fs-extra";
import { BsmZipExtractor } from "main/models/bsm-zip-extractor.class";
const TEST_FOLDER = path.resolve(__dirname, "..", "assets", "zip");
const STANDARD_ZIP = path.join(TEST_FOLDER, "standard.zip");
const WINDOWS_LEGACY_MAP_ZIP = path.join(TEST_FOLDER, "windows_legacy.zip");
const SUBFOLDERS_ZIP = path.join(TEST_FOLDER, "subfolders.zip");
const MANIFEST_ZIP = path.join(TEST_FOLDER, "manifest.zip");
const EMPTY_ZIP = path.join(TEST_FOLDER, "empty.zip");
const UNICODE_ZIP = path.join(TEST_FOLDER, "unicode.zip");
const DESTINATION_FOLDER = path.join(TEST_FOLDER, "out");
describe("Test BsmZipExtractor class", () => {
let zip: BsmZipExtractor;
beforeAll(async () => {
if (pathExistsSync(DESTINATION_FOLDER)) {
await rm(DESTINATION_FOLDER, { recursive: true, force: true });
}
await mkdir(DESTINATION_FOLDER);
});
afterEach(async () => {
if (zip) {
zip.close();
}
if (pathExistsSync(DESTINATION_FOLDER)) {
await rm(DESTINATION_FOLDER, { recursive: true, force: true });
}
await mkdir(DESTINATION_FOLDER);
});
afterAll(async () => {
if (pathExistsSync(DESTINATION_FOLDER)) {
await rm(DESTINATION_FOLDER, { recursive: true, force: true });
}
});
it("Extract standard zip", async () => {
zip = await BsmZipExtractor.fromPath(STANDARD_ZIP);
const res = await zip.extract(DESTINATION_FOLDER);
expect(res.sort()).toEqual([
"file_1.2.txt",
"file_1.3.txt",
"folder_1.1/",
"folder_1.1/file_2.1.txt",
"folder_1.1/file_2.2.txt",
].sort())
for (const file of [
"file_1.2.txt",
"file_1.3.txt",
]) {
expect(pathExistsSync(path.join(DESTINATION_FOLDER, file)))
.toBe(true);
}
const SUBFOLDER_PATH = path.join(DESTINATION_FOLDER, "folder_1.1");
for (const file of [
"file_2.1.txt",
"file_2.2.txt",
]) {
expect(pathExistsSync(path.join(SUBFOLDER_PATH, file)))
.toBe(true);
}
});
it("Extract map zips using back slashes", async () => {
zip = await BsmZipExtractor.fromPath(WINDOWS_LEGACY_MAP_ZIP);
const res = await zip.extract(DESTINATION_FOLDER);
expect(res.sort()).toEqual([
"BPMInfo.dat",
"cover.jpg",
"ExpertLegacy.dat",
"ExpertPlusLawless.dat",
"ExpertPlusLegacy.dat",
"ExpertPlusStandard.dat",
"ExpertStandard.dat",
"Info.dat",
"song.egg",
"pfp/",
"pfp/galaxyCompressed.jpg",
"pfp/gojiCompressed.jpg",
].sort());
// Expect all the files to exists
for (const file of [
"BPMInfo.dat",
"cover.jpg",
"ExpertLegacy.dat",
"ExpertPlusLawless.dat",
"ExpertPlusLegacy.dat",
"ExpertPlusStandard.dat",
"ExpertStandard.dat",
"Info.dat",
"song.egg",
]) {
expect(pathExistsSync(path.join(DESTINATION_FOLDER, file)))
.toBe(true);
}
const SUBFOLDER_PATH = path.join(DESTINATION_FOLDER, "pfp");
for (const file of [
"galaxyCompressed.jpg",
"gojiCompressed.jpg",
]) {
expect(pathExistsSync(path.join(SUBFOLDER_PATH, file)))
.toBe(true);
}
});
it("Read Zip with multiple subfolders", async () => {
const zip = await BsmZipExtractor.fromPath(SUBFOLDERS_ZIP);
const res = await zip.extract(DESTINATION_FOLDER);
expect(res.sort()).toEqual([
"1/",
"1/2/",
"1/2/3/",
"1/2/3/4/",
"1/2/3/4/5.txt",
].sort());
for (const folder of [
"1/",
"1/2/",
"1/2/3/",
"1/2/3/4/",
]) {
expect(pathExistsSync(path.join(DESTINATION_FOLDER, folder)))
.toBe(true);
}
const file = "1/2/3/4/5.txt";
expect(pathExistsSync(path.join(DESTINATION_FOLDER, file)))
.toBe(true);
});
it("Read manifest.json from zip file", async () => {
const zipBuffer = await readFile(MANIFEST_ZIP);
zip = await BsmZipExtractor.fromBuffer(zipBuffer);
const buffer = await (await zip.findEntry(entry => entry.fileName === "manifest.json")).read();
const manifest = JSON.parse(buffer.toString());
expect(manifest).toBeTruthy();
// Check some of the fields if they are correct
expect(manifest.appId).toBe("some-app-id");
expect(manifest.canonicalName).toBe("some-canonical-name");
expect(manifest.isCore).toBe(true);
});
it("Should find an entry using findEntry", async () => {
zip = await BsmZipExtractor.fromPath(STANDARD_ZIP);
const entry = await zip.findEntry(entry => entry.fileName === "file_1.2.txt");
expect(entry).toBeTruthy();
expect(entry.fileName).toBe("file_1.2.txt");
const contentBuffer = await entry.read();
const content = contentBuffer.toString();
expect(content).toContain("This is file 1.2"); // Assurez-vous que le contenu correspond
});
it("Should filter entries using filterEntries", async () => {
zip = await BsmZipExtractor.fromPath(STANDARD_ZIP);
const entries = await zip.filterEntries(entry => !entry.isDirectory && entry.fileName.startsWith("folder_1.1/"));
expect(entries.length).toBe(2);
const fileNames = entries.map(entry => entry.fileName).sort();
expect(fileNames).toEqual([
"folder_1.1/file_2.1.txt",
"folder_1.1/file_2.2.txt",
]);
});
it("Should get an entry by file name using getEntry", async () => {
zip = await BsmZipExtractor.fromPath(STANDARD_ZIP);
const entry = await zip.getEntry("file_1.3.txt");
expect(entry).toBeTruthy();
expect(entry.fileName).toBe("file_1.3.txt");
const contentBuffer = await entry.read();
const content = contentBuffer.toString();
expect(content).toContain("This is file 1.3"); // Assurez-vous que le contenu correspond
});
it("Should extract only specified entries using entriesNames option", async () => {
zip = await BsmZipExtractor.fromPath(STANDARD_ZIP);
const res = await zip.extract(DESTINATION_FOLDER, { entriesNames: ["file_1.2.txt", "folder_1.1/file_2.1.txt"] });
expect(res.sort()).toEqual([
"file_1.2.txt",
"folder_1.1/",
"folder_1.1/file_2.1.txt",
].sort());
for (const file of [
"file_1.2.txt",
"folder_1.1/file_2.1.txt",
]) {
expect(pathExistsSync(path.join(DESTINATION_FOLDER, file)))
.toBe(true);
}
// Vérifiez que les autres fichiers ne sont pas extraits
expect(pathExistsSync(path.join(DESTINATION_FOLDER, "file_1.3.txt")))
.toBe(false);
expect(pathExistsSync(path.join(DESTINATION_FOLDER, "folder_1.1/file_2.2.txt")))
.toBe(false);
});
it("Should not extract any files if abortToken is already aborted", async () => {
zip = await BsmZipExtractor.fromPath(STANDARD_ZIP);
const abortController = new AbortController();
// Abortez avant d'appeler extract
abortController.abort();
const res = await zip.extract(DESTINATION_FOLDER, { abortToken: abortController });
expect(res.length).toBe(0);
// Vérifiez qu'aucun fichier n'a été extrait
expect(pathExistsSync(path.join(DESTINATION_FOLDER, "file_1.2.txt")))
.toBe(false);
});
it("Should throw an error when opening a non-existent zip file", async () => {
await expect(BsmZipExtractor.fromPath("non_existent.zip"))
.rejects
.toThrow();
});
it("Should handle empty zip file correctly", async () => {
zip = await BsmZipExtractor.fromPath(EMPTY_ZIP);
const entries = [];
for await (const entry of zip.entries()) {
entries.push(entry);
}
expect(entries.length).toBe(0);
const res = await zip.extract(DESTINATION_FOLDER);
expect(res.length).toBe(0);
});
it("Should return the same entries on multiple calls to entries()", async () => {
zip = await BsmZipExtractor.fromPath(STANDARD_ZIP);
const entries1 = [];
for await (const entry of zip.entries()) {
entries1.push(entry);
}
expect(entries1.length).toBe(5); // Basé sur le contenu de STANDARD_ZIP
const entries2 = [];
for await (const entry of zip.entries()) {
entries2.push(entry);
}
expect(entries2.length).toBe(5);
expect(entries2.map(e => e.fileName).sort()).toEqual(entries1.map(e => e.fileName).sort());
});
it("Should not be able to read entries after zip is closed", async () => {
zip = await BsmZipExtractor.fromPath(STANDARD_ZIP);
zip.close();
await expect(zip.entries().next()).rejects.toThrow();
});
it("Should handle zip file from buffer", async () => {
const zipBuffer = await readFile(STANDARD_ZIP);
zip = await BsmZipExtractor.fromBuffer(zipBuffer);
const entries = [];
for await (const entry of zip.entries()) {
entries.push(entry.fileName);
}
expect(entries.sort()).toEqual([
"file_1.2.txt",
"file_1.3.txt",
"folder_1.1/",
"folder_1.1/file_2.1.txt",
"folder_1.1/file_2.2.txt",
].sort());
});
it("Should read and extract files with Unicode filenames", async () => {
zip = await BsmZipExtractor.fromPath(UNICODE_ZIP);
const entries = [];
for await (const entry of zip.entries()) {
entries.push(entry);
}
expect(entries.map(e => e.fileName)).toContain("こんにちは.txt"); // "Bonjour" en japonais
const res = await zip.extract(DESTINATION_FOLDER);
expect(res).toContain("こんにちは.txt");
expect(pathExistsSync(path.join(DESTINATION_FOLDER, "こんにちは.txt")))
.toBe(true);
});
it("Should throw an error when reading a corrupted zip file", async () => {
const CORRUPTED_ZIP = path.join(TEST_FOLDER, "corrupted.zip");
await expect(BsmZipExtractor.fromPath(CORRUPTED_ZIP)).rejects.toThrow(/.*[Error: Invalid comment length].*/)
});
});
-28
View File
@@ -1,28 +0,0 @@
import JSZip from "jszip";
import { pathExist } from "./fs.helpers";
import path from "path";
import { mkdir, writeFile } from "fs/promises";
export async function extractZip(zip: JSZip, dest: string): Promise<string[]> {
if (!(await pathExist(dest))) {
throw new Error(`Path ${dest} does not exist`);
}
const files: string[] = [];
for (const [relativePath, entry] of Object.entries(zip.files)) {
if (entry.dir) {
continue;
}
const content = await entry.async("nodebuffer");
const outPath = path.join(dest, relativePath);
const outDir = path.dirname(outPath);
await mkdir(outDir, { recursive: true });
await writeFile(outPath, content);
files.push(outPath);
}
return files;
}
+218
View File
@@ -0,0 +1,218 @@
import { createWriteStream, ensureDir } from "fs-extra";
import path from "path";
import { Readable } from "stream";
import yauzl, { ZipFile, Options, Entry } from "yauzl"
export class BsmZipExtractor {
private static readonly YAUZL_OPEN_OPTIONS: Options = {
lazyEntries: true,
decodeStrings: true,
autoClose: false,
};
public static fromPath(path: string): Promise<BsmZipExtractor> {
return new Promise((resolve, reject) => {
yauzl.open(path, BsmZipExtractor.YAUZL_OPEN_OPTIONS, (error: Error, zip: yauzl.ZipFile) => {
if (error) return reject(error);
resolve(new BsmZipExtractor(zip));
});
});
}
public static fromBuffer(buffer: Buffer): Promise<BsmZipExtractor> {
return new Promise((resolve, reject) => {
yauzl.fromBuffer(buffer, BsmZipExtractor.YAUZL_OPEN_OPTIONS, (error: Error, zip: yauzl.ZipFile) => {
if (error) return reject(error);
resolve(new BsmZipExtractor(zip));
});
});
}
private readonly zip: ZipFile;
private readonly entriesMap = new Map<string, BsmZipExtractorEntry>();
private constructor(zip: ZipFile) {
this.zip = zip;
}
private async readEntry(): Promise<BsmZipExtractorEntry> {
return new Promise((resolve, reject) => {
const onEntry = (entry: Entry) => {
cleanup();
resolve(new BsmZipExtractorEntry({ entry, zip: this.zip }));
};
const onEnd = () => {
cleanup();
resolve(null);
};
const onError = (err: Error) => {
cleanup();
reject(err);
};
const cleanup = () => {
this.zip.removeListener("entry", onEntry);
this.zip.removeListener("end", onEnd);
this.zip.removeListener("error", onError);
};
this.zip.once("entry", onEntry);
this.zip.once("end", onEnd);
this.zip.once("error", onError);
this.zip.readEntry();
});
}
// The first time this method is called, it will read all the entries and store them in a map
// The next times it will return the entries from the map to avoid reopening the zip file again
public async *entries(): AsyncGenerator<BsmZipExtractorEntry> {
for (const entry of Array.from(this.entriesMap.values())) {
yield entry;
}
let entry: BsmZipExtractorEntry = await this.readEntry();
while (entry) {
this.entriesMap.set(entry.fileName, entry);
yield entry;
entry = await this.readEntry();
}
}
public async findEntry(func: (entry: BsmZipExtractorEntry) => boolean): Promise<BsmZipExtractorEntry|null> {
for await (const entry of this.entries()) {
if (func(entry)) {
return entry;
}
}
return null;
}
public async filterEntries(func: (entry: BsmZipExtractorEntry) => boolean): Promise<BsmZipExtractorEntry[]> {
const filtered: BsmZipExtractorEntry[] = [];
for await (const entry of this.entries()) {
if (func(entry)) {
filtered.push(entry);
}
}
return filtered;
}
public async getEntry(fileName: string): Promise<BsmZipExtractorEntry> {
return this.entriesMap.get(fileName) ?? this.findEntry(entry => entry.fileName === fileName);
}
/**
* Extracts all entries from the zip file to the destination folder
* @param destination
* @param opt - { entriesNames?: string[], abortToken: AbortController }
* @returns {Promise<string[]>} The list of extracted files (relative paths in the destination folder)
*
* `opt` object:
* - `entriesNames` - The list of entries to extract (can be regexs or glob pattern). If not provided, all entries will be extracted
* - `abortToken` - The AbortController instance to abort the extraction
*
*/
public async extract(destination: string, opt?: { entriesNames?: (string|RegExp)[], abortToken?: AbortController }): Promise<string[]> {
const entriesNames = opt?.entriesNames;
const abortToken = opt?.abortToken;
if (abortToken?.signal?.aborted) {
return [];
}
await ensureDir(destination);
const extracted = new Set<string>()
for await (const entry of this.entries()) {
if (abortToken?.signal?.aborted) {
break;
}
if (entriesNames && !entriesNames.some(name => typeof name === "string" ? entry.fileName === name : name.test(entry.fileName))){
continue;
}
const extractedFile = await entry.extract(destination);
const dirname = path.dirname(extractedFile);
// Make zip that use backslash as separator have the same behavior as zip that use forward slash
if(dirname !== "."){
const split = dirname.split(path.posix.sep);
for (let i = 1; i <= split.length; i++) {
const folder = split.slice(0, i).join(path.posix.sep);
extracted.add(folder + path.posix.sep);
}
}
extracted.add(extractedFile);
}
return Array.from(extracted);
}
public close(): void {
this.zip.close();
}
}
class BsmZipExtractorEntry {
private readonly entry: Entry;
private readonly zip: ZipFile;
constructor(opt: { entry: Entry, zip: ZipFile }) {
this.entry = opt.entry;
this.zip = opt.zip;
}
public async read(): Promise<Buffer> {
return new Promise((resolve, reject) => {
this.zip.openReadStream(this.entry, (error: Error, stream: Readable) => {
if (error) return reject(error);
const buffers: Buffer[] = [];
stream.on("data", (data) => buffers.push(data as Buffer));
stream.on("end", () => resolve(Buffer.concat(buffers)));
stream.on("error", reject);
});
});
}
public async extract(destination: string): Promise<string> {
const destPath = path.join(destination, this.entry.fileName);
if (this.isDirectory) {
return ensureDir(destPath).then(() => this.entry.fileName);
}
return new Promise((resolve, reject) => {
this.zip.openReadStream(this.entry, async (error: Error, stream: Readable) => {
if (error) return reject(error);
await ensureDir(path.dirname(destPath));
const writeStream = createWriteStream(destPath);
stream.pipe(writeStream);
writeStream.on("finish", () => resolve(this.entry.fileName));
writeStream.on("error", reject);
});
});
}
public get isDirectory(): boolean {
return this.entry.fileName.endsWith("/");
}
public get fileName(): string {
return this.entry.fileName;
}
}
+27 -14
View File
@@ -1,4 +1,3 @@
import JSZip from "jszip";
import fetch from "node-fetch";
import { CustomError } from "../../shared/models/exceptions/custom-error.class";
import { mkdirs, createWriteStream, pathExists, WriteStream } from "fs-extra";
@@ -7,6 +6,8 @@ import { inflate } from "pako"
import { EMPTY, Observable, ReplaySubject, Subscriber, catchError, filter, from, lastValueFrom, mergeMap, scan, share, tap } from "rxjs";
import { Progression, hashFile } from "../helpers/fs.helpers";
import { OculusDownloaderErrorCodes } from "../../shared/models/bs-version-download/oculus-download.model";
import { BsmZipExtractor } from "./bsm-zip-extractor.class";
import { tryit } from "../../shared/helpers/error.helpers";
export class OculusDownloader {
@@ -23,26 +24,38 @@ export class OculusDownloader {
return `https://securecdn.oculus.com/binaries/segment/?access_token=${token}&binary_id=${binaryId}&segment_sha256=${segmentSha256}`;
}
private async downloadManifestZip(manifestUrl: string): Promise<JSZip> {
private async downloadManifestZip(manifestUrl: string): Promise<Buffer> {
const response = await fetch(manifestUrl);
const arrBuffer = await response.arrayBuffer();
return JSZip.loadAsync(arrBuffer);
return Buffer.from(arrBuffer);
}
private async getManifest(): Promise<OculusManifest> {
const downloadUrl = this.getDownloadManifestUrl(this.options.accessToken, this.options.binaryId);
const manifestZip = await this.downloadManifestZip(downloadUrl).catch(err => CustomError.throw(err, "DOWNLOAD_MANIFEST_FAILED"));
const manifestFile = manifestZip.file("manifest.json");
const buffer = await this.downloadManifestZip(downloadUrl)
.catch(err => CustomError.throw(err, "DOWNLOAD_MANIFEST_FAILED"));
if(!manifestFile){
const zip = await BsmZipExtractor.fromBuffer(buffer);
const entry = await zip.getEntry("manifest.json");
if(!entry) {
throw new CustomError("Manifest file not found", "MANIFEST_FILE_NOT_FOUND");
}
return manifestFile.async("text").then(JSON.parse).catch(err => CustomError.throw(err, "PARSE_MANIFEST_FILE_FAILED"));
const manifest = await entry.read();
zip.close();
const { result, error } = tryit(() => JSON.parse(manifest.toString()) as OculusManifest);
if(error){
throw CustomError.throw(error, "PARSE_MANIFEST_FILE_FAILED")
}
return result;
}
private downloadManifestFile(file: OculusManifestFile, destination: string): Observable<Progression<OculusManifestFile>> {
const downloadSegment = async (segment: OculusManifestFileSegment): Promise<ArrayBuffer> => {
const segmentUrl = this.getDownloadSegmentUrl(this.options.accessToken, this.options.binaryId, segment[1]);
const response = await fetch(segmentUrl);
@@ -66,11 +79,11 @@ export class OculusDownloader {
const arrBuffer = await downloadSegment(segment);
const inflated = inflate(arrBuffer);
await writeStream.write(inflated);
writeStream.write(inflated);
progress.current += inflated.byteLength;
progress.diff = inflated.byteLength;
sub.next(progress);
}
@@ -86,7 +99,7 @@ export class OculusDownloader {
});
}
private isFileIntegrityValid(file: OculusFileWithName, folder: string): Promise<boolean> {
private async isFileIntegrityValid(file: OculusFileWithName, folder: string): Promise<boolean> {
const [fileName, fileData] = file;
const destination = path.join(folder, fileName);
@@ -138,7 +151,7 @@ export class OculusDownloader {
if(this.isDownloading){
throw new CustomError("Already downloading", "ALREADY_DOWNLOADING");
}
this.options = options;
this.isDownloading = true;
@@ -182,7 +195,7 @@ export class OculusDownloader {
})));
const integrity = await lastValueFrom(this.verifyIntegrity(manifest, options.destination)).catch(err => CustomError.throw(err, "VERIFY_INTEGRITY_FAILED"));
if(integrity.data.length > 0){
throw new CustomError("Some files failed to download", "SOME_FILES_FAILED_TO_DOWNLOAD", integrity.data);
}
@@ -242,4 +255,4 @@ interface Logger {
info: (...args: unknown[]) => void;
warn: (...args: unknown[]) => void;
error: (...args: unknown[]) => void;
}
}
@@ -8,7 +8,6 @@ import { UtilsService } from "../utils.service";
import crypto from "crypto";
import { lstatSync } from "fs";
import { copy, createReadStream, ensureDir, pathExists, realpath, unlink } from "fs-extra";
import StreamZip from "node-stream-zip";
import { RequestService } from "../request.service";
import sanitize from "sanitize-filename";
import { DeepLinkService } from "../deep-link.service";
@@ -23,6 +22,7 @@ import { allSettled } from "../../../shared/helpers/promise.helpers";
import { splitIntoChunk } from "../../../shared/helpers/array.helpers";
import { IpcService } from "../ipc.service";
import { parseMapInfoDat } from "../../../shared/parsers/maps/map-info.parser";
import { BsmZipExtractor } from "../../models/bsm-zip-extractor.class";
export class LocalMapsManagerService {
private static instance: LocalMapsManagerService;
@@ -137,18 +137,15 @@ export class LocalMapsManagerService {
return { mapInfo, coverUrl, songUrl, hash, path: mapPath };
}
private async downloadMapZip(zipUrl: string): Promise<{ zip: StreamZip.StreamZipAsync; zipPath: string }> {
private async downloadMapZip(zipUrl: string): Promise<string> {
const fileName = `${path.basename(zipUrl, ".zip")}-${crypto.randomUUID()}.zip`;
const tempPath = this.utils.getTempPath();
await ensureFolderExist(this.utils.getTempPath());
const zipPath = (await lastValueFrom(this.reqService.downloadFile(zipUrl, {
return (await lastValueFrom(this.reqService.downloadFile(zipUrl, {
destFolder: tempPath,
filename: fileName
}))).data;
const zip = new StreamZip.async({ file: zipPath });
return { zip, zipPath };
}
private openOneClickDownloadMapWindow(mapId: string, isHash = false): void {
@@ -279,16 +276,15 @@ export class LocalMapsManagerService {
return installedMap;
}
const { zip, zipPath } = await this.downloadMapZip(zipUrl);
const zipPath = await this.downloadMapZip(zipUrl);
const zip = await BsmZipExtractor.fromPath(zipPath);
if (!zip) {
throw `Cannot download ${zipUrl}`;
}
await ensureFolderExist(mapPath);
await zip.extract(null, mapPath);
await zip.close();
await zip.extract(mapPath);
zip.close();
await unlink(zipPath);
const localMap = await this.loadMapInfoFromPath(mapPath);
@@ -11,11 +11,11 @@ import { BS_EXECUTABLE } from "../../constants";
import log from "electron-log";
import { deleteFolder, pathExist, unlinkPath } from "../../helpers/fs.helpers";
import { lastValueFrom } from "rxjs";
import JSZip from "jszip";
import { extractZip } from "../../helpers/zip.helpers";
import recursiveReadDir from "recursive-readdir";
import { sToMs } from "../../../shared/helpers/time.helpers";
import { ensureDir, pathExistsSync } from "fs-extra";
import { pathExistsSync } from "fs-extra";
import { BsmZipExtractor } from "../../models/bsm-zip-extractor.class";
import crypto from "crypto";
export class BsModsManagerService {
private static instance: BsModsManagerService;
@@ -45,6 +45,7 @@ export class BsModsManagerService {
this.bsLocalService = BSLocalVersionService.getInstance();
this.utilsService = UtilsService.getInstance();
this.requestService = RequestService.getInstance();
this.utilsService = UtilsService.getInstance();
}
private async getModFromHash(hash: string): Promise<Mod> {
@@ -123,7 +124,7 @@ export class BsModsManagerService {
return this.getIpaFromHash(injectorMd5);
}
private async downloadZip(zipUrl: string): Promise<JSZip> {
private async downloadZip(zipUrl: string): Promise<BsmZipExtractor> {
zipUrl = path.join(this.beatModsApi.BEAT_MODS_URL, zipUrl);
log.info("Download mod zip", zipUrl);
@@ -139,10 +140,7 @@ export class BsModsManagerService {
return null;
}
return JSZip.loadAsync(buffer).catch(e => {
log.error("ZIP", "Error while loading zip", e);
return null;
});
return BsmZipExtractor.fromBuffer(buffer);
}
private async executeBSIPA(version: BSVersion, args: string[]): Promise<boolean> {
@@ -209,43 +207,38 @@ export class BsModsManagerService {
log.info("Start download mod zip", mod.name, download.url);
const zip = await this.downloadZip(download.url);
log.info("Mod zip download end", mod.name, download.url, !!zip);
log.info("Mod zip download end", mod.name, download.url);
if (!zip) {
return false;
}
const crypto = require("crypto");
const files = await zip.files;
let hashCount = 0;
for await (const entry of zip.entries()) {
const buffer = await entry.read();
const md5Hash = crypto.createHash("md5")
.update(buffer)
.digest("hex");
hashCount += +download.hashMd5.some(md5 => md5.hash === md5Hash);
}
const checkedEntries = (
await Promise.all(
Object.values(files).map(async entry => {
const data = await entry.async("nodebuffer");
const entryMd5 = crypto.createHash("md5").update(data).digest("hex");
return download.hashMd5.some(md5 => md5.hash === entryMd5) ? entry : undefined;
})
).catch(e => {
log.error("Error while checking mod zip entries", mod.name, e);
throw e;
})
).filter(entry => !!entry);
if (checkedEntries.length !== download.hashMd5.length) {
if (hashCount !== download.hashMd5.length) {
return false;
}
const verionPath = await this.bsLocalService.getVersionPath(version);
const versionPath = await this.bsLocalService.getVersionPath(version);
const isBSIPA = mod.name.toLowerCase() === "bsipa";
const destDir = isBSIPA ? verionPath : path.join(verionPath, ModsInstallFolder.PENDING);
const destDir = isBSIPA ? versionPath : path.join(versionPath, ModsInstallFolder.PENDING);
await ensureDir(destDir);
log.info("Start extracting mod zip", mod.name, "to", destDir);
const extracted = await extractZip(zip, destDir)
const extracted = await zip.extract(destDir)
.then(() => true)
.catch(e => {
log.error("Error while extracting mod zip", e);
return false;
})
.finally(() => {
zip.close();
});
log.info("Mod zip extraction end", mod.name, "to", destDir, "success:", extracted);
+18
View File
@@ -0,0 +1,18 @@
const HashAlgorithmsLengths = {
sha1: 40,
} as const;
export function findHashInString(str: string, algorithm: keyof typeof HashAlgorithmsLengths = 'sha1'): string | undefined {
if(!str) { return undefined; }
const hashLength = HashAlgorithmsLengths[algorithm];
const regex = new RegExp(`[a-fA-F0-9]{${hashLength}}`, "g");
const match = regex.exec(str);
return match ? match[0] : undefined;
}
export function escapeRegExp(str: string): string {
// Regex taken from lodash escapeRegExp function
return str.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&');
}