diff --git a/assets/jsons/translations/de.json b/assets/jsons/translations/de.json index 5f5b89b0..46fb348e 100644 --- a/assets/jsons/translations/de.json +++ b/assets/jsons/translations/de.json @@ -515,8 +515,7 @@ "success": "Alle Karten wurden erfolgreich importiert.", "some-success": "Einige Karten wurden erfolgreich importiert.", "only-accept-zip": "Nur ZIP-Dateien werden unterstützt.", - "not-found-zip": "Die ZIP-Datei existiert nicht.", - "invalid-zip": "Die ZIP-Datei enthält keine DAT-Dateien." + "invalid-zip": "Die Zip-Datei(en) enthält/en keine \"Info.dat\"-Datei." } } }, diff --git a/assets/jsons/translations/en.json b/assets/jsons/translations/en.json index bd8c5579..f60e2293 100644 --- a/assets/jsons/translations/en.json +++ b/assets/jsons/translations/en.json @@ -522,8 +522,7 @@ "success": "Imported all maps successfully.", "some-success": "Imported some maps successfully.", "only-accept-zip": "Only zip files are supported.", - "not-found-zip": "Zip file does not exists.", - "invalid-zip": "Zip file does not contain any dat files." + "invalid-zip": "Zip/s file do not contain any \"Info.dat\" file." } } }, diff --git a/assets/jsons/translations/es.json b/assets/jsons/translations/es.json index 494b6c5c..eaa0d283 100644 --- a/assets/jsons/translations/es.json +++ b/assets/jsons/translations/es.json @@ -515,8 +515,7 @@ "success": "Se importaron todas las mapas con éxito.", "some-success": "Se importaron algunos mapas con éxito.", "only-accept-zip": "Solo se admiten archivos ZIP.", - "not-found-zip": "El archivo ZIP no existe.", - "invalid-zip": "El archivo ZIP no contiene archivos DAT." + "invalid-zip": "El/Los archivo(s) Zip no contienen ningún archivo \"Info.dat\"." } } }, diff --git a/assets/jsons/translations/fr.json b/assets/jsons/translations/fr.json index ec222200..73a95500 100644 --- a/assets/jsons/translations/fr.json +++ b/assets/jsons/translations/fr.json @@ -515,8 +515,7 @@ "success": "Toutes les cartes ont été importées avec succès.", "some-success": "Certaines cartes ont été importées avec succès.", "only-accept-zip": "Seules les fichiers ZIP sont pris en charge.", - "not-found-zip": "Le fichier ZIP n'existe pas.", - "invalid-zip": "Le fichier ZIP ne contient aucun fichier DAT." + "invalid-zip": "Le(s) fichier(s) Zip ne contient/contiennent aucun fichier \"Info.dat\"." } } }, diff --git a/assets/jsons/translations/ja.json b/assets/jsons/translations/ja.json index 9af9b1af..c5ac792c 100644 --- a/assets/jsons/translations/ja.json +++ b/assets/jsons/translations/ja.json @@ -515,8 +515,7 @@ "success": "すべてのマップが正常にインポートされました。", "some-success": "一部のマップが正常にインポートされました。", "only-accept-zip": "ZIPファイルのみがサポートされています。", - "not-found-zip": "ZIPファイルが存在しません。", - "invalid-zip": "ZIPファイルにはDATファイルが含まれていません。" + "invalid-zip": "ZIPファイルには「Info.dat」ファイルが含まれていません。" } } }, diff --git a/assets/jsons/translations/ru.json b/assets/jsons/translations/ru.json index 7774b18b..7d58731e 100644 --- a/assets/jsons/translations/ru.json +++ b/assets/jsons/translations/ru.json @@ -515,8 +515,7 @@ "success": "Все карты были успешно импортированы.", "some-success": "Некоторые карты были успешно импортированы.", "only-accept-zip": "Поддерживаются только ZIP-файлы.", - "not-found-zip": "ZIP-файл не существует.", - "invalid-zip": "ZIP-файл не содержит файлов DAT." + "invalid-zip": "Файл(ы) ZIP не содержат файл(ы) \"Info.dat\"." } } }, diff --git a/assets/jsons/translations/zh-tw.json b/assets/jsons/translations/zh-tw.json index 2c6e2c33..7fa340df 100644 --- a/assets/jsons/translations/zh-tw.json +++ b/assets/jsons/translations/zh-tw.json @@ -515,8 +515,7 @@ "success": "所有地圖已成功導入。", "some-success": "一些地圖已成功導入。", "only-accept-zip": "只支持 ZIP 文件。", - "not-found-zip": "ZIP 文件不存在。", - "invalid-zip": "ZIP 文件不包含任何 DAT 文件。" + "invalid-zip": "ZIP 檔案不包含任何 \"Info.dat\" 檔案。" } } }, diff --git a/assets/jsons/translations/zh.json b/assets/jsons/translations/zh.json index 0336e096..c7f8994d 100644 --- a/assets/jsons/translations/zh.json +++ b/assets/jsons/translations/zh.json @@ -515,8 +515,7 @@ "success": "所有地图已成功导入。", "some-success": "一些地图已成功导入。", "only-accept-zip": "只支持 ZIP 文件。", - "not-found-zip": "ZIP 文件不存在。", - "invalid-zip": "ZIP 文件不包含任何 DAT 文件。" + "invalid-zip": "ZIP 文件不包含任何 \"Info.dat\" 文件。" } } }, diff --git a/src/main/helpers/zip.helpers.ts b/src/main/helpers/zip.helpers.ts index 3a2604a0..f1587133 100644 --- a/src/main/helpers/zip.helpers.ts +++ b/src/main/helpers/zip.helpers.ts @@ -1,44 +1,7 @@ import JSZip from "jszip"; import { pathExist } from "./fs.helpers"; import path from "path"; -import { mkdir, writeFile, readFile } from "fs/promises"; -import { pathExistsSync } from "fs-extra"; - -// JSZip config defaults for now to avoid zip bombs -const MAX_FILES = 1_000; -const MAX_SIZE = 1024 * 1024 * 100; // 100MB - - -export async function processZip( - // path to the zip or the JSZip object itself - zip: string | JSZip, - // Should return the number of bytes read - handleFile: (relativePath: string, file: JSZip.JSZipObject) => Promise | number -): Promise { - if (typeof zip === "string") { - if (!pathExistsSync(zip)) { - throw new Error(`Path ${zip} does not exists`); - } - - const data = await readFile(zip); - zip = await JSZip.loadAsync(data); - } - - let fileCount = 0; - let totalSize = 0; - - for (const [relativePath, file] of Object.entries(zip.files)) { - ++fileCount; - if (fileCount > MAX_FILES) { - throw new Error(`Reached maximum number of files on "${zip}"`); - } - - totalSize += await handleFile(relativePath, file); - if (totalSize > MAX_SIZE) { - throw new Error(`Reached maximum size on "${zip}"`); - } - } -} +import { mkdir, writeFile } from "fs/promises"; export async function extractZip(zip: JSZip, dest: string): Promise { if (!(await pathExist(dest))) { diff --git a/src/main/services/additional-content/maps/local-maps-manager.service.ts b/src/main/services/additional-content/maps/local-maps-manager.service.ts index a5c1ac98..0f6a60a8 100644 --- a/src/main/services/additional-content/maps/local-maps-manager.service.ts +++ b/src/main/services/additional-content/maps/local-maps-manager.service.ts @@ -7,14 +7,14 @@ import { InstallationLocationService } from "../../installation-location.service import { UtilsService } from "../../utils.service"; import crypto, { BinaryLike } from "crypto"; import { lstatSync } from "fs"; -import { copy, createReadStream, ensureDir, pathExists, pathExistsSync, realpath, unlink, writeFile } from "fs-extra"; +import { copy, createReadStream, ensureDir, mkdir, pathExists, pathExistsSync, realpath, unlink, unlinkSync, writeFile } from "fs-extra"; import StreamZip from "node-stream-zip"; import { RequestService } from "../../request.service"; import sanitize from "sanitize-filename"; import { DeepLinkService } from "../../deep-link.service"; import log from "electron-log"; import { WindowManagerService } from "../../window-manager.service"; -import { Observable, Subject, lastValueFrom } from "rxjs"; +import { Observable, Subject, Subscriber, lastValueFrom } from "rxjs"; import { Archive } from "../../../models/archive.class"; import { Progression, deleteFolder, ensureFolderExist, getFilesInFolder, getFoldersInFolder, pathExist } from "../../../helpers/fs.helpers"; import { readFile } from "fs/promises"; @@ -29,7 +29,6 @@ import { FieldRequired } from "shared/helpers/type.helpers"; import { MapInfo } from "shared/models/maps/info/map-info.model"; import { parseMapInfoDat } from "shared/parsers/maps/map-info.parser"; import { tryit } from "shared/helpers/error.helpers"; -import { processZip } from "main/helpers/zip.helpers"; import JSZip from "jszip"; import { CustomError } from "shared/models/exceptions/custom-error.class"; @@ -53,6 +52,10 @@ export class LocalMapsManagerService { ScoreSaber: "web+bsmap", }; + private readonly INFO_DAT_REGEX = path.sep === "/" + ? /(^|\/)(I|i)nfo.dat$/ + : /(^|\\\\)(I|i)nfo.dat$/; + private readonly localVersion: BSLocalVersionService; private readonly installLocation: InstallationLocationService; private readonly utils: UtilsService; @@ -328,76 +331,163 @@ export class LocalMapsManagerService { return null; } - - public importMaps(zipPaths: string[], version?: BSVersion): Observable> { - const progress: Progression = { - total: zipPaths.length, - current: 0, - }; - return new Observable>(observer => { - (async () => { - try { - observer.next(progress); // 0% - - for (const zipPath of zipPaths) { - ++progress.current; - try { - progress.data = await this.importMap(zipPath, version); - } catch (error: any) { - log.error(`Could not import "${zipPath}"`, error); - progress.data = undefined; - } - observer.next(progress); - } - } catch(error: any) { - observer.error(error); - } finally { - observer.complete(); - } - })(); + this.handleZipPaths(zipPaths, observer, version) + .catch(error => observer.error(error)) + .finally(() => observer.complete()); }); } - private async importMap(zipPath: string, version?: BSVersion): Promise { + private async handleZipPaths( + zipPaths: string[], + observer: Subscriber>, + version?: BSVersion + ): Promise { + const info: { + zips: ({ + path: string; + // if the zip contains single or multiple maps + single: boolean; + // paths within the zip file where a map is located + folders: string[]; + })[]; + total: number; + } = { + zips: [], + total: 0, + }; + + for (const zipPath of zipPaths) { + try { + const zip = await JSZip.loadAsync(await readFile(zipPath)); + const files = zip.file(this.INFO_DAT_REGEX); + if (files.length === 0) { + log.warn(`Zip file "${zipPath}" does not contain any "Info.dat" file`); + continue; + } + + info.total += files.length; + info.zips.push({ + path: zipPath, + single: files.findIndex(file => file.name.includes(path.sep)) === -1, + folders: files.map(file => path.dirname(file.name)), + }); + } catch (error: any) { + log.warn(`Could not count maps ${zipPath}`, error); + } + } + + if (info.total === 0) { + throw new CustomError("No \"Info.dat\" file located in any of the zip files", "invalid-zip"); + } + + // Setting up the progress bar + const progress: Progression = { + total: info.total, + current: 0, + }; + const mapsFolder = await this.getMapsFolderPath(version); + + observer.next(progress); // 0% + + for (const zipInfo of info.zips) { + try { + const content = await readFile(zipInfo.path); + const zip = await JSZip.loadAsync(content); + + // Zip containing only a single map + if (zipInfo.single) { + ++progress.current; + + progress.data = await this.importMap( + zip, zipInfo.path, "", + path.basename(zipInfo.path, ".zip"), + mapsFolder + ) + observer.next(progress); + continue; + } + + // Zip containing multiple maps + for (const relativeFolder of zipInfo.folders) { + ++progress.current; + try { + progress.data = await this.importMap( + zip, zipInfo.path, + relativeFolder, + path.basename(relativeFolder), + mapsFolder + ); + } catch (error: any) { + log.error(`Could not import "${zipInfo.path}"`, error); + progress.data = undefined; + } finally { + observer.next(progress); + } + } + } catch (error: any) { + log.error(`Could not import "${zipInfo.path}"`, error); + progress.data = undefined; + observer.next(progress); + } + } + } + + private async importMap( + zip: JSZip, + zipPath: string, // where the zip file is located + relativeFolder: string, // where is the map relative in the zip file + mapName: string, // Map/Song name + mapsFolder: string // Maps folder depending on the version + ): Promise { + let mapPath = ""; + let existing = false; try { - if (!pathExistsSync(zipPath)) { - throw new CustomError(`Zip file "${zipPath}" does not exist`, "not-found-zip"); + mapPath = path.join(mapsFolder, mapName); + existing = pathExistsSync(mapPath); + log.info(`Importing map from "${zipPath}" in "${relativeFolder}" to "${mapPath}"`); + + if (!existing) { + await mkdir(mapPath, { recursive: true }); } - const mapFolderName = path.basename(zipPath, ".zip"); - const mapsFolder = await this.getMapsFolderPath(version); - const mapPath = path.join(mapsFolder, mapFolderName); - - log.info(`Importing map "${zipPath}" to "${mapPath}"`); - - const zip = await JSZip.loadAsync(await readFile(zipPath)); - const infoFiles = zip.file(/(I|i)nfo.dat/); - if (infoFiles.length === 0) { // Simple check for importing maps - throw new CustomError(`Invalid zip file "${zipPath}"`, "invalid-zip"); + // Prep the files + let files: { [key: string]: JSZip.JSZipObject; } = {}; + if (relativeFolder === "") { + // Zip containing only a single map + files = zip.files; + } else { + // Zip containing multiple maps + // async doesn't work here and zip.folder().files does not work as you expect + zip.folder(relativeFolder).forEach((relativeFolder, file) => { + files[relativeFolder] = file; + }); } - await ensureFolderExist(mapPath); - await processZip(zip, async (relativePath, file) => { + for (const [relativePath, file] of Object.entries(files)) { const filepath = path.join(mapPath, relativePath); if (file.dir) { await ensureFolderExist(filepath); - return 0; + continue; } - log.info(`Extracting "${filepath}"`); + log.info(`Extracting to "${filepath}"`); const content = await file.async("nodebuffer"); await writeFile(filepath, content); - - return content.length; - }); + } const localMap = await this.loadMapInfoFromPath(mapPath); localMap.songDetails = this.songDetailsCache.getSongDetails(localMap.hash); - return localMap; } catch (error: any) { + // If the mapPath isn't yet added, delete the map folder + if (!existing && mapPath) { + unlinkSync(mapPath); + } else if (existing && mapPath) { + log.warn(`Map folder ${mapPath} could be broken`); + } + throw error instanceof CustomError ? error : CustomError.fromError(error); diff --git a/src/renderer/components/maps-playlists-panel/maps-playlists-panel.component.tsx b/src/renderer/components/maps-playlists-panel/maps-playlists-panel.component.tsx index 95eceeaf..861f5d1c 100644 --- a/src/renderer/components/maps-playlists-panel/maps-playlists-panel.component.tsx +++ b/src/renderer/components/maps-playlists-panel/maps-playlists-panel.component.tsx @@ -121,23 +121,7 @@ export function MapsPlaylistsPanel({ version, isActive }: Props) { return; } - const importCount = await mapsManager.importMaps(paths, version); - if (importCount === 0) { - return; - } - - if (importCount < paths.length) { - notifications.notifySuccess({ - title: "notifications.maps.import-map.titles.success", - desc: "notifications.maps.import-map.msgs.some-success", - }); - return; - } - - notifications.notifySuccess({ - title: "notifications.maps.import-map.titles.success", - desc: "notifications.maps.import-map.msgs.success", - }); + await mapsManager.importMaps(paths, version); } const dropDownItems = ((): DropDownItem[] => { diff --git a/src/renderer/components/maps-playlists-panel/maps/local-maps-list-panel.component.tsx b/src/renderer/components/maps-playlists-panel/maps/local-maps-list-panel.component.tsx index c35ce18b..5d361e39 100644 --- a/src/renderer/components/maps-playlists-panel/maps/local-maps-list-panel.component.tsx +++ b/src/renderer/components/maps-playlists-panel/maps/local-maps-list-panel.component.tsx @@ -123,20 +123,24 @@ export const LocalMapsListPanel = forwardRef(({ ve }, [isActiveOnce, version]) - const importListener = (importMap: BsmLocalMap, targetVersion?: BSVersion) => { + const importListener = (importMaps: BsmLocalMap[], targetVersion?: BSVersion) => { if (!equal(targetVersion, version)) { return; } const mapsCopy = maps ? [ ...maps ] : []; - // importMap can collide with existing map - const index = maps.findIndex(map => map.songDetails?.name === importMap.songDetails.name); - if (index > -1) { - mapsCopy.splice(index, 1); + for (const importMap of importMaps) { + if (importMap.songDetails) { + // importMap can collide with existing map + const index = maps.findIndex(map => map.songDetails?.name === importMap.songDetails.name); + if (index > -1) { + mapsCopy.splice(index, 1); + } + } } - setMaps([importMap, ...mapsCopy]); + setMaps([...importMaps, ...mapsCopy]); }; const loadMaps = () => { diff --git a/src/renderer/services/maps-manager.service.ts b/src/renderer/services/maps-manager.service.ts index 4c6055b1..f73c4529 100644 --- a/src/renderer/services/maps-manager.service.ts +++ b/src/renderer/services/maps-manager.service.ts @@ -9,7 +9,7 @@ import { DeleteMapsModal } from "renderer/components/modal/modal-types/delete-ma import { ProgressBarService } from "./progress-bar.service"; import { NotificationService } from "./notification.service"; import { ConfigurationService } from "./configuration.service"; -import { map, last, catchError } from "rxjs/operators"; +import { map, last, catchError, tap } from "rxjs/operators"; import { ProgressionInterface } from "shared/models/progress-bar"; import { FolderLinkState, VersionFolderLinkerService } from "./version-folder-linker.service"; import { SongDetails } from "shared/models/maps"; @@ -29,6 +29,8 @@ export class MapsManagerService { public static readonly REMEMBER_CHOICE_DELETE_MAP_KEY = "not-confirm-delete-map"; public static readonly RELATIVE_MAPS_FOLDER = window.electron.path.join("Beat Saber_Data", "CustomLevels"); + private readonly IMPORT_BATCH_SIZE = 8; + private readonly ipcService: IpcService; private readonly modal: ModalService; private readonly progressBar: ProgressBarService; @@ -39,8 +41,7 @@ export class MapsManagerService { private readonly lastLinkedVersion$: Subject = new Subject(); private readonly lastUnlinkedVersion$: Subject = new Subject(); - private importListeners = new Set<((map: BsmLocalMap, version?: BSVersion) => void)>(); - private importCount = 0; + private importListeners = new Set<((maps: BsmLocalMap[], version?: BSVersion) => void)>(); private constructor() { this.ipcService = IpcService.getInstance(); @@ -199,41 +200,70 @@ export class MapsManagerService { }) } - public async importMaps(paths: string[], version?: BSVersion): Promise { + public async importMaps(paths: string[], version?: BSVersion): Promise { try { if (!this.progressBar.require()) { - return 0; + return; } - this.importCount = 0; - const importObserver$ = this.ipcService.sendV2( "bs-maps.import-maps", { paths, version } ); - importObserver$.subscribe(progress => { - if (!progress.data) { - return; - } - - ++this.importCount; - this.importListeners.forEach(listeners => listeners(progress.data, version)); - }); this.progressBar.show(importObserver$.pipe( + catchError(() => of()), map(progress => ({ progression: (progress.current / progress.total) * 100, label: progress.data?.songDetails?.name } as ProgressionInterface)) )); - await lastValueFrom(importObserver$); - return this.importCount; + // Processing + let importCount = 0; + let importTotal = 0; + const mapBatch: BsmLocalMap[] = []; // For batching + + await lastValueFrom(importObserver$.pipe( + tap(progress => { + if (!progress.data) { + importTotal = progress.total; + return; + } + + ++importCount; + mapBatch.push(progress.data); + if (mapBatch.length >= this.IMPORT_BATCH_SIZE) { + const currentBatch = mapBatch.splice(0, this.IMPORT_BATCH_SIZE); + this.importListeners.forEach( + listeners => listeners(currentBatch, version) + ); + } + }), + )); + if (mapBatch.length > 0) { + this.importListeners.forEach(listeners => listeners(mapBatch, version)); + } + + // Done processing + if (importCount === importTotal) { + this.notifications.notifySuccess({ + title: "notifications.maps.import-map.titles.success", + desc: "notifications.maps.import-map.msgs.success", + }); + } else if (importCount > 0) { + this.notifications.notifySuccess({ + title: "notifications.maps.import-map.titles.success", + desc: "notifications.maps.import-map.msgs.some-success", + }); + } } catch (error: any) { this.notifications.notifyError({ title: "notifications.maps.import-map.titles.error", + desc: ["invalid-zip"].includes(error?.code) + ? `notifications.maps.import-map.msgs.${error.code}` + : "misc.unknown" }); - return 0; } finally { this.progressBar.hide(); } @@ -255,11 +285,11 @@ export class MapsManagerService { return lastValueFrom(this.ipcService.sendV2("unregister-maps-deep-link")); } - public addImportListener(listener: (map: BsmLocalMap, version?: BSVersion) => void): void { + public addImportListener(listener: (maps: BsmLocalMap[], version?: BSVersion) => void): void { this.importListeners.add(listener); } - public removeImportListener(listener: (map: BsmLocalMap, version?: BSVersion) => void): void { + public removeImportListener(listener: (maps: BsmLocalMap[], version?: BSVersion) => void): void { this.importListeners.delete(listener); }