From 79bdba74a5af0a73dae4feeed353dbce2eb7f4a1 Mon Sep 17 00:00:00 2001 From: FinnHornhoover Date: Wed, 27 Sep 2023 00:13:59 +0300 Subject: [PATCH] integrated download and hash functionality --- assets/js/server-selector.js | 409 ++++++++--------------------------- index.js | 166 ++++++++------ 2 files changed, 185 insertions(+), 390 deletions(-) diff --git a/assets/js/server-selector.js b/assets/js/server-selector.js index 0a1a875..7642a75 100644 --- a/assets/js/server-selector.js +++ b/assets/js/server-selector.js @@ -261,181 +261,6 @@ function getCacheInfoCell(versionString, cacheMode) { return cellCache; } -function getFileHash(filePath) { - var readCount = 0; - var buff = new Buffer(chunkSize); - var hash = createHash("sha256"); - var file = remotefs.openSync(filePath, "r"); - - while ((readCount = remotefs.readSync(file, buff, 0, chunkSize, null)) > 0) { - hash.update(buff.slice(0, readCount)); - } - - remotefs.closeSync(file); - return hash.digest(encoding="hex"); -} - -/* -function downloadFiles(root, client, sizes, hashes, updateCallback, endCallback) { - if (hashes.length === 0) { - endCallback(); - return; - } - - var filePath = Object.keys(hashes)[0]; - var fileHash = hashes[filePath]; - delete hashes[filePath]; - - var fullFilePath = path.join(root, filePath); - var fullCDNPath = "http://" + [cdn, "ff", "big", filePath].join("/"); - - if (remotefs.existsSync(fullFilePath) && fileHash === getFileHash(fullFilePath)) { - console.log(fullFilePath + " is intact, skipping..."); - return; - } - - downloadFile(client, fullCDNPath, fullFilePath, function () { - var sizeRead = remotefs.statSync(fullFilePath).size; - - if (fileHash === getFileHash(fullFilePath)) { - sizes.intact += sizeRead; - } else { - sizes.altered += sizeRead; - } - - console.log(fullFilePath + " was downloaded from " + fullCDNPath); - - setTimeout(function () { - updateCallback(sizes); - downloadFiles(root, client, sizes, hashes, updateCallback, endCallback); - }, 100); - }); -} - -function downloadFile(client, fullCDNPath, fullFilePath, callback) { - remotefs.ensureDirSync(path.dirname(fullFilePath)); - - var urlParts = url.parse(fullCDNPath); - var req = client.request("GET", urlParts.path, { - "Host": urlParts.hostname, - "Content-Type": "application/octet-stream", - "Referer": fullCDNPath.split("/").slice(0, 4).join("/") + "/", - "Connection": "keep-alive", - }); - var writeStream = remotefs.createWriteStream(fullFilePath); - - var retry = function (err) { - writeStream.end(); - writeStream.destroy(); - - remotefs.removeSync(fullFilePath); - - console.log("Error writing file " + fullFilePath + "\n" + err); - - setTimeout(function () { - console.log("Retrying " + fullCDNPath); - downloadFile(client, fullCDNPath, fullFilePath, callback); - }, 1000); - } - - writeStream.on("error", retry); - - req.on("response", function (res) { - if (res.statusCode >= 300) { - console.log("Error in fetching file " + fullFilePath + " from " + fullCDNPath); - retry("Status Code: " + res.statusCode); - return; - } - - res.pipe(writeStream); - - res.on("end", callback); - res.on("error", retry); - }); - - req.on("error", retry); - - req.end(); -} -*/ - - -function downloadFile(nginxDir, localDir, relativePath, callback) { - var nginxUrl = path.dirname(nginxDir) + "/" + relativePath; - var localPath = path.join(localDir, relativePath); - - // Create directories if they don't exist - var dirName = path.dirname(localPath); - remotefs.ensureDirSync(dirName); - - // HTTP request to download the file - var fileStream = remotefs.createWriteStream(localPath); - var urlParse = url.parse(nginxUrl); - var client = http.createClient(80, urlParse.hostname); - var options = { - url: urlParse.hostname, - port: 80, - path: urlParse.path, - headers: { - "Host": urlParse.hostname, - "Content-Type": "application/octet-stream", - "Referer": nginxDir, - "Connection": "keep-alive", - } - }; - - var request = client.request("GET", urlParse.path, options, function(response) { - // Handle errors - response.on("error", function(err) { - console.error("Error downloading " + relativePath + ": " + err.message); - retryDownload(nginxDir, localDir, relativePath, callback); // Retry download - }); - - response.pipe(fileStream); - - // When the download is complete, invoke the callback - response.on("end", function() { - fileStream.end(); - callback(null, relativePath); - }); - }); - - // Handle HTTP errors - request.on("error", function(err) { - console.error("Error downloading " + relativePath + ": " + err.message); - retryDownload(nginxDir, localDir, relativePath, callback); // Retry download - }); - - request.end(); -} - - // Function to retry downloading a file after a delay -function retryDownload(nginxDir, localDir, relativePath, callback) { - setTimeout(function() { - downloadFile(nginxDir, localDir, relativePath, callback); - }, 1000); // Retry after 1 second -} - - // Function to download multiple files in parallel -function downloadFiles(nginxDir, localDir, fileRelativePaths, allDoneCallback) { - async.eachLimit( - fileRelativePaths, - 5, // Number of parallel downloads - function(relativePath, callback) { - downloadFile(nginxDir, localDir, relativePath, callback); - }, - function(err) { - if (err) { - console.error("Download failed: " + err); - } else { - console.log("All files downloaded successfully."); - allDoneCallback(); - } - } - ); -} - - function loadCacheList() { var versionjson = remotefs.readJsonSync(versionsPath); versionArray = versionjson["versions"]; @@ -516,151 +341,48 @@ function deletePlayableCache(versionString) { resetCacheNames(); if (versionString === "Offline") { + console.log("Cannot delete Offline directory!"); return; } remotefs.removeSync(path.join(cacheRoot, versionString)); console.log("Playable cache " + versionString + " has been removed!"); - checkPlayableCache(versionString); + // this updates the labels etc. properly + ipc.send("hash-check", { + localDir: cacheRoot, + cacheMode: "playable", + versionString: versionString, + hashes: versionHashes.playable[versionString], + }); } function downloadOfflineCache(versionString) { - var buttonDownload = document.getElementById(getCacheButtonID(versionString, "offline", "download")); - var buttonFix = document.getElementById(getCacheButtonID(versionString, "offline", "fix")); - var buttonDelete = document.getElementById(getCacheButtonID(versionString, "offline", "delete")); - var label = document.getElementById(getCacheElemID(versionString, "offline", "label")); - var version = versionArray.filter(function (value) { return value.name === versionString; })[0]; - var sizes = { - intact: 0, - altered: 0, - total: version.offline_size - }; - buttonDownload.setAttribute("disabled", ""); - buttonFix.setAttribute("disabled", ""); - buttonDelete.setAttribute("disabled", ""); - - buttonDownload.children[0].setAttribute("class", "fas fa-spinner fa-spin fa-fw"); - buttonFix.children[0].setAttribute("class", "fas fa-spinner fa-spin fa-fw"); - - /* - var client = http.createClient(80, cdn); - - downloadFiles( - offlineRoot, - client, - sizes, - JSON.parse(JSON.stringify(versionHashes.offline[versionString])), - function (sizesNow) { - label.innerHTML = getCacheLabelText(sizesNow); - }, - function () { - buttonDownload.children[0].setAttribute("class", "fas fa-download"); - buttonFix.children[0].setAttribute("class", "fas fa-hammer"); - checkOfflineCache(versionString); - } - ); - */ - - /* - downloadFiles( - version.url, - offlineRoot, - Object.keys(JSON.parse(JSON.stringify(versionHashes.offline[versionString]))), - function () { - buttonDownload.children[0].setAttribute("class", "fas fa-download"); - buttonFix.children[0].setAttribute("class", "fas fa-hammer"); - checkOfflineCache(versionString); - } - ); - */ - ipc.send("download-files", { - nginxDir: version.url, + // download-files will run hash-check after download + ipc.send("download-files", { + cdnDir: version.url, localDir: offlineRoot, - fileRelativePaths: Object.keys(versionHashes.offline[versionString]), + hashes: versionHashes.offline[versionString], + cacheMode: "offline", versionString: versionString, - }); + }); } function deleteOfflineCache(versionString) { remotefs.removeSync(path.join(offlineRoot, versionString)); console.log("Offline cache " + versionString + " has been removed!"); - checkOfflineCache(versionString); -} - -function checkPlayableCache(versionString) { - var button = document.getElementById(getCacheButtonID(versionString, "playable", "delete")); - var label = document.getElementById(getCacheElemID(versionString, "playable", "label")); - - var sizes = versionSizes.playable[versionString]; - - resetCacheNames(); - - $.each(versionHashes.playable[versionString], function (filePath, fileHash) { - var fullFilePath = path.join(cacheRoot, filePath); - - if (remotefs.existsSync(fullFilePath)) { - var fileSize = remotefs.statSync(fullFilePath).size; - - if (fileHash === getFileHash(fullFilePath)) { - sizes.intact += fileSize; - } else { - sizes.altered += fileSize; - } - } + // this updates the labels etc. properly + ipc.send("hash-check", { + localDir: offlineRoot, + cacheMode: "offline", + versionString: versionString, + hashes: versionHashes.offline[versionString], }); - - if (sizes.intact > 0 || sizes.altered > 0) { - button.removeAttribute("disabled"); - } else { - button.setAttribute("disabled", ""); - } - - label.innerHTML = getCacheLabelText(sizes); -} - -function checkOfflineCache(versionString) { - var buttonDownload = document.getElementById(getCacheButtonID(versionString, "offline", "download")); - var buttonFix = document.getElementById(getCacheButtonID(versionString, "offline", "fix")); - var buttonDelete = document.getElementById(getCacheButtonID(versionString, "offline", "delete")); - var label = document.getElementById(getCacheElemID(versionString, "offline", "label")); - - var sizes = versionSizes.offline[versionString]; - - $.each(versionHashes.offline[versionString], function (filePath, fileHash) { - var fullFilePath = path.join(offlineRoot, filePath); - - if (remotefs.existsSync(fullFilePath)) { - var fileSize = remotefs.statSync(fullFilePath).size; - - if (fileHash === getFileHash(fullFilePath)) { - sizes.intact += fileSize; - } else { - sizes.altered += fileSize; - } - } - }); - - if (sizes.intact > 0 || sizes.altered > 0) { - buttonDownload.setAttribute("disabled", ""); - buttonDelete.removeAttribute("disabled"); - } else { - buttonDownload.removeAttribute("disabled"); - buttonDelete.setAttribute("disabled", ""); - } - - if (sizes.altered > 0) { - buttonFix.removeAttribute("disabled"); - } else { - buttonFix.setAttribute("disabled", ""); - } - - label.innerHTML = getCacheLabelText(sizes); } function resetCacheNames() { @@ -876,34 +598,81 @@ $("#of-deleteservermodal").on("show.bs.modal", function (e) { $("#deleteserver-servername").html(result.description); }); -ipc.on("download-update", function (arg) { - var sizes = versionSizes.offline[arg.versionString]; - sizes.intact += arg.size; +ipc.on("storage-loading-start", function (arg) { + var sizes = versionSizes[arg.cacheMode][arg.versionString]; + + if (arg.resetIntactSize) { + sizes.intact = 0; + } + sizes.altered = 0; + + var buttonDelete = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "delete")); + var buttonDownload = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "download")); + var buttonFix = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "fix")); + var label = document.getElementById(getCacheElemID(arg.versionString, arg.cacheMode, "label")); + + buttonDelete.setAttribute("disabled", ""); + buttonDelete.children[0].setAttribute("class", "fas fa-spinner fa-spin fa-fw"); + + if (arg.cacheMode === "offline") { + buttonDownload.setAttribute("disabled", ""); + buttonDownload.children[0].setAttribute("class", "fas fa-spinner fa-spin fa-fw"); + + buttonFix.setAttribute("disabled", ""); + buttonFix.children[0].setAttribute("class", "fas fa-spinner fa-spin fa-fw"); + } - var label = document.getElementById(getCacheElemID(arg.versionString, "offline", "label")); label.innerHTML = getCacheLabelText(sizes); }); -ipc.on("download-success", function (versionString) { - var buttonDownload = document.getElementById(getCacheButtonID(versionString, "offline", "download")); - var buttonFix = document.getElementById(getCacheButtonID(versionString, "offline", "fix")); - - buttonDownload.children[0].setAttribute("class", "fas fa-download"); - buttonFix.children[0].setAttribute("class", "fas fa-hammer"); - checkOfflineCache(versionString); -}); - -ipc.on("hash-update", function (arg) { +ipc.on("storage-label-update", function (arg) { var sizes = versionSizes[arg.cacheMode][arg.versionString]; - - if (arg.sizes) { - sizes.intact += arg.sizes.intact; - sizes.altered += arg.sizes.altered; - } else { - sizes.intact = 0; - sizes.altered = 0; - } + sizes.intact += arg.sizes.intact; + sizes.altered += arg.sizes.altered; var label = document.getElementById(getCacheElemID(arg.versionString, arg.cacheMode, "label")); label.innerHTML = getCacheLabelText(sizes); +}); + +ipc.on("download-complete", function (arg) { + ipc.send("hash-check", { + localDir: (arg.cacheMode === "offline") ? offlineRoot : cacheRoot, + cacheMode: arg.cacheMode, + versionString: arg.versionString, + hashes: versionHashes[arg.cacheMode][arg.versionString], + }); +}); + +ipc.on("hash-check-complete", function (arg) { + var sizes = versionSizes[arg.cacheMode][arg.versionString]; + + var buttonDelete = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "delete")); + var buttonDownload = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "download")); + var buttonFix = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "fix")); + + buttonDelete.children[0].setAttribute("class", "fas fa-trash-alt"); + + if (arg.cacheMode === "offline") { + buttonDownload.children[0].setAttribute("class", "fas fa-download"); + buttonFix.children[0].setAttribute("class", "fas fa-hammer"); + } + + if (sizes.intact > 0 || sizes.altered > 0) { + buttonDelete.removeAttribute("disabled"); + + if (arg.cacheMode === "offline") { + buttonDownload.setAttribute("disabled", ""); + + if (sizes.altered > 0 || sizes.intact < sizes.total) { + buttonFix.removeAttribute("disabled"); + } + } + } else { + buttonDelete.setAttribute("disabled", ""); + + if (arg.cacheMode === "offline") { + buttonDownload.removeAttribute("disabled"); + buttonFix.setAttribute("disabled", ""); + } + } }); \ No newline at end of file diff --git a/index.js b/index.js index e553241..91af1ce 100644 --- a/index.js +++ b/index.js @@ -125,36 +125,62 @@ app.on("ready", function () { }); ipc.on("download-files", function (event, arg) { + mainWindow.webContents.send("storage-loading-start", { + cacheMode: arg.cacheMode, + versionString: arg.versionString, + resetIntactSize: false, + }); + downloadFiles( - arg.nginxDir, + arg.cdnDir, arg.localDir, - arg.fileRelativePaths, - function (size) { - mainWindow.webContents.send("download-update", { - size: size, + arg.hashes, + function (sizes) { + mainWindow.webContents.send("storage-label-update", { + cacheMode: arg.cacheMode, versionString: arg.versionString, + sizes: sizes, }); }, function () { - mainWindow.webContents.send("download-success", arg.versionString); + mainWindow.webContents.send("download-complete", { + cacheMode: arg.cacheMode, + versionString: arg.versionString, + }); + }, + function (err) { + dialog.showErrorBox("Error!", "Download was unsuccessful:\n" + err); } ); }); ipc.on("hash-check", function (event, arg) { - mainWindow.webContents.send("hash-update", { + mainWindow.webContents.send("storage-loading-start", { cacheMode: arg.cacheMode, versionString: arg.versionString, - // no size sent, reset sizes + resetIntactSize: true, }); - checkHashes(arg.localDir, arg.hashes, function (sizes) { - mainWindow.webContents.send("hash-update", { - cacheMode: arg.cacheMode, - versionString: arg.versionString, - sizes: sizes, - }); - }); + checkHashes( + arg.localDir, + arg.hashes, + function (sizes) { + mainWindow.webContents.send("storage-label-update", { + cacheMode: arg.cacheMode, + versionString: arg.versionString, + sizes: sizes, + }); + }, + function () { + mainWindow.webContents.send("hash-check-complete", { + cacheMode: arg.cacheMode, + versionString: arg.versionString, + }); + }, + function (err) { + dialog.showErrorBox("Error!", "Could not verify file integrity:\n" + err); + } + ); }); }); @@ -207,78 +233,71 @@ function showMainWindow() { }); } -function downloadFile(nginxDir, localDir, relativePath, callback, updateCallback) { - var nginxUrl = path.dirname(nginxDir) + "/" + relativePath; +function downloadFile(cdnDir, localDir, relativePath, fileHash, callback, updateCallback) { + var nginxUrl = path.dirname(cdnDir) + "/" + relativePath; var localPath = path.join(localDir, relativePath); // Create directories if they don't exist var dirName = path.dirname(localPath); - fs.ensureDirSync(dirName); - - // HTTP request to download the file - var fileStream = fs.createWriteStream(localPath); - - var urlParse = url.parse(nginxUrl); - var client = http.createClient(80, urlParse.hostname); - var options = { - method: "GET", - url: urlParse.hostname, - port: 80, - path: urlParse.path, - headers: { - "Host": urlParse.hostname, - "Content-Type": "application/octet-stream", - "Referer": nginxDir, - "Connection": "keep-alive", + fs.ensureDir(dirName, function (createDirErr) { + if (createDirErr) { + console.log("Could not create path " + dirName + ": " + createDirErr); + callback(createDirErr); + return; } - }; - var request = client.request("GET", urlParse.path, options); + // HTTP request to download the file + var fileStream = fs.createWriteStream(localPath); - request.on("response", function(response) { - response.pipe(fileStream); + var urlParse = url.parse(nginxUrl); + var client = http.createClient(80, urlParse.hostname); + var options = { + method: "GET", + url: urlParse.hostname, + port: 80, + path: urlParse.path, + headers: { + "Host": urlParse.hostname, + "Content-Type": "application/octet-stream", + "Referer": cdnDir, + "Connection": "keep-alive", + } + }; - // When the download is complete, invoke the callback - response.on("end", function() { - fileStream.end(); - updateCallback(fs.statSync(localPath).size); - callback(null, relativePath); + var request = client.request("GET", urlParse.path, options); + + request.on("response", function (response) { + response.pipe(fileStream); + + // When the download is complete, invoke the callback + response.on("end", function() { + fileStream.end(); + checkHash(localDir, relativePath, fileHash, callback, updateCallback); + }); + + // Handle errors + response.on("error", callback); }); - // Handle errors - response.on("error", function(err) { - console.error("Error downloading " + relativePath + ": " + err.message); - retryDownload(nginxDir, localDir, relativePath, callback, updateCallback); // Retry download - }); + // Handle HTTP errors + request.on("error", callback); + + request.end(); }); - - // Handle HTTP errors - request.on("error", function(err) { - console.error("Error downloading " + relativePath + ": " + err.message); - retryDownload(nginxDir, localDir, relativePath, callback, updateCallback); // Retry download - }); - - request.end(); -} - - // Function to retry downloading a file after a delay -function retryDownload(nginxDir, localDir, relativePath, callback, updateCallback) { - setTimeout(function() { - downloadFile(nginxDir, localDir, relativePath, callback, updateCallback); - }, 1000); // Retry after 1 second } // Function to download multiple files in parallel -function downloadFiles(nginxDir, localDir, fileRelativePaths, updateCallback, allDoneCallback) { +function downloadFiles(cdnDir, localDir, hashes, updateCallback, allDoneCallback, errorCallback) { async.eachLimit( - fileRelativePaths, + Object.keys(hashes), 5, // Number of parallel downloads function (relativePath, callback) { - downloadFile(nginxDir, localDir, relativePath, callback, updateCallback); + downloadFile(cdnDir, localDir, relativePath, hashes[relativePath], callback, updateCallback); }, function (err) { if (err) { console.error("Download failed: " + err); + errorCallback(err); } else { console.log("All files downloaded successfully."); allDoneCallback(); @@ -297,8 +316,11 @@ function checkHash(localDir, relativePath, fileHash, callback, updateCallback) { fs.open(localPath, "r", function (openErr, file) { if (openErr) { - if (openErr.code !== "ENOENT") { + if (openErr.code === "ENOENT") { + callback(); + } else { console.log("Error opening file for hash check: " + openErr); + callback(openErr); } return; } @@ -310,6 +332,7 @@ function checkHash(localDir, relativePath, fileHash, callback, updateCallback) { updater = function (readErr, readSize) { if (readErr) { console.log("Error reading file for hash check: " + readErr); + callback(readErr); } else if (readSize > 0) { hash.update(buff.slice(0, readSize)); totalCount += readSize; @@ -323,9 +346,10 @@ function checkHash(localDir, relativePath, fileHash, callback, updateCallback) { fs.close(file, function (fileCloseErr) { if (fileCloseErr) { console.log("Error closing file for hash check: " + fileCloseErr); + callback(fileCloseErr); } else { - callback(null, relativePath); updateCallback(sizes); + callback(); } }); } @@ -335,8 +359,7 @@ function checkHash(localDir, relativePath, fileHash, callback, updateCallback) { }); } -function checkHashes(localDir, hashes, updateCallback) { - console.log(hashes); +function checkHashes(localDir, hashes, updateCallback, allDoneCallback, errorCallback) { async.eachLimit( Object.keys(hashes), 20, @@ -346,6 +369,9 @@ function checkHashes(localDir, hashes, updateCallback) { function (err) { if (err) { console.log("Hash check failed: " + err); + errorCallback(err); + } else { + allDoneCallback(); } } );