integrated download and hash functionality

assets/js/server-selector.js

@@ -261,181 +261,6 @@ function getCacheInfoCell(versionString, cacheMode) {
     return cellCache;
 }
 
-function getFileHash(filePath) {
-    var readCount = 0;
-    var buff = new Buffer(chunkSize);
-    var hash = createHash("sha256");
-    var file = remotefs.openSync(filePath, "r");
-
-    while ((readCount = remotefs.readSync(file, buff, 0, chunkSize, null)) > 0) {
-        hash.update(buff.slice(0, readCount));
-    }
-
-    remotefs.closeSync(file);
-    return hash.digest(encoding="hex");
-}
-
-/*
-function downloadFiles(root, client, sizes, hashes, updateCallback, endCallback) {
-    if (hashes.length === 0) {
-        endCallback();
-        return;
-    }
-
-    var filePath = Object.keys(hashes)[0];
-    var fileHash = hashes[filePath];
-    delete hashes[filePath];
-
-    var fullFilePath = path.join(root, filePath);
-    var fullCDNPath = "http://" + [cdn, "ff", "big", filePath].join("/");
-
-    if (remotefs.existsSync(fullFilePath) && fileHash === getFileHash(fullFilePath)) {
-        console.log(fullFilePath + " is intact, skipping...");
-        return;
-    }
-
-    downloadFile(client, fullCDNPath, fullFilePath, function () {
-        var sizeRead = remotefs.statSync(fullFilePath).size;
-
-        if (fileHash === getFileHash(fullFilePath)) {
-            sizes.intact += sizeRead;
-        } else {
-            sizes.altered += sizeRead;
-        }
-
-        console.log(fullFilePath + " was downloaded from " + fullCDNPath);
-
-        setTimeout(function () {
-            updateCallback(sizes);
-            downloadFiles(root, client, sizes, hashes, updateCallback, endCallback);
-        }, 100);
-    });
-}
-
-function downloadFile(client, fullCDNPath, fullFilePath, callback) {
-    remotefs.ensureDirSync(path.dirname(fullFilePath));
-
-    var urlParts = url.parse(fullCDNPath);
-    var req = client.request("GET", urlParts.path, {
-        "Host": urlParts.hostname,
-        "Content-Type": "application/octet-stream",
-        "Referer": fullCDNPath.split("/").slice(0, 4).join("/") + "/",
-        "Connection": "keep-alive",
-    });
-    var writeStream = remotefs.createWriteStream(fullFilePath);
-
-    var retry = function (err) {
-        writeStream.end();
-        writeStream.destroy();
-
-        remotefs.removeSync(fullFilePath);
-
-        console.log("Error writing file " + fullFilePath + "\n" + err);
-
-        setTimeout(function () {
-            console.log("Retrying " + fullCDNPath);
-            downloadFile(client, fullCDNPath, fullFilePath, callback);
-        }, 1000);
-    }
-
-    writeStream.on("error", retry);
-
-    req.on("response", function (res) {
-        if (res.statusCode >= 300) {
-            console.log("Error in fetching file " + fullFilePath + " from " + fullCDNPath);
-            retry("Status Code: " + res.statusCode);
-            return;
-        }
-
-        res.pipe(writeStream);
-
-        res.on("end", callback);
-        res.on("error", retry);
-    });
-
-    req.on("error", retry);
-
-    req.end();
-}
-*/
-
-
-function downloadFile(nginxDir, localDir, relativePath, callback) {
-    var nginxUrl = path.dirname(nginxDir) + "/" + relativePath;
-    var localPath = path.join(localDir, relativePath);
-
-    // Create directories if they don't exist
-    var dirName = path.dirname(localPath);
-    remotefs.ensureDirSync(dirName);
-
-    // HTTP request to download the file
-    var fileStream = remotefs.createWriteStream(localPath);
-    var urlParse = url.parse(nginxUrl);
-    var client = http.createClient(80, urlParse.hostname);
-    var options = {
-        url: urlParse.hostname,
-        port: 80,
-        path: urlParse.path,
-        headers: {
-            "Host": urlParse.hostname,
-            "Content-Type": "application/octet-stream",
-            "Referer": nginxDir,
-            "Connection": "keep-alive",
-        }
-    };
-
-    var request = client.request("GET", urlParse.path, options, function(response) {
-        // Handle errors
-        response.on("error", function(err) {
-            console.error("Error downloading " + relativePath + ": " + err.message);
-            retryDownload(nginxDir, localDir, relativePath, callback); // Retry download
-        });
-
-        response.pipe(fileStream);
-
-        // When the download is complete, invoke the callback
-        response.on("end", function() {
-            fileStream.end();
-            callback(null, relativePath);
-        });
-    });
-
-    // Handle HTTP errors
-    request.on("error", function(err) {
-        console.error("Error downloading " + relativePath + ": " + err.message);
-        retryDownload(nginxDir, localDir, relativePath, callback); // Retry download
-    });
-
-    request.end();
-}
-
-  // Function to retry downloading a file after a delay
-function retryDownload(nginxDir, localDir, relativePath, callback) {
-    setTimeout(function() {
-        downloadFile(nginxDir, localDir, relativePath, callback);
-    }, 1000); // Retry after 1 second
-}
-
-  // Function to download multiple files in parallel
-function downloadFiles(nginxDir, localDir, fileRelativePaths, allDoneCallback) {
-    async.eachLimit(
-        fileRelativePaths,
-        5, // Number of parallel downloads
-        function(relativePath, callback) {
-            downloadFile(nginxDir, localDir, relativePath, callback);
-        },
-        function(err) {
-            if (err) {
-                console.error("Download failed: " + err);
-            } else {
-                console.log("All files downloaded successfully.");
-                allDoneCallback();
-            }
-        }
-    );
-}
-
-
 function loadCacheList() {
     var versionjson = remotefs.readJsonSync(versionsPath);
     versionArray = versionjson["versions"];
@@ -516,72 +341,33 @@ function deletePlayableCache(versionString) {
     resetCacheNames();
 
     if (versionString === "Offline") {
+        console.log("Cannot delete Offline directory!");
         return;
     }
 
     remotefs.removeSync(path.join(cacheRoot, versionString));
     console.log("Playable cache " + versionString + " has been removed!");
 
-    checkPlayableCache(versionString);
+    // this updates the labels etc. properly
+    ipc.send("hash-check", {
+        localDir: cacheRoot,
+        cacheMode: "playable",
+        versionString: versionString,
+        hashes: versionHashes.playable[versionString],
+    });
 }
 
 function downloadOfflineCache(versionString) {
-    var buttonDownload = document.getElementById(getCacheButtonID(versionString, "offline", "download"));
-    var buttonFix = document.getElementById(getCacheButtonID(versionString, "offline", "fix"));
-    var buttonDelete = document.getElementById(getCacheButtonID(versionString, "offline", "delete"));
-    var label = document.getElementById(getCacheElemID(versionString, "offline", "label"));
-
     var version = versionArray.filter(function (value) {
         return value.name === versionString;
     })[0];
-    var sizes = {
-        intact: 0,
-        altered: 0,
-        total: version.offline_size
-    };
 
-    buttonDownload.setAttribute("disabled", "");
+    // download-files will run hash-check after download
-    buttonFix.setAttribute("disabled", "");
-    buttonDelete.setAttribute("disabled", "");
-
-    buttonDownload.children[0].setAttribute("class", "fas fa-spinner fa-spin fa-fw");
-    buttonFix.children[0].setAttribute("class", "fas fa-spinner fa-spin fa-fw");
-
-    /*
-    var client = http.createClient(80, cdn);
-
-    downloadFiles(
-        offlineRoot,
-        client,
-        sizes,
-        JSON.parse(JSON.stringify(versionHashes.offline[versionString])),
-        function (sizesNow) {
-            label.innerHTML = getCacheLabelText(sizesNow);
-        },
-        function () {
-            buttonDownload.children[0].setAttribute("class", "fas fa-download");
-            buttonFix.children[0].setAttribute("class", "fas fa-hammer");
-            checkOfflineCache(versionString);
-        }
-    );
-    */
-
-    /*
-    downloadFiles(
-        version.url,
-        offlineRoot,
-        Object.keys(JSON.parse(JSON.stringify(versionHashes.offline[versionString]))),
-        function () {
-            buttonDownload.children[0].setAttribute("class", "fas fa-download");
-            buttonFix.children[0].setAttribute("class", "fas fa-hammer");
-            checkOfflineCache(versionString);
-        }
-    );
-    */
     ipc.send("download-files", {
-        nginxDir: version.url,
+        cdnDir: version.url,
         localDir: offlineRoot,
-        fileRelativePaths: Object.keys(versionHashes.offline[versionString]),
+        hashes: versionHashes.offline[versionString],
+        cacheMode: "offline",
         versionString: versionString,
     });
 }
@@ -590,77 +376,13 @@ function deleteOfflineCache(versionString) {
     remotefs.removeSync(path.join(offlineRoot, versionString));
     console.log("Offline cache " + versionString + " has been removed!");
 
-    checkOfflineCache(versionString);
+    // this updates the labels etc. properly
-}
+    ipc.send("hash-check", {
-
+        localDir: offlineRoot,
-function checkPlayableCache(versionString) {
+        cacheMode: "offline",
-    var button = document.getElementById(getCacheButtonID(versionString, "playable", "delete"));
+        versionString: versionString,
-    var label = document.getElementById(getCacheElemID(versionString, "playable", "label"));
+        hashes: versionHashes.offline[versionString],
-
-    var sizes = versionSizes.playable[versionString];
-
-    resetCacheNames();
-
-    $.each(versionHashes.playable[versionString], function (filePath, fileHash) {
-        var fullFilePath = path.join(cacheRoot, filePath);
-
-        if (remotefs.existsSync(fullFilePath)) {
-            var fileSize = remotefs.statSync(fullFilePath).size;
-
-            if (fileHash === getFileHash(fullFilePath)) {
-                sizes.intact += fileSize;
-            } else {
-                sizes.altered += fileSize;
-            }
-        }
     });
-
-    if (sizes.intact > 0 || sizes.altered > 0) {
-        button.removeAttribute("disabled");
-    } else {
-        button.setAttribute("disabled", "");
-    }
-
-    label.innerHTML = getCacheLabelText(sizes);
-}
-
-function checkOfflineCache(versionString) {
-    var buttonDownload = document.getElementById(getCacheButtonID(versionString, "offline", "download"));
-    var buttonFix = document.getElementById(getCacheButtonID(versionString, "offline", "fix"));
-    var buttonDelete = document.getElementById(getCacheButtonID(versionString, "offline", "delete"));
-    var label = document.getElementById(getCacheElemID(versionString, "offline", "label"));
-
-    var sizes = versionSizes.offline[versionString];
-
-    $.each(versionHashes.offline[versionString], function (filePath, fileHash) {
-        var fullFilePath = path.join(offlineRoot, filePath);
-
-        if (remotefs.existsSync(fullFilePath)) {
-            var fileSize = remotefs.statSync(fullFilePath).size;
-
-            if (fileHash === getFileHash(fullFilePath)) {
-                sizes.intact += fileSize;
-            } else {
-                sizes.altered += fileSize;
-            }
-        }
-    });
-
-    if (sizes.intact > 0 || sizes.altered > 0) {
-        buttonDownload.setAttribute("disabled", "");
-        buttonDelete.removeAttribute("disabled");
-    } else {
-        buttonDownload.removeAttribute("disabled");
-        buttonDelete.setAttribute("disabled", "");
-    }
-
-    if (sizes.altered > 0) {
-        buttonFix.removeAttribute("disabled");
-    } else {
-        buttonFix.setAttribute("disabled", "");
-    }
-
-    label.innerHTML = getCacheLabelText(sizes);
 }
 
 function resetCacheNames() {
@@ -876,34 +598,81 @@ $("#of-deleteservermodal").on("show.bs.modal", function (e) {
     $("#deleteserver-servername").html(result.description);
 });
 
-ipc.on("download-update", function (arg) {
+ipc.on("storage-loading-start", function (arg) {
-    var sizes = versionSizes.offline[arg.versionString];
+    var sizes = versionSizes[arg.cacheMode][arg.versionString];
-    sizes.intact += arg.size;
+
+    if (arg.resetIntactSize) {
+        sizes.intact = 0;
+    }
+    sizes.altered = 0;
+
+    var buttonDelete = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "delete"));
+    var buttonDownload = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "download"));
+    var buttonFix = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "fix"));
+    var label = document.getElementById(getCacheElemID(arg.versionString, arg.cacheMode, "label"));
+
+    buttonDelete.setAttribute("disabled", "");
+    buttonDelete.children[0].setAttribute("class", "fas fa-spinner fa-spin fa-fw");
+
+    if (arg.cacheMode === "offline") {
+        buttonDownload.setAttribute("disabled", "");
+        buttonDownload.children[0].setAttribute("class", "fas fa-spinner fa-spin fa-fw");
+
+        buttonFix.setAttribute("disabled", "");
+        buttonFix.children[0].setAttribute("class", "fas fa-spinner fa-spin fa-fw");
+    }
 
-    var label = document.getElementById(getCacheElemID(arg.versionString, "offline", "label"));
     label.innerHTML = getCacheLabelText(sizes);
 });
 
-ipc.on("download-success", function (versionString) {
+ipc.on("storage-label-update", function (arg) {
-    var buttonDownload = document.getElementById(getCacheButtonID(versionString, "offline", "download"));
-    var buttonFix = document.getElementById(getCacheButtonID(versionString, "offline", "fix"));
-
-    buttonDownload.children[0].setAttribute("class", "fas fa-download");
-    buttonFix.children[0].setAttribute("class", "fas fa-hammer");
-    checkOfflineCache(versionString);
-});
-
-ipc.on("hash-update", function (arg) {
     var sizes = versionSizes[arg.cacheMode][arg.versionString];
-
-    if (arg.sizes) {
     sizes.intact += arg.sizes.intact;
     sizes.altered += arg.sizes.altered;
-    } else {
-        sizes.intact = 0;
-        sizes.altered = 0;
-    }
 
     var label = document.getElementById(getCacheElemID(arg.versionString, arg.cacheMode, "label"));
     label.innerHTML = getCacheLabelText(sizes);
 });
+
+ipc.on("download-complete", function (arg) {
+    ipc.send("hash-check", {
+        localDir: (arg.cacheMode === "offline") ? offlineRoot : cacheRoot,
+        cacheMode: arg.cacheMode,
+        versionString: arg.versionString,
+        hashes: versionHashes[arg.cacheMode][arg.versionString],
+    });
+});
+
+ipc.on("hash-check-complete", function (arg) {
+    var sizes = versionSizes[arg.cacheMode][arg.versionString];
+
+    var buttonDelete = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "delete"));
+    var buttonDownload = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "download"));
+    var buttonFix = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "fix"));
+
+    buttonDelete.children[0].setAttribute("class", "fas fa-trash-alt");
+
+    if (arg.cacheMode === "offline") {
+        buttonDownload.children[0].setAttribute("class", "fas fa-download");
+        buttonFix.children[0].setAttribute("class", "fas fa-hammer");
+    }
+
+    if (sizes.intact > 0 || sizes.altered > 0) {
+        buttonDelete.removeAttribute("disabled");
+
+        if (arg.cacheMode === "offline") {
+            buttonDownload.setAttribute("disabled", "");
+
+            if (sizes.altered > 0 || sizes.intact < sizes.total) {
+                buttonFix.removeAttribute("disabled");
+            }
+        }
+    } else {
+        buttonDelete.setAttribute("disabled", "");
+
+        if (arg.cacheMode === "offline") {
+            buttonDownload.removeAttribute("disabled");
+            buttonFix.setAttribute("disabled", "");
+        }
+    }
+});

index.js

@@ -125,36 +125,62 @@ app.on("ready", function () {
     });
 
     ipc.on("download-files", function (event, arg) {
-        downloadFiles(
+        mainWindow.webContents.send("storage-loading-start", {
-            arg.nginxDir,
+            cacheMode: arg.cacheMode,
-            arg.localDir,
-            arg.fileRelativePaths,
-            function (size) {
-                mainWindow.webContents.send("download-update", {
-                    size: size,
             versionString: arg.versionString,
+            resetIntactSize: false,
+        });
+
+        downloadFiles(
+            arg.cdnDir,
+            arg.localDir,
+            arg.hashes,
+            function (sizes) {
+                mainWindow.webContents.send("storage-label-update", {
+                    cacheMode: arg.cacheMode,
+                    versionString: arg.versionString,
+                    sizes: sizes,
                 });
             },
             function () {
-                mainWindow.webContents.send("download-success", arg.versionString);
+                mainWindow.webContents.send("download-complete", {
+                    cacheMode: arg.cacheMode,
+                    versionString: arg.versionString,
+                });
+            },
+            function (err) {
+                dialog.showErrorBox("Error!", "Download was unsuccessful:\n" + err);
             }
         );
     });
 
     ipc.on("hash-check", function (event, arg) {
-        mainWindow.webContents.send("hash-update", {
+        mainWindow.webContents.send("storage-loading-start", {
             cacheMode: arg.cacheMode,
             versionString: arg.versionString,
-            // no size sent, reset sizes
+            resetIntactSize: true,
         });
 
-        checkHashes(arg.localDir, arg.hashes, function (sizes) {
+        checkHashes(
-            mainWindow.webContents.send("hash-update", {
+            arg.localDir,
+            arg.hashes,
+            function (sizes) {
+                mainWindow.webContents.send("storage-label-update", {
                     cacheMode: arg.cacheMode,
                     versionString: arg.versionString,
                     sizes: sizes,
                 });
+            },
+            function () {
+                mainWindow.webContents.send("hash-check-complete", {
+                    cacheMode: arg.cacheMode,
+                    versionString: arg.versionString,
                 });
+            },
+            function (err) {
+                dialog.showErrorBox("Error!", "Could not verify file integrity:\n" + err);
+            }
+        );
     });
 });
 
@@ -207,13 +233,18 @@ function showMainWindow() {
     });
 }
 
-function downloadFile(nginxDir, localDir, relativePath, callback, updateCallback) {
+function downloadFile(cdnDir, localDir, relativePath, fileHash, callback, updateCallback) {
-    var nginxUrl = path.dirname(nginxDir) + "/" + relativePath;
+    var nginxUrl = path.dirname(cdnDir) + "/" + relativePath;
     var localPath = path.join(localDir, relativePath);
 
     // Create directories if they don't exist
     var dirName = path.dirname(localPath);
-    fs.ensureDirSync(dirName);
+    fs.ensureDir(dirName, function (createDirErr) {
+        if (createDirErr) {
+            console.log("Could not create path " + dirName + ": " + createDirErr);
+            callback(createDirErr);
+            return;
+        }
 
         // HTTP request to download the file
         var fileStream = fs.createWriteStream(localPath);
@@ -228,7 +259,7 @@ function downloadFile(nginxDir, localDir, relativePath, callback, updateCallback
             headers: {
                 "Host": urlParse.hostname,
                 "Content-Type": "application/octet-stream",
-            "Referer": nginxDir,
+                "Referer": cdnDir,
                 "Connection": "keep-alive",
             }
         };
@@ -241,44 +272,32 @@ function downloadFile(nginxDir, localDir, relativePath, callback, updateCallback
             // When the download is complete, invoke the callback
             response.on("end", function() {
                 fileStream.end();
-            updateCallback(fs.statSync(localPath).size);
+                checkHash(localDir, relativePath, fileHash, callback, updateCallback);
-            callback(null, relativePath);
             });
 
             // Handle errors
-        response.on("error", function(err) {
+            response.on("error", callback);
-            console.error("Error downloading " + relativePath + ": " + err.message);
-            retryDownload(nginxDir, localDir, relativePath, callback, updateCallback); // Retry download
-        });
         });
 
         // Handle HTTP errors
-    request.on("error", function(err) {
+        request.on("error", callback);
-        console.error("Error downloading " + relativePath + ": " + err.message);
-        retryDownload(nginxDir, localDir, relativePath, callback, updateCallback); // Retry download
-    });
 
         request.end();
-}
+    });
-
-  // Function to retry downloading a file after a delay
-function retryDownload(nginxDir, localDir, relativePath, callback, updateCallback) {
-    setTimeout(function() {
-        downloadFile(nginxDir, localDir, relativePath, callback, updateCallback);
-    }, 1000); // Retry after 1 second
 }
 
   // Function to download multiple files in parallel
-function downloadFiles(nginxDir, localDir, fileRelativePaths, updateCallback, allDoneCallback) {
+function downloadFiles(cdnDir, localDir, hashes, updateCallback, allDoneCallback, errorCallback) {
     async.eachLimit(
-        fileRelativePaths,
+        Object.keys(hashes),
         5, // Number of parallel downloads
         function (relativePath, callback) {
-            downloadFile(nginxDir, localDir, relativePath, callback, updateCallback);
+            downloadFile(cdnDir, localDir, relativePath, hashes[relativePath], callback, updateCallback);
         },
         function (err) {
             if (err) {
                 console.error("Download failed: " + err);
+                errorCallback(err);
             } else {
                 console.log("All files downloaded successfully.");
                 allDoneCallback();
@@ -297,8 +316,11 @@ function checkHash(localDir, relativePath, fileHash, callback, updateCallback) {
 
     fs.open(localPath, "r", function (openErr, file) {
         if (openErr) {
-            if (openErr.code !== "ENOENT") {
+            if (openErr.code === "ENOENT") {
+                callback();
+            } else {
                 console.log("Error opening file for hash check: " + openErr);
+                callback(openErr);
             }
             return;
         }
@@ -310,6 +332,7 @@ function checkHash(localDir, relativePath, fileHash, callback, updateCallback) {
         updater = function (readErr, readSize) {
             if (readErr) {
                 console.log("Error reading file for hash check: " + readErr);
+                callback(readErr);
             } else if (readSize > 0) {
                 hash.update(buff.slice(0, readSize));
                 totalCount += readSize;
@@ -323,9 +346,10 @@ function checkHash(localDir, relativePath, fileHash, callback, updateCallback) {
                 fs.close(file, function (fileCloseErr) {
                     if (fileCloseErr) {
                         console.log("Error closing file for hash check: " + fileCloseErr);
+                        callback(fileCloseErr);
                     } else {
-                        callback(null, relativePath);
                         updateCallback(sizes);
+                        callback();
                     }
                 });
             }
@@ -335,8 +359,7 @@ function checkHash(localDir, relativePath, fileHash, callback, updateCallback) {
     });
 }
 
-function checkHashes(localDir, hashes, updateCallback) {
+function checkHashes(localDir, hashes, updateCallback, allDoneCallback, errorCallback) {
-    console.log(hashes);
     async.eachLimit(
         Object.keys(hashes),
         20,
@@ -346,6 +369,9 @@ function checkHashes(localDir, hashes, updateCallback) {
         function (err) {
             if (err) {
                 console.log("Hash check failed: " + err);
+                errorCallback(err);
+            } else {
+                allDoneCallback();
             }
         }
     );