adjusted download and hash check logic and data

assets/js/server-selector.js

@@ -3,29 +3,18 @@ var remote = require("remote");
 var remotefs = remote.require("fs-extra");
 var dns = remote.require("dns");
 var path = remote.require("path");
-var url = remote.require("url");
-var http = remote.require("http");
-var createHash = remote.require("crypto").createHash;
-var async = remote.require("async");
 
 var userData = remote.require("app").getPath("userData");
 var configPath = path.join(userData, "config.json");
 var serversPath = path.join(userData, "servers.json");
 var versionsPath = path.join(userData, "versions.json");
-var hashPath = path.join(userData, "hash.txt");
 var cacheRoot = path.join(
     userData,
     "/../../LocalLow/Unity/Web Player/Cache"
 );
 var offlineRoot = path.join(cacheRoot, "Offline");
 
-var chunkSize = 1 << 16;
-var gb = 1 << 30;
-var cdn = "cdn.dexlabs.systems";
-
 var versionArray;
-var versionHashes;
-var versionSizes;
 var serverArray;
 var config;
 
@@ -186,6 +175,7 @@ function getCacheButtonID(versionString, cacheMode, buttonMode) {
 }
 
 function getCacheLabelText(sizes) {
+    var gb = 1 << 30;
     var labelText = (sizes.intact / gb).toFixed(2) + " / " + (sizes.total / gb).toFixed(2) + " GB";
 
     if (sizes.altered > 0) {
@@ -261,50 +251,23 @@ function getCacheInfoCell(versionString, cacheMode) {
     return cellCache;
 }
 
+function startHashCheck(versionString, cacheMode) {
+    ipc.send("hash-check", {
+        localDir: (cacheMode === "offline") ? offlineRoot : cacheRoot,
+        cacheMode: cacheMode,
+        versionString: versionString,
+    });
+}
+
 function loadCacheList() {
+    resetCacheNames();
+
     var versionjson = remotefs.readJsonSync(versionsPath);
     versionArray = versionjson["versions"];
 
-    versionHashes = { playable: {}, offline: {} };
-    var hashlines = remotefs.readFileSync(hashPath, ( encoding = "utf-8" ));
-    $.each(hashlines.split(/\r\n|\r|\n/), function (key, line) {
-        if (line.length === 0) {
-            return;
-        }
-
-        var linearr = line.split(" ", 2);
-        var fileHash = linearr[0];
-        var filePath = linearr[1].substr(3);
-        var pathArray = filePath.split("/");
-        var hashDict = (pathArray[0] === "Offline") ?
-            versionHashes.offline :
-            versionHashes.playable;
-        var versionString = (pathArray[0] === "Offline") ?
-            pathArray[1] :
-            pathArray[0];
-
-        if (!hashDict.hasOwnProperty(versionString)) {
-            hashDict[versionString] = {};
-        }
-
-        hashDict[versionString][filePath.replace("Offline/", "")] = fileHash;
-    });
-
     $(".cache-listing-entry").remove();
 
-    versionSizes = { playable: {}, offline: {} };
     $.each(versionArray, function (key, value) {
-        versionSizes.playable[value.name] = {
-            intact: 0,
-            altered: 0,
-            total: value.playable_size,
-        };
-        versionSizes.offline[value.name] = {
-            intact: 0,
-            altered: 0,
-            total: value.offline_size,
-        };
-
         var row = document.createElement("tr");
         row.className = "cache-listing-entry"
         row.setAttribute("id", value.name);
@@ -322,39 +285,27 @@ function loadCacheList() {
 
         $("#cache-tablebody").append(row);
 
-        ipc.send("hash-check", {
+        setTimeout(function () {
-            localDir: cacheRoot,
+            startHashCheck(value.name, "playable");
-            cacheMode: "playable",
+            startHashCheck(value.name, "offline");
-            versionString: value.name,
+        }, 0);
-            hashes: versionHashes.playable[value.name],
-        });
-        ipc.send("hash-check", {
-            localDir: offlineRoot,
-            cacheMode: "offline",
-            versionString: value.name,
-            hashes: versionHashes.offline[value.name],
-        });
     })
 }
 
 function deletePlayableCache(versionString) {
-    resetCacheNames();
-
     if (versionString === "Offline") {
         console.log("Cannot delete Offline directory!");
         return;
     }
 
+    // TODO: remove this function
+    resetCacheNames();
+
     remotefs.removeSync(path.join(cacheRoot, versionString));
     console.log("Playable cache " + versionString + " has been removed!");
 
     // this updates the labels etc. properly
-    ipc.send("hash-check", {
+    startHashCheck(versionString, "playable");
-        localDir: cacheRoot,
-        cacheMode: "playable",
-        versionString: versionString,
-        hashes: versionHashes.playable[versionString],
-    });
 }
 
 function downloadOfflineCache(versionString) {
@@ -366,7 +317,6 @@ function downloadOfflineCache(versionString) {
     ipc.send("download-files", {
         cdnDir: version.url,
         localDir: offlineRoot,
-        hashes: versionHashes.offline[versionString],
         cacheMode: "offline",
         versionString: versionString,
     });
@@ -377,12 +327,7 @@ function deleteOfflineCache(versionString) {
     console.log("Offline cache " + versionString + " has been removed!");
 
     // this updates the labels etc. properly
-    ipc.send("hash-check", {
+    startHashCheck(versionString, "offline");
-        localDir: offlineRoot,
-        cacheMode: "offline",
-        versionString: versionString,
-        hashes: versionHashes.offline[versionString],
-    });
 }
 
 function resetCacheNames() {
@@ -599,13 +544,6 @@ $("#of-deleteservermodal").on("show.bs.modal", function (e) {
 });
 
 ipc.on("storage-loading-start", function (arg) {
-    var sizes = versionSizes[arg.cacheMode][arg.versionString];
-
-    if (arg.resetIntactSize) {
-        sizes.intact = 0;
-    }
-    sizes.altered = 0;
-
     var buttonDelete = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "delete"));
     var buttonDownload = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "download"));
     var buttonFix = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "fix"));
@@ -622,30 +560,15 @@ ipc.on("storage-loading-start", function (arg) {
         buttonFix.children[0].setAttribute("class", "fas fa-spinner fa-spin fa-fw");
     }
 
-    label.innerHTML = getCacheLabelText(sizes);
+    label.innerHTML = getCacheLabelText(arg.sizes);
 });
 
 ipc.on("storage-label-update", function (arg) {
-    var sizes = versionSizes[arg.cacheMode][arg.versionString];
-    sizes.intact += arg.sizes.intact;
-    sizes.altered += arg.sizes.altered;
-
     var label = document.getElementById(getCacheElemID(arg.versionString, arg.cacheMode, "label"));
-    label.innerHTML = getCacheLabelText(sizes);
+    label.innerHTML = getCacheLabelText(arg.sizes);
 });
 
-ipc.on("download-complete", function (arg) {
+ipc.on("storage-loading-complete", function (arg) {
-    ipc.send("hash-check", {
-        localDir: (arg.cacheMode === "offline") ? offlineRoot : cacheRoot,
-        cacheMode: arg.cacheMode,
-        versionString: arg.versionString,
-        hashes: versionHashes[arg.cacheMode][arg.versionString],
-    });
-});
-
-ipc.on("hash-check-complete", function (arg) {
-    var sizes = versionSizes[arg.cacheMode][arg.versionString];
-
     var buttonDelete = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "delete"));
     var buttonDownload = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "download"));
     var buttonFix = document.getElementById(getCacheButtonID(arg.versionString, arg.cacheMode, "fix"));
@@ -657,13 +580,13 @@ ipc.on("hash-check-complete", function (arg) {
         buttonFix.children[0].setAttribute("class", "fas fa-hammer");
     }
 
-    if (sizes.intact > 0 || sizes.altered > 0) {
+    if (arg.sizes.intact > 0 || arg.sizes.altered > 0) {
         buttonDelete.removeAttribute("disabled");
 
         if (arg.cacheMode === "offline") {
             buttonDownload.setAttribute("disabled", "");
 
-            if (sizes.altered > 0 || sizes.intact < sizes.total) {
+            if (arg.sizes.altered > 0 || arg.sizes.intact < arg.sizes.total) {
                 buttonFix.removeAttribute("disabled");
             }
         }

index.js

@@ -33,7 +33,10 @@ var userData = app.getPath("userData");
 var configPath = path.join(userData, "config.json");
 var serversPath = path.join(userData, "servers.json");
 var versionsPath = path.join(userData, "versions.json");
-var hashPath = path.join(userData, "hash.txt");
+var hashPath = path.join(userData, "hashes.json");
+
+var versionHashes;
+var versionSizes;
 
 function initialSetup(firstTime) {
     if (!firstTime) {
@@ -55,7 +58,7 @@ function initialSetup(firstTime) {
     // Copy default versions and config
     fs.copySync(path.join(__dirname, "/defaults/versions.json"), versionsPath);
     fs.copySync(path.join(__dirname, "/defaults/config.json"), configPath);
-    fs.copySync(path.join(__dirname, "/defaults/hash.txt"), hashPath);
+    fs.copySync(path.join(__dirname, "/defaults/hashes.json"), hashPath);
 
     console.log("JSON files copied.");
     showMainWindow();
@@ -71,6 +74,26 @@ app.on("window-all-closed", function () {
 });
 
 app.on("ready", function () {
+    versionHashes = fs.readJsonSync(hashPath);
+
+    versionSizes = {}
+    Object.keys(versionHashes).forEach(function (versionString) {
+        var value = versionHashes[versionString];
+
+        versionSizes[versionString] = {
+            playable: {
+                intact: 0,
+                altered: 0,
+                total: value.playable_size,
+            },
+            offline: {
+                intact: 0,
+                altered: 0,
+                total: value.offline_size,
+            },
+        };
+    });
+
     // Check just in case the user forgot to extract the zip.
     zipCheck = app.getPath("exe").includes(os.tmpdir());
     if (zipCheck) {
@@ -125,27 +148,35 @@ app.on("ready", function () {
     });
 
     ipc.on("download-files", function (event, arg) {
+        var currentSizes = versionSizes[arg.versionString][arg.cacheMode];
+        currentSizes.intact = 0;
+        currentSizes.altered = 0;
+
         mainWindow.webContents.send("storage-loading-start", {
             cacheMode: arg.cacheMode,
             versionString: arg.versionString,
-            resetIntactSize: false,
+            sizes: currentSizes,
         });
 
         downloadFiles(
             arg.cdnDir,
-            arg.localDir,
+            path.join(arg.localDir, arg.versionString),
-            arg.hashes,
+            versionHashes[arg.versionString][arg.cacheMode],
             function (sizes) {
+                currentSizes.intact += sizes.intact;
+                currentSizes.altered += sizes.altered;
+
                 mainWindow.webContents.send("storage-label-update", {
                     cacheMode: arg.cacheMode,
                     versionString: arg.versionString,
-                    sizes: sizes,
+                    sizes: currentSizes,
                 });
             },
             function () {
-                mainWindow.webContents.send("download-complete", {
+                mainWindow.webContents.send("storage-loading-complete", {
                     cacheMode: arg.cacheMode,
                     versionString: arg.versionString,
+                    sizes: currentSizes,
                 });
             },
             function (err) {
@@ -155,26 +186,34 @@ app.on("ready", function () {
     });
 
     ipc.on("hash-check", function (event, arg) {
+        var currentSizes = versionSizes[arg.versionString][arg.cacheMode];
+        currentSizes.intact = 0;
+        currentSizes.altered = 0;
+
         mainWindow.webContents.send("storage-loading-start", {
             cacheMode: arg.cacheMode,
             versionString: arg.versionString,
-            resetIntactSize: true,
+            sizes: currentSizes,
         });
 
         checkHashes(
-            arg.localDir,
+            path.join(arg.localDir, arg.versionString),
-            arg.hashes,
+            versionHashes[arg.versionString][arg.cacheMode],
             function (sizes) {
+                currentSizes.intact += sizes.intact;
+                currentSizes.altered += sizes.altered;
+
                 mainWindow.webContents.send("storage-label-update", {
                     cacheMode: arg.cacheMode,
                     versionString: arg.versionString,
-                    sizes: sizes,
+                    sizes: currentSizes,
                 });
             },
             function () {
-                mainWindow.webContents.send("hash-check-complete", {
+                mainWindow.webContents.send("storage-loading-complete", {
                     cacheMode: arg.cacheMode,
                     versionString: arg.versionString,
+                    sizes: currentSizes,
                 });
             },
             function (err) {
@@ -234,18 +273,12 @@ function showMainWindow() {
 }
 
 function downloadFile(cdnDir, localDir, relativePath, fileHash, callback, updateCallback) {
-    var nginxUrl = path.dirname(cdnDir) + "/" + relativePath;
+    var nginxUrl = cdnDir + "/" + relativePath;
     var localPath = path.join(localDir, relativePath);
-
-    // Create directories if they don't exist
     var dirName = path.dirname(localPath);
-    fs.ensureDir(dirName, function (createDirErr) {
-        if (createDirErr) {
-            console.log("Could not create path " + dirName + ": " + createDirErr);
-            callback(createDirErr);
-            return;
-        }
 
+    // define the download function
+    var downloader = function () {
         // HTTP request to download the file
         var fileStream = fs.createWriteStream(localPath);
 
@@ -272,6 +305,8 @@ function downloadFile(cdnDir, localDir, relativePath, fileHash, callback, update
             // When the download is complete, invoke the callback
             response.on("end", function() {
                 fileStream.end();
+                fileStream.destroy();
+                // Don't fail on altered download results, just report on it
                 checkHash(localDir, relativePath, fileHash, callback, updateCallback);
             });
 
@@ -283,6 +318,40 @@ function downloadFile(cdnDir, localDir, relativePath, fileHash, callback, update
         request.on("error", callback);
 
         request.end();
+    };
+
+    // Create directories if they don't exist
+    fs.ensureDir(dirName, function (createDirErr) {
+        if (createDirErr) {
+            console.log("Could not create path " + dirName + ": " + createDirErr);
+            callback(createDirErr);
+            return;
+        }
+
+        // start with the initial file check, call downloader if necessary
+        checkHash(
+            localDir,
+            relativePath,
+            fileHash,
+            function (err) {
+                if (err) {
+                    if (err.code === "ENOENT") {
+                        downloader();
+                    } else {
+                        callback(err);
+                    }
+                }
+                // allow the happy-path to continue
+            },
+            function (sizes) {
+                if (sizes.intact === 0) {
+                    downloader();
+                } else {
+                    updateCallback(sizes);
+                    callback();
+                }
+            }
+        );
     });
 }
 
@@ -296,7 +365,7 @@ function downloadFiles(cdnDir, localDir, hashes, updateCallback, allDoneCallback
         },
         function (err) {
             if (err) {
-                console.error("Download failed: " + err);
+                console.log("Download failed: " + err);
                 errorCallback(err);
             } else {
                 console.log("All files downloaded successfully.");
@@ -306,56 +375,33 @@ function downloadFiles(cdnDir, localDir, hashes, updateCallback, allDoneCallback
     );
 }
 
-function checkHash(localDir, relativePath, fileHash, callback, updateCallback) {
+function checkHash(localDir, relativePath, fileHash, callback, updateCallback, skipMissing) {
     var localPath = path.join(localDir, relativePath);
 
     var chunkSize = 1 << 16;
     var totalCount = 0;
-    var buff = new Buffer(chunkSize);
     var hash = createHash("sha256");
 
-    fs.open(localPath, "r", function (openErr, file) {
+    var fileStream = fs.createReadStream(localPath, { bufferSize: chunkSize });
-        if (openErr) {
-            if (openErr.code === "ENOENT") {
-                callback();
-            } else {
-                console.log("Error opening file for hash check: " + openErr);
-                callback(openErr);
-            }
-            return;
-        }
 
-        var updater;
+    fileStream.on("error", function (err) {
-        var reader = function () {
+        callback((skipMissing && err.code === "ENOENT") ? null : err);
-            fs.read(file, buff, 0, chunkSize, null, updater);
+    });
-        };
-        updater = function (readErr, readSize) {
-            if (readErr) {
-                console.log("Error reading file for hash check: " + readErr);
-                callback(readErr);
-            } else if (readSize > 0) {
-                hash.update(buff.slice(0, readSize));
-                totalCount += readSize;
 
-                reader();
+    fileStream.on("data", function (data) {
-            } else {
+        hash.update(data);
-                var state = (fileHash === hash.digest(encoding="hex")) ? "intact" : "altered";
+        totalCount += data.length;
-                var sizes = { intact: 0, altered: 0 };
+    });
-                sizes[state] = totalCount;
 
-                fs.close(file, function (fileCloseErr) {
+    fileStream.on("end", function () {
-                    if (fileCloseErr) {
+        fileStream.destroy();
-                        console.log("Error closing file for hash check: " + fileCloseErr);
-                        callback(fileCloseErr);
-                    } else {
-                        updateCallback(sizes);
-                        callback();
-                    }
-                });
-            }
-        };
 
-        reader();
+        var sizes = { intact: 0, altered: 0 };
+        var state = (fileHash !== hash.digest(encoding="hex")) ? "altered" : "intact";
+        sizes[state] = totalCount;
+
+        updateCallback(sizes);
+        callback();
     });
 }
 
@@ -364,7 +410,7 @@ function checkHashes(localDir, hashes, updateCallback, allDoneCallback, errorCal
         Object.keys(hashes),
         20,
         function (relativePath, callback) {
-            checkHash(localDir, relativePath, hashes[relativePath], callback, updateCallback);
+            checkHash(localDir, relativePath, hashes[relativePath], callback, updateCallback, (skipMissing = true));
         },
         function (err) {
             if (err) {