diff --git a/assets/js/server-selector.js b/assets/js/server-selector.js index acac1d6..84fbe14 100644 --- a/assets/js/server-selector.js +++ b/assets/js/server-selector.js @@ -10,7 +10,6 @@ var userData = remote.require("app").getPath("userData"); var configPath = path.join(userData, "config.json"); var serversPath = path.join(userData, "servers.json"); var versionsPath = path.join(userData, "versions.json"); -var hashPath = path.join(userData, "hashes.json"); var cacheRoot = path.join( userData, "/../../LocalLow/Unity/Web Player/Cache" @@ -289,8 +288,8 @@ function storageLoadingUpdate(allSizes) { $.each(allSizes, function (versionString, vSizes) { $.each(vSizes, function (cacheMode, sizes) { var label = document.getElementById(getCacheElemID(versionString, cacheMode, "label")); - if (label) - label.innerHTML = getCacheLabelText(sizes); + if (!label) return; + label.innerHTML = getCacheLabelText(sizes); }); }); } @@ -341,7 +340,7 @@ function handleCache(mode, versionString, cacheMode, callback) { "http://cdn.dexlabs.systems/ff/big" : path.dirname(versions[0].url); - var lastSizes = {}; + var lastSizes = { intact: 0, altered: 0, total: 0 }; var buf = ""; storageLoadingStart(versionString, cacheMode); @@ -369,7 +368,6 @@ function handleCache(mode, versionString, cacheMode, callback) { server.listen(0, "localhost", function () { spawn(path.join(__dirname, "lib", "cache_handler.exe"), [ "--mode", (mode === "fix") ? "download" : mode, - "--hash-file", hashPath, "--playable-root", cacheRoot, "--offline-root", offlineRoot, "--user-dir", userData, diff --git a/cache_handler/cache_handler.py b/cache_handler/cache_handler.py new file mode 100644 index 0000000..40ffdbf --- /dev/null +++ b/cache_handler/cache_handler.py @@ -0,0 +1,205 @@ +import json +import random +import asyncio +import hashlib +from pathlib import Path +from dataclasses import dataclass +from typing import List, Dict +from argparse import Namespace, ArgumentParser + +import httpx +import aiofiles + + +# hack to get pyinstaller 3.5 to work +if False: + import anyio._backends._asyncio + + +BUF_SIZE = 1 << 16 +SizeDict = Dict[str, Dict[str, Dict[str, int]]] +size_dict: SizeDict = {} + + +@dataclass +class FileInfo: + version: str + mode: str + url_path: str + local_path: Path + sha256: str + + +async def send_message(writer: asyncio.StreamWriter, obj: SizeDict) -> None: + message = (json.dumps(obj) + '\n').encode('utf-8') + writer.write(message) + await writer.drain() + + +async def check_file_hash(file_info: FileInfo) -> bool: + size = 0 + state = 'altered' + sha256 = hashlib.sha256() + + try: + async with aiofiles.open(file_info.local_path, mode='rb') as rb: + while True: + data = await rb.read(BUF_SIZE) + if not data: + break + sha256.update(data) + size += len(data) + + state = 'intact' if sha256.hexdigest() == file_info.sha256 else 'altered' + size_dict[file_info.version][file_info.mode][state] += size + except: + pass + + return state == 'intact' + + +async def download_file(writer: asyncio.StreamWriter, client: httpx.AsyncClient, file_info: FileInfo, retries: int = 5) -> None: + if (await check_file_hash(file_info)): + await send_message(writer, size_dict) + return + + for i in range(retries): + try: + async with client.stream('GET', file_info.url_path) as stream: + stream.raise_for_status() + + async with aiofiles.open(file_info.local_path, mode='wb') as wb: + async for chunk in stream.aiter_bytes(chunk_size=BUF_SIZE): + await wb.write(chunk) + except: + await asyncio.sleep(i + 1) + + if (await check_file_hash(file_info)): + break + + await send_message(writer, size_dict) + + +async def hash_check(writer: asyncio.StreamWriter, file_info_list: List[FileInfo], update_freq: int = 50) -> None: + coroutines = [check_file_hash(file_info) for file_info in file_info_list] + random.shuffle(coroutines) + + for i in range(0, len(coroutines), update_freq): + await asyncio.gather(*coroutines[i:i+update_freq]) + await send_message(writer, size_dict) + + +async def download(writer: asyncio.StreamWriter, file_info_list: List[FileInfo], max_connections: int = 5) -> None: + for file_info in file_info_list: + file_info.local_path.parent.mkdir(parents=True, exist_ok=True) + + async with httpx.AsyncClient(limits=httpx.Limits(max_connections=max_connections), + timeout=httpx.Timeout(None)) as client: + + coroutines = [download_file(writer, client, file_info) for file_info in file_info_list] + random.shuffle(coroutines) + + await asyncio.gather(*coroutines) + + +async def remove(writer: asyncio.StreamWriter, file_info_list: List[FileInfo]) -> None: + roots = set() + for file_info in file_info_list: + if file_info.local_path.parent.is_dir(): + roots.add(file_info.local_path.parent) + if file_info.local_path.is_file(): + file_info.local_path.unlink() + + await send_message(writer, size_dict) + + roots_list: List[Path] = sorted(roots, key=lambda p: len(p.parts), reverse=True) + for root_dir in roots_list: + if not any(root_dir.iterdir()): + root_dir.rmdir() + + +async def prep_and_run_coroutine(args: Namespace) -> None: + file_info_list = compile_file_list(args) + + _, writer = await asyncio.open_connection('localhost', args.port) + + coroutines = { + 'hash-check': hash_check, + 'download': download, + 'delete': remove, + } + await coroutines[args.mode](writer, file_info_list) + + writer.close() + await writer.wait_closed() + + +def swapped_path(local_root: str, user_dir: str, cache_version: str) -> Path: + current_cache = Path(local_root) / 'FusionFall' + named_cache = Path(local_root) / cache_version + record_path = Path(user_dir) / '.lastver' + + if (not named_cache.is_dir() and + current_cache.is_dir() and + record_path.is_file() and + cache_version == record_path.read_text(encoding='utf-8')): + return current_cache + + return named_cache + + +def compile_file_list(args: Namespace) -> List[FileInfo]: + with open(Path(args.user_dir) / 'hashes.json') as r: + hash_dict = json.load(r) + + cache_modes = ['offline', 'playable'] if args.cache_mode == 'all' else [args.cache_mode] + cache_versions = list(hash_dict) if args.cache_version == 'all' else [args.cache_version] + + file_info_list = [] + + for cache_version in cache_versions: + for cache_mode in cache_modes: + local_root = args.offline_root if cache_mode == 'offline' else args.playable_root + local_dir = swapped_path(local_root, args.user_dir, cache_version) + + if cache_version not in size_dict: + size_dict[cache_version] = {} + + size_dict[cache_version][cache_mode] = { + 'intact': 0, + 'altered': 0, + 'total': hash_dict[cache_version][cache_mode + '_size'], + } + + file_info_list.extend([ + FileInfo( + version=cache_version, + mode=cache_mode, + url_path='/'.join([args.cdn_root, cache_version, rel_path]), + local_path=(local_dir / rel_path), + sha256=file_hash, + ) for rel_path, file_hash in hash_dict[cache_version][cache_mode].items() + ]) + + return file_info_list + + +def parse_args() -> Namespace: + parser = ArgumentParser('Python executable for tasks relating to OpenFusionClient.') + parser.add_argument('--mode', type=str, required=True, choices=['hash-check', 'download', 'delete']) + parser.add_argument('--playable-root', dest='playable_root', type=str) + parser.add_argument('--offline-root', dest='offline_root', type=str) + parser.add_argument('--user-dir', dest='user_dir', type=str, required=True) + parser.add_argument('--cdn-root', dest='cdn_root', type=str, default='http://cdn.dexlabs.systems/ff/big') + parser.add_argument('--cache-mode', dest='cache_mode', type=str, default='all', choices=['all', 'offline', 'playable']) + parser.add_argument('--cache-version', dest='cache_version', type=str, default='all') + parser.add_argument('--port', type=str, required=True) + return parser.parse_args() + + +def main() -> None: + asyncio.run(prep_and_run_coroutine(parse_args())) + + +if __name__ == '__main__': + main() diff --git a/cache_handler/cache_handler.spec b/cache_handler/cache_handler.spec new file mode 100644 index 0000000..6c24636 --- /dev/null +++ b/cache_handler/cache_handler.spec @@ -0,0 +1,33 @@ +# -*- mode: python ; coding: utf-8 -*- + +block_cipher = None + + +a = Analysis(['cache_handler.py'], + pathex=['Z:\\src'], + binaries=[], + datas=[], + hiddenimports=[], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name='cache_handler', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True ) diff --git a/cache_handler/requirements.txt b/cache_handler/requirements.txt new file mode 100644 index 0000000..c74bf67 --- /dev/null +++ b/cache_handler/requirements.txt @@ -0,0 +1,3 @@ +aiofiles +httpx +pyinstaller==3.5 \ No newline at end of file diff --git a/index.js b/index.js index a910539..315e9bb 100644 --- a/index.js +++ b/index.js @@ -32,7 +32,7 @@ var hashPath = path.join(userData, "hashes.json"); function initialSetup(firstTime) { if (!firstTime) { - // Migration from pre-1.4 + // Migration from pre-1.5 // Back everything up, just in case fs.copySync(configPath, configPath + ".bak"); fs.copySync(serversPath, serversPath + ".bak"); @@ -93,8 +93,8 @@ app.on("ready", function () { initialSetup(true); } else { var config = fs.readJsonSync(configPath); - if (!config["last-version-initialized"]) { - console.log("Pre-1.4 config detected. Running migration."); + if (config["last-version-initialized"] !== "1.5") { + console.log("Pre-1.5 config detected. Running migration."); initialSetup(false); } else { showMainWindow(); diff --git a/lib/cache_handler.exe b/lib/cache_handler.exe index 280ccf9..5f4a285 100755 Binary files a/lib/cache_handler.exe and b/lib/cache_handler.exe differ