From 99193a198bbf2801da474455a2a492d92e92da56 Mon Sep 17 00:00:00 2001 From: Gent Semaj Date: Wed, 24 Jan 2024 05:15:00 -0500 Subject: [PATCH] Add `json2xdb` --- json2xdb/.gitignore | 2 + json2xdb/README.md | 9 ++ json2xdb/docker-compose.yml | 12 ++ json2xdb/json2xdb.py | 158 ++++++++++++++++++++ json2xdb/mappings.json | 43 ++++++ json2xdb/prototyping.ipynb | 283 ++++++++++++++++++++++++++++++++++++ 6 files changed, 507 insertions(+) create mode 100644 json2xdb/.gitignore create mode 100644 json2xdb/README.md create mode 100644 json2xdb/docker-compose.yml create mode 100644 json2xdb/json2xdb.py create mode 100644 json2xdb/mappings.json create mode 100644 json2xdb/prototyping.ipynb diff --git a/json2xdb/.gitignore b/json2xdb/.gitignore new file mode 100644 index 0000000..faa1838 --- /dev/null +++ b/json2xdb/.gitignore @@ -0,0 +1,2 @@ +.vscode + diff --git a/json2xdb/README.md b/json2xdb/README.md new file mode 100644 index 0000000..0ed545c --- /dev/null +++ b/json2xdb/README.md @@ -0,0 +1,9 @@ +# json2xdb +This script populates a functional, near-complete FusionFall XDB tabledata server. +You need an existing MySQL server (an old version; 5.5.42 seems to work with the FusionFall client). This can be set up pretty easily using Docker. +You also need a copy of xdt.json from the [OpenFusion tabledata repository](https://github.com/OpenFusionProject/tabledata). + +It is interesting to note that the JSON tabledata file is really just a Unity ScriptableObject containing all the XDT/XDB state packaged into a FusionFall client build. The devs likely kept a central tabledata server around (XDB) and, whenever it was time for a client build, they fetched it into local binary files (XDT) before finally packing them into the XdtTableScript asset. + +I would like to thank my girlfriend for showing me the wonders of `tqdm`. It really helped being able to see that things were happening. + diff --git a/json2xdb/docker-compose.yml b/json2xdb/docker-compose.yml new file mode 100644 index 0000000..bd587e4 --- /dev/null +++ b/json2xdb/docker-compose.yml @@ -0,0 +1,12 @@ +# Uses default credentials +version: '3.1' + +services: + db: + image: mysql:5.5.42 + restart: always + ports: + - 3306:3306 + environment: + MYSQL_ROOT_PASSWORD: mypassword + MYSQL_DATABASE: tabledata diff --git a/json2xdb/json2xdb.py b/json2xdb/json2xdb.py new file mode 100644 index 0000000..0c380ab --- /dev/null +++ b/json2xdb/json2xdb.py @@ -0,0 +1,158 @@ +# %% +import json +import sys +from tqdm import tqdm +import mysql.connector + +# %% +def get_db_column_name(xdt_field_name): + # special case 1 + if xdt_field_name == "m_iitemID": + return "ItemID" + + try: + # find the first uppercase character and split the string there + idx_of_first_uppercase = next(i for i, c in enumerate(xdt_field_name) if c.isupper()) + except StopIteration: + # special case 2 + if xdt_field_name == "m_ibattery": + idx_of_first_uppercase = 3 + else: + print(f"Could not find uppercase character in {xdt_field_name}") + sys.exit(1) + prefix = xdt_field_name[:idx_of_first_uppercase] + db_field_name = xdt_field_name[idx_of_first_uppercase:] + return db_field_name + +# %% +def table_entry_to_tuple(table_entry): + vals = [] + for field_name in table_entry: + field = table_entry[field_name] + vals.append(field) + return tuple(vals) + +def flatten_table_entry(table_entry): + flattened_entry = {} + for field_name in table_entry: + field = table_entry[field_name] + if type(field) == list: + for i, item in enumerate(field): + flattened_entry[f"{field_name}{i}"] = item + else: + flattened_entry[field_name] = field + return flattened_entry + +def handle_dict_table(table_entries, identifier_key, items_key): + new_table_entries = [] + for table_entry in table_entries: + identifier = table_entry[identifier_key] + items = table_entry[items_key] + for item in items: + new_item = {} + new_item[identifier_key] = identifier # needs to be first + for field_name in item: + new_item[field_name] = item[field_name] + new_table_entries.append(new_item) + return new_table_entries + + +# %% +def gen_column_sql(field_name, field_value): + field_type = type(field_value) + if field_type == int: + return f"`{field_name}` INT," + elif field_type == float: + return f"`{field_name}` FLOAT," + elif field_type == str: + # TODO maybe ascii vs unicode? + return f"`{field_name}` TEXT," + else: + print(f"Unknown type {field_type} for field {field_name}, skipping") + return "" + +# %% +def table_create(cursor, table_name, xdt_template_entry): + sql = f"CREATE TABLE {table_name} (" + sql += "id INT AUTO_INCREMENT PRIMARY KEY," + for field_name in xdt_template_entry: + db_field_name = get_db_column_name(field_name) + val = xdt_template_entry[field_name] + sql += gen_column_sql(db_field_name, val) + sql = sql[:-1] # remove trailing comma + sql += ")" + cursor.execute(sql) + +# %% +def table_populate(cursor, table_name, table_entries): + # generate the SQL first + sql = f"INSERT INTO {table_name} (" + template_entry = table_entries[0] + for field_name in template_entry: + db_field_name = get_db_column_name(field_name) + sql += f"`{db_field_name}`," + sql = sql[:-1] # remove trailing comma + sql += ") VALUES (" + for field_name in template_entry: + sql += f"%s," + sql = sql[:-1] # remove trailing comma + sql += ")" + + vals = [table_entry_to_tuple(entry) for entry in table_entries] + try: + cursor.executemany(sql, vals) + except Exception as e: + print(sql) + print(vals) + raise e + +# %% +def process_xdt_table(cursor, root, table_name, mappings): + table = root[table_name] + for (i, subtable_name) in tqdm(enumerate(table), desc=table_name, total=len(table)): + db_table_name = mappings[table_name][i] + #print(f"{subtable_name} => {db_table_name}") + + table_entries = table[subtable_name] + if db_table_name == "CutSceneText": + table_entries = handle_dict_table(table_entries, "m_iEvent", "m_TextElement") + table_entries = [flatten_table_entry(entry) for entry in table_entries] + + # clear the table + drop_sql = f"DROP TABLE IF EXISTS {db_table_name}" + cursor.execute(drop_sql) + + # create the table + table_create(cursor, db_table_name, table_entries[0]) + table_populate(cursor, db_table_name, table_entries) + +# %% +def main(conn, xdt_path): + with open("mappings.json", 'r') as f: + mappings = json.load(f) + with open(xdt_path, 'r') as f: + root = json.load(f) + cursor = conn.cursor() + for table_name in root: + if "Table" in table_name: + process_xdt_table(cursor, root, table_name, mappings) + conn.commit() + +# %% +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: python3 json2xdb.py ") + sys.exit(1) + xdt_path = sys.argv[1] + conn = mysql.connector.connect( + host="localhost", + user="root", + password="mypassword", + database="tabledata" + ) + main(conn, xdt_path) + +# %% + + + diff --git a/json2xdb/mappings.json b/json2xdb/mappings.json new file mode 100644 index 0000000..657b695 --- /dev/null +++ b/json2xdb/mappings.json @@ -0,0 +1,43 @@ +{ + "m_pNpcTable": [ "NpcTable", "NpcString", "NpcIcon", "BarkerTable", "NpcMesh", "NpcGroup", "ServiceString" ], + "m_pBackItemTable": [ "ItemBackTable", "ItemBackString", "ItemBackIcon", "ItemBackSound", "ItemBackMesh" ], + "m_pFaceItemTable": [ "ItemFaceTable", "ItemFaceString", "ItemFaceIcon", "ItemFaceSound", "ItemFaceMesh" ], + "m_pGlassItemTable": [ "ItemGlassTable", "ItemGlassString", "ItemGlassIcon", "ItemGlassSound", "ItemGlassMesh" ], + "m_pHatItemTable": [ "ItemHatTable", "ItemHatString", "ItemHatIcon", "ItemHatSound", "ItemHatMesh" ], + "m_pHeadItemTable": [ "ItemHeadTable", "ItemHeadString", "ItemHeadIcon", "ItemHeadSound", "ItemHeadMesh" ], + "m_pPantsItemTable": [ "ItemPantsTable", "ItemPantsString", "ItemPantsIcon", "ItemPantsSound", "ItemPantsMesh" ], + "m_pShirtsItemTable": [ "ItemShirtTable", "ItemShirtString", "ItemShirtIcon", "ItemShirtSound", "ItemShirtMesh" ], + "m_pShoesItemTable": [ "ItemShoesTable", "ItemShoesString", "ItemShoesIcon", "ItemShoesSound", "ItemShoesMesh" ], + "m_pWeaponItemTable": [ "ItemWpnTable", "ItemWpnString", "ItemWpnIcon", "ItemWpnSound", "ItemWpnMesh" ], + "m_pVehicleItemTable": [ "ItemVehicleTable", "ItemVehicleString", "ItemVehicleIcon", "ItemVehicleSound", "ItemVehicleMesh" ], + "m_pNameTable": [ "FirstNameTable", "MiddleNameTable", "LastNameTable" ], + "m_pChatTable": [ "1stChatTable", "2ndChatTable", "3rdChatTable", "ChatTable", "ChatString", "ClassTable", "EmoteLink" ], + "m_pAvatarTable": [ "AvatarTable", "AvatarGrowTable" ], + "m_pEmoteTable": [ "EmoteTable", "EmoteTexture" ], + "m_pGeneralItemTable": [ "ItemGeneralTable", "ItemGeneralString", "ItemGeneralIcon" ], + "m_pChestItemTable": [ "ItemChestTable", "ItemChestString", "ChestIconTable" ], + "m_pQuestItemTable": [ "ItemQuestTable", "ItemQuestString", "ItemQuestIcon" ], + "m_pAnimationTable": [ "M", "Mob", "Nano" ], + "m_pGuideTable": [ "GuideTable", "GuideStringTable" ], + "m_pInstanceTable": [ "InstanceTable", "WarpTable", "NameString" ], + "m_pMessageTable": [ "SystemMessage" ], + "m_pMissionTable": [ "MissionField", "MissionStringTable", "Journal_ID", "Reward" ], + "m_pNanoTable": [ "NanoTable", "NanoString", "NanoMesh", "NanoIcon", "NanoTuneTable", "NanoTuneString", "NanoTuneIcon" ], + "m_pShinyTable": [ "ShinyTable", "ShinyMesh", "ShinyString" ], + "m_pSkillTable": [ "SkillTable", "SkillIcon", "SkillBuffEffect", "SkillString" ], + "m_pConditionTable": [ "StatusTable" ], + "m_pTransportationTable": [ "TransportationTable", "TransportationMesh", "WarpLocationTable", "TransportationWarpString", "WyvernLocationTable", "TransportationWyvernString", "TransIcon" ], + "m_pVendorTable": [ "VendorTable" ], + "m_pXComTable": [ "XComTable" ], + "m_pCreationItemTable": [ "ItemCreationTable" ], + "m_pFirstUseTable": [ "FirstUseTable", "FirstUseString" ], + "m_pRulesTable": [ "RulesTable", "RulesString" ], + "m_pHelpTable": [ "HelpTable", "HelpString", "DescriptionTable", "Description", "DescriptionString" ], + "m_pCutSceneTable": [ "CutSceneText" ], + "m_pCombiningTable": [ "CombiningTable" ], + "m_pFilterTable": [ "UnfilterTable", "FilterTable", "NamefilterTable" ], + "m_pClassTable": [ "ClassType", "ClassString", "ClassWpnType", "ClassIcon" ], + "m_pEnchantTable": [ "EnchantTable" ], + "m_pClassSkillTable": [ "ClassSkill_Charging", "ClassSkill_Manager", "ClassSkill_Skill", "ClassSkill_String", "ClassSkill_BuffEffect", "ClassSkill_Icon", "ClassSkill_Sound", "Condition_Character" ], + "m_pSkillBookTable": [ "ItemSkillBookTable", "ItemSkillBookString", "ItemSkillBookIcon" ] +} \ No newline at end of file diff --git a/json2xdb/prototyping.ipynb b/json2xdb/prototyping.ipynb new file mode 100644 index 0000000..916434d --- /dev/null +++ b/json2xdb/prototyping.ipynb @@ -0,0 +1,283 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import sys\n", + "from tqdm import tqdm\n", + "import mysql.connector" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [], + "source": [ + "def get_db_column_name(xdt_field_name):\n", + " # special case 1\n", + " if xdt_field_name == \"m_iitemID\":\n", + " return \"ItemID\"\n", + " \n", + " try:\n", + " # find the first uppercase character and split the string there\n", + " idx_of_first_uppercase = next(i for i, c in enumerate(xdt_field_name) if c.isupper())\n", + " except StopIteration:\n", + " # special case 2\n", + " if xdt_field_name == \"m_ibattery\":\n", + " idx_of_first_uppercase = 3\n", + " else:\n", + " print(f\"Could not find uppercase character in {xdt_field_name}\")\n", + " sys.exit(1)\n", + " prefix = xdt_field_name[:idx_of_first_uppercase]\n", + " db_field_name = xdt_field_name[idx_of_first_uppercase:]\n", + " return db_field_name" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": {}, + "outputs": [], + "source": [ + "def table_entry_to_tuple(table_entry):\n", + " vals = []\n", + " for field_name in table_entry:\n", + " field = table_entry[field_name]\n", + " vals.append(field)\n", + " return tuple(vals)\n", + "\n", + "def flatten_table_entry(table_entry):\n", + " flattened_entry = {}\n", + " for field_name in table_entry:\n", + " field = table_entry[field_name]\n", + " if type(field) == list:\n", + " for i, item in enumerate(field):\n", + " flattened_entry[f\"{field_name}{i}\"] = item\n", + " else:\n", + " flattened_entry[field_name] = field\n", + " return flattened_entry\n", + "\n", + "def handle_dict_table(table_entries, identifier_key, items_key):\n", + " new_table_entries = []\n", + " for table_entry in table_entries:\n", + " identifier = table_entry[identifier_key]\n", + " items = table_entry[items_key]\n", + " for item in items:\n", + " new_item = {}\n", + " new_item[identifier_key] = identifier # needs to be first\n", + " for field_name in item:\n", + " new_item[field_name] = item[field_name]\n", + " new_table_entries.append(new_item)\n", + " return new_table_entries\n" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [], + "source": [ + "def gen_column_sql(field_name, field_value):\n", + " field_type = type(field_value)\n", + " if field_type == int:\n", + " return f\"`{field_name}` INT,\"\n", + " elif field_type == float:\n", + " return f\"`{field_name}` FLOAT,\"\n", + " elif field_type == str:\n", + " # TODO maybe ascii vs unicode?\n", + " return f\"`{field_name}` TEXT,\"\n", + " else:\n", + " print(f\"Unknown type {field_type} for field {field_name}, skipping\")\n", + " return \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": {}, + "outputs": [], + "source": [ + "def table_create(cursor, table_name, xdt_template_entry):\n", + " sql = f\"CREATE TABLE {table_name} (\"\n", + " sql += \"id INT AUTO_INCREMENT PRIMARY KEY,\"\n", + " for field_name in xdt_template_entry:\n", + " db_field_name = get_db_column_name(field_name)\n", + " val = xdt_template_entry[field_name]\n", + " sql += gen_column_sql(db_field_name, val)\n", + " sql = sql[:-1] # remove trailing comma\n", + " sql += \")\"\n", + " cursor.execute(sql)" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": {}, + "outputs": [], + "source": [ + "def table_populate(cursor, table_name, table_entries):\n", + " # generate the SQL first\n", + " sql = f\"INSERT INTO {table_name} (\"\n", + " template_entry = table_entries[0]\n", + " for field_name in template_entry:\n", + " db_field_name = get_db_column_name(field_name)\n", + " sql += f\"`{db_field_name}`,\"\n", + " sql = sql[:-1] # remove trailing comma\n", + " sql += \") VALUES (\"\n", + " for field_name in template_entry:\n", + " sql += f\"%s,\"\n", + " sql = sql[:-1] # remove trailing comma\n", + " sql += \")\"\n", + " \n", + " vals = [table_entry_to_tuple(entry) for entry in table_entries]\n", + " try:\n", + " cursor.executemany(sql, vals)\n", + " except Exception as e:\n", + " print(sql)\n", + " print(vals)\n", + " raise e" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [], + "source": [ + "def process_xdt_table(cursor, root, table_name, mappings):\n", + " table = root[table_name]\n", + " for (i, subtable_name) in tqdm(enumerate(table), desc=table_name, total=len(table)):\n", + " db_table_name = mappings[table_name][i]\n", + " #print(f\"{subtable_name} => {db_table_name}\")\n", + " \n", + " table_entries = table[subtable_name]\n", + " if db_table_name == \"CutSceneText\":\n", + " table_entries = handle_dict_table(table_entries, \"m_iEvent\", \"m_TextElement\")\n", + " table_entries = [flatten_table_entry(entry) for entry in table_entries]\n", + "\n", + " # clear the table\n", + " drop_sql = f\"DROP TABLE IF EXISTS {db_table_name}\"\n", + " cursor.execute(drop_sql)\n", + "\n", + " # create the table\n", + " table_create(cursor, db_table_name, table_entries[0])\n", + " table_populate(cursor, db_table_name, table_entries)" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [], + "source": [ + "def main(conn, xdt_path):\n", + " with open(\"mappings.json\", 'r') as f:\n", + " mappings = json.load(f)\n", + " with open(xdt_path, 'r') as f:\n", + " root = json.load(f)\n", + " cursor = conn.cursor()\n", + " for table_name in root:\n", + " if \"Table\" in table_name:\n", + " process_xdt_table(cursor, root, table_name, mappings)\n", + " conn.commit()" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "m_pAnimationTable: 100%|██████████| 3/3 [00:00<00:00, 9.30it/s]\n", + "m_pAvatarTable: 100%|██████████| 2/2 [00:00<00:00, 5.76it/s]\n", + "m_pChatTable: 100%|██████████| 7/7 [00:00<00:00, 9.55it/s]\n", + "m_pEmoteTable: 100%|██████████| 2/2 [00:00<00:00, 12.84it/s]\n", + "m_pGuideTable: 100%|██████████| 2/2 [00:00<00:00, 11.23it/s]\n", + "m_pInstanceTable: 100%|██████████| 3/3 [00:00<00:00, 9.99it/s]\n", + "m_pMessageTable: 100%|██████████| 1/1 [00:00<00:00, 8.64it/s]\n", + "m_pMissionTable: 100%|██████████| 4/4 [00:01<00:00, 2.57it/s]\n", + "m_pNameTable: 100%|██████████| 3/3 [00:00<00:00, 9.74it/s]\n", + "m_pNanoTable: 100%|██████████| 7/7 [00:00<00:00, 9.92it/s]\n", + "m_pNpcTable: 100%|██████████| 7/7 [00:01<00:00, 4.86it/s]\n", + "m_pShinyTable: 100%|██████████| 3/3 [00:00<00:00, 9.07it/s]\n", + "m_pSkillTable: 100%|██████████| 4/4 [00:00<00:00, 11.26it/s]\n", + "m_pConditionTable: 100%|██████████| 1/1 [00:00<00:00, 11.58it/s]\n", + "m_pTransportationTable: 100%|██████████| 7/7 [00:00<00:00, 10.35it/s]\n", + "m_pVendorTable: 100%|██████████| 1/1 [00:00<00:00, 4.68it/s]\n", + "m_pXComTable: 100%|██████████| 1/1 [00:00<00:00, 9.32it/s]\n", + "m_pBackItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.05it/s]\n", + "m_pFaceItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.85it/s]\n", + "m_pGlassItemTable: 100%|██████████| 5/5 [00:00<00:00, 8.95it/s]\n", + "m_pHatItemTable: 100%|██████████| 5/5 [00:00<00:00, 10.58it/s]\n", + "m_pHeadItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.31it/s]\n", + "m_pPantsItemTable: 100%|██████████| 5/5 [00:00<00:00, 8.16it/s]\n", + "m_pShirtsItemTable: 100%|██████████| 5/5 [00:00<00:00, 7.10it/s]\n", + "m_pShoesItemTable: 100%|██████████| 5/5 [00:00<00:00, 6.49it/s]\n", + "m_pWeaponItemTable: 100%|██████████| 5/5 [00:00<00:00, 6.26it/s]\n", + "m_pVehicleItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.41it/s]\n", + "m_pGeneralItemTable: 100%|██████████| 3/3 [00:00<00:00, 11.56it/s]\n", + "m_pChestItemTable: 100%|██████████| 3/3 [00:00<00:00, 6.83it/s]\n", + "m_pQuestItemTable: 100%|██████████| 3/3 [00:00<00:00, 11.19it/s]\n", + "m_pCreationItemTable: 100%|██████████| 1/1 [00:00<00:00, 12.50it/s]\n", + "m_pFirstUseTable: 100%|██████████| 2/2 [00:00<00:00, 10.72it/s]\n", + "m_pRulesTable: 100%|██████████| 2/2 [00:00<00:00, 8.38it/s]\n", + "m_pHelpTable: 100%|██████████| 5/5 [00:00<00:00, 9.11it/s]\n", + "m_pCutSceneTable: 100%|██████████| 1/1 [00:00<00:00, 11.51it/s]\n", + "m_pCombiningTable: 100%|██████████| 1/1 [00:00<00:00, 13.88it/s]\n", + "m_pFilterTable: 100%|██████████| 3/3 [00:00<00:00, 9.08it/s]\n", + "m_pClassTable: 100%|██████████| 4/4 [00:00<00:00, 10.94it/s]\n", + "m_pEnchantTable: 100%|██████████| 1/1 [00:00<00:00, 11.75it/s]\n", + "m_pClassSkillTable: 100%|██████████| 8/8 [00:00<00:00, 9.37it/s]\n", + "m_pSkillBookTable: 100%|██████████| 3/3 [00:00<00:00, 10.67it/s]\n" + ] + } + ], + "source": [ + "xdt_path = \"tdata/xdt.json\"\n", + "conn = mysql.connector.connect(\n", + " host=\"localhost\",\n", + " user=\"root\",\n", + " password=\"mypassword\",\n", + " database=\"tabledata\"\n", + ")\n", + "main(conn, xdt_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}