Add `json2xdb`

This commit is contained in:
Gent Semaj 2024-01-24 05:15:00 -05:00
parent 249af1d2d3
commit 99193a198b
Signed by untrusted user: ycc
GPG Key ID: 2D76C57BF6BEADC4
6 changed files with 507 additions and 0 deletions

2
json2xdb/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
.vscode

9
json2xdb/README.md Normal file
View File

@ -0,0 +1,9 @@
# json2xdb
This script populates a functional, near-complete FusionFall XDB tabledata server.
You need an existing MySQL server (an old version; 5.5.42 seems to work with the FusionFall client). This can be set up pretty easily using Docker.
You also need a copy of xdt.json from the [OpenFusion tabledata repository](https://github.com/OpenFusionProject/tabledata).
It is interesting to note that the JSON tabledata file is really just a Unity ScriptableObject containing all the XDT/XDB state packaged into a FusionFall client build. The devs likely kept a central tabledata server around (XDB) and, whenever it was time for a client build, they fetched it into local binary files (XDT) before finally packing them into the XdtTableScript asset.
I would like to thank my girlfriend for showing me the wonders of `tqdm`. It really helped being able to see that things were happening.

View File

@ -0,0 +1,12 @@
# Uses default credentials
version: '3.1'
services:
db:
image: mysql:5.5.42
restart: always
ports:
- 3306:3306
environment:
MYSQL_ROOT_PASSWORD: mypassword
MYSQL_DATABASE: tabledata

158
json2xdb/json2xdb.py Normal file
View File

@ -0,0 +1,158 @@
# %%
import json
import sys
from tqdm import tqdm
import mysql.connector
# %%
def get_db_column_name(xdt_field_name):
# special case 1
if xdt_field_name == "m_iitemID":
return "ItemID"
try:
# find the first uppercase character and split the string there
idx_of_first_uppercase = next(i for i, c in enumerate(xdt_field_name) if c.isupper())
except StopIteration:
# special case 2
if xdt_field_name == "m_ibattery":
idx_of_first_uppercase = 3
else:
print(f"Could not find uppercase character in {xdt_field_name}")
sys.exit(1)
prefix = xdt_field_name[:idx_of_first_uppercase]
db_field_name = xdt_field_name[idx_of_first_uppercase:]
return db_field_name
# %%
def table_entry_to_tuple(table_entry):
vals = []
for field_name in table_entry:
field = table_entry[field_name]
vals.append(field)
return tuple(vals)
def flatten_table_entry(table_entry):
flattened_entry = {}
for field_name in table_entry:
field = table_entry[field_name]
if type(field) == list:
for i, item in enumerate(field):
flattened_entry[f"{field_name}{i}"] = item
else:
flattened_entry[field_name] = field
return flattened_entry
def handle_dict_table(table_entries, identifier_key, items_key):
new_table_entries = []
for table_entry in table_entries:
identifier = table_entry[identifier_key]
items = table_entry[items_key]
for item in items:
new_item = {}
new_item[identifier_key] = identifier # needs to be first
for field_name in item:
new_item[field_name] = item[field_name]
new_table_entries.append(new_item)
return new_table_entries
# %%
def gen_column_sql(field_name, field_value):
field_type = type(field_value)
if field_type == int:
return f"`{field_name}` INT,"
elif field_type == float:
return f"`{field_name}` FLOAT,"
elif field_type == str:
# TODO maybe ascii vs unicode?
return f"`{field_name}` TEXT,"
else:
print(f"Unknown type {field_type} for field {field_name}, skipping")
return ""
# %%
def table_create(cursor, table_name, xdt_template_entry):
sql = f"CREATE TABLE {table_name} ("
sql += "id INT AUTO_INCREMENT PRIMARY KEY,"
for field_name in xdt_template_entry:
db_field_name = get_db_column_name(field_name)
val = xdt_template_entry[field_name]
sql += gen_column_sql(db_field_name, val)
sql = sql[:-1] # remove trailing comma
sql += ")"
cursor.execute(sql)
# %%
def table_populate(cursor, table_name, table_entries):
# generate the SQL first
sql = f"INSERT INTO {table_name} ("
template_entry = table_entries[0]
for field_name in template_entry:
db_field_name = get_db_column_name(field_name)
sql += f"`{db_field_name}`,"
sql = sql[:-1] # remove trailing comma
sql += ") VALUES ("
for field_name in template_entry:
sql += f"%s,"
sql = sql[:-1] # remove trailing comma
sql += ")"
vals = [table_entry_to_tuple(entry) for entry in table_entries]
try:
cursor.executemany(sql, vals)
except Exception as e:
print(sql)
print(vals)
raise e
# %%
def process_xdt_table(cursor, root, table_name, mappings):
table = root[table_name]
for (i, subtable_name) in tqdm(enumerate(table), desc=table_name, total=len(table)):
db_table_name = mappings[table_name][i]
#print(f"{subtable_name} => {db_table_name}")
table_entries = table[subtable_name]
if db_table_name == "CutSceneText":
table_entries = handle_dict_table(table_entries, "m_iEvent", "m_TextElement")
table_entries = [flatten_table_entry(entry) for entry in table_entries]
# clear the table
drop_sql = f"DROP TABLE IF EXISTS {db_table_name}"
cursor.execute(drop_sql)
# create the table
table_create(cursor, db_table_name, table_entries[0])
table_populate(cursor, db_table_name, table_entries)
# %%
def main(conn, xdt_path):
with open("mappings.json", 'r') as f:
mappings = json.load(f)
with open(xdt_path, 'r') as f:
root = json.load(f)
cursor = conn.cursor()
for table_name in root:
if "Table" in table_name:
process_xdt_table(cursor, root, table_name, mappings)
conn.commit()
# %%
if __name__ == "__main__":
if len(sys.argv) != 2:
print("Usage: python3 json2xdb.py <path to xdt file>")
sys.exit(1)
xdt_path = sys.argv[1]
conn = mysql.connector.connect(
host="localhost",
user="root",
password="mypassword",
database="tabledata"
)
main(conn, xdt_path)
# %%

43
json2xdb/mappings.json Normal file
View File

@ -0,0 +1,43 @@
{
"m_pNpcTable": [ "NpcTable", "NpcString", "NpcIcon", "BarkerTable", "NpcMesh", "NpcGroup", "ServiceString" ],
"m_pBackItemTable": [ "ItemBackTable", "ItemBackString", "ItemBackIcon", "ItemBackSound", "ItemBackMesh" ],
"m_pFaceItemTable": [ "ItemFaceTable", "ItemFaceString", "ItemFaceIcon", "ItemFaceSound", "ItemFaceMesh" ],
"m_pGlassItemTable": [ "ItemGlassTable", "ItemGlassString", "ItemGlassIcon", "ItemGlassSound", "ItemGlassMesh" ],
"m_pHatItemTable": [ "ItemHatTable", "ItemHatString", "ItemHatIcon", "ItemHatSound", "ItemHatMesh" ],
"m_pHeadItemTable": [ "ItemHeadTable", "ItemHeadString", "ItemHeadIcon", "ItemHeadSound", "ItemHeadMesh" ],
"m_pPantsItemTable": [ "ItemPantsTable", "ItemPantsString", "ItemPantsIcon", "ItemPantsSound", "ItemPantsMesh" ],
"m_pShirtsItemTable": [ "ItemShirtTable", "ItemShirtString", "ItemShirtIcon", "ItemShirtSound", "ItemShirtMesh" ],
"m_pShoesItemTable": [ "ItemShoesTable", "ItemShoesString", "ItemShoesIcon", "ItemShoesSound", "ItemShoesMesh" ],
"m_pWeaponItemTable": [ "ItemWpnTable", "ItemWpnString", "ItemWpnIcon", "ItemWpnSound", "ItemWpnMesh" ],
"m_pVehicleItemTable": [ "ItemVehicleTable", "ItemVehicleString", "ItemVehicleIcon", "ItemVehicleSound", "ItemVehicleMesh" ],
"m_pNameTable": [ "FirstNameTable", "MiddleNameTable", "LastNameTable" ],
"m_pChatTable": [ "1stChatTable", "2ndChatTable", "3rdChatTable", "ChatTable", "ChatString", "ClassTable", "EmoteLink" ],
"m_pAvatarTable": [ "AvatarTable", "AvatarGrowTable" ],
"m_pEmoteTable": [ "EmoteTable", "EmoteTexture" ],
"m_pGeneralItemTable": [ "ItemGeneralTable", "ItemGeneralString", "ItemGeneralIcon" ],
"m_pChestItemTable": [ "ItemChestTable", "ItemChestString", "ChestIconTable" ],
"m_pQuestItemTable": [ "ItemQuestTable", "ItemQuestString", "ItemQuestIcon" ],
"m_pAnimationTable": [ "M", "Mob", "Nano" ],
"m_pGuideTable": [ "GuideTable", "GuideStringTable" ],
"m_pInstanceTable": [ "InstanceTable", "WarpTable", "NameString" ],
"m_pMessageTable": [ "SystemMessage" ],
"m_pMissionTable": [ "MissionField", "MissionStringTable", "Journal_ID", "Reward" ],
"m_pNanoTable": [ "NanoTable", "NanoString", "NanoMesh", "NanoIcon", "NanoTuneTable", "NanoTuneString", "NanoTuneIcon" ],
"m_pShinyTable": [ "ShinyTable", "ShinyMesh", "ShinyString" ],
"m_pSkillTable": [ "SkillTable", "SkillIcon", "SkillBuffEffect", "SkillString" ],
"m_pConditionTable": [ "StatusTable" ],
"m_pTransportationTable": [ "TransportationTable", "TransportationMesh", "WarpLocationTable", "TransportationWarpString", "WyvernLocationTable", "TransportationWyvernString", "TransIcon" ],
"m_pVendorTable": [ "VendorTable" ],
"m_pXComTable": [ "XComTable" ],
"m_pCreationItemTable": [ "ItemCreationTable" ],
"m_pFirstUseTable": [ "FirstUseTable", "FirstUseString" ],
"m_pRulesTable": [ "RulesTable", "RulesString" ],
"m_pHelpTable": [ "HelpTable", "HelpString", "DescriptionTable", "Description", "DescriptionString" ],
"m_pCutSceneTable": [ "CutSceneText" ],
"m_pCombiningTable": [ "CombiningTable" ],
"m_pFilterTable": [ "UnfilterTable", "FilterTable", "NamefilterTable" ],
"m_pClassTable": [ "ClassType", "ClassString", "ClassWpnType", "ClassIcon" ],
"m_pEnchantTable": [ "EnchantTable" ],
"m_pClassSkillTable": [ "ClassSkill_Charging", "ClassSkill_Manager", "ClassSkill_Skill", "ClassSkill_String", "ClassSkill_BuffEffect", "ClassSkill_Icon", "ClassSkill_Sound", "Condition_Character" ],
"m_pSkillBookTable": [ "ItemSkillBookTable", "ItemSkillBookString", "ItemSkillBookIcon" ]
}

283
json2xdb/prototyping.ipynb Normal file
View File

@ -0,0 +1,283 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 59,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"import sys\n",
"from tqdm import tqdm\n",
"import mysql.connector"
]
},
{
"cell_type": "code",
"execution_count": 60,
"metadata": {},
"outputs": [],
"source": [
"def get_db_column_name(xdt_field_name):\n",
" # special case 1\n",
" if xdt_field_name == \"m_iitemID\":\n",
" return \"ItemID\"\n",
" \n",
" try:\n",
" # find the first uppercase character and split the string there\n",
" idx_of_first_uppercase = next(i for i, c in enumerate(xdt_field_name) if c.isupper())\n",
" except StopIteration:\n",
" # special case 2\n",
" if xdt_field_name == \"m_ibattery\":\n",
" idx_of_first_uppercase = 3\n",
" else:\n",
" print(f\"Could not find uppercase character in {xdt_field_name}\")\n",
" sys.exit(1)\n",
" prefix = xdt_field_name[:idx_of_first_uppercase]\n",
" db_field_name = xdt_field_name[idx_of_first_uppercase:]\n",
" return db_field_name"
]
},
{
"cell_type": "code",
"execution_count": 61,
"metadata": {},
"outputs": [],
"source": [
"def table_entry_to_tuple(table_entry):\n",
" vals = []\n",
" for field_name in table_entry:\n",
" field = table_entry[field_name]\n",
" vals.append(field)\n",
" return tuple(vals)\n",
"\n",
"def flatten_table_entry(table_entry):\n",
" flattened_entry = {}\n",
" for field_name in table_entry:\n",
" field = table_entry[field_name]\n",
" if type(field) == list:\n",
" for i, item in enumerate(field):\n",
" flattened_entry[f\"{field_name}{i}\"] = item\n",
" else:\n",
" flattened_entry[field_name] = field\n",
" return flattened_entry\n",
"\n",
"def handle_dict_table(table_entries, identifier_key, items_key):\n",
" new_table_entries = []\n",
" for table_entry in table_entries:\n",
" identifier = table_entry[identifier_key]\n",
" items = table_entry[items_key]\n",
" for item in items:\n",
" new_item = {}\n",
" new_item[identifier_key] = identifier # needs to be first\n",
" for field_name in item:\n",
" new_item[field_name] = item[field_name]\n",
" new_table_entries.append(new_item)\n",
" return new_table_entries\n"
]
},
{
"cell_type": "code",
"execution_count": 62,
"metadata": {},
"outputs": [],
"source": [
"def gen_column_sql(field_name, field_value):\n",
" field_type = type(field_value)\n",
" if field_type == int:\n",
" return f\"`{field_name}` INT,\"\n",
" elif field_type == float:\n",
" return f\"`{field_name}` FLOAT,\"\n",
" elif field_type == str:\n",
" # TODO maybe ascii vs unicode?\n",
" return f\"`{field_name}` TEXT,\"\n",
" else:\n",
" print(f\"Unknown type {field_type} for field {field_name}, skipping\")\n",
" return \"\""
]
},
{
"cell_type": "code",
"execution_count": 63,
"metadata": {},
"outputs": [],
"source": [
"def table_create(cursor, table_name, xdt_template_entry):\n",
" sql = f\"CREATE TABLE {table_name} (\"\n",
" sql += \"id INT AUTO_INCREMENT PRIMARY KEY,\"\n",
" for field_name in xdt_template_entry:\n",
" db_field_name = get_db_column_name(field_name)\n",
" val = xdt_template_entry[field_name]\n",
" sql += gen_column_sql(db_field_name, val)\n",
" sql = sql[:-1] # remove trailing comma\n",
" sql += \")\"\n",
" cursor.execute(sql)"
]
},
{
"cell_type": "code",
"execution_count": 64,
"metadata": {},
"outputs": [],
"source": [
"def table_populate(cursor, table_name, table_entries):\n",
" # generate the SQL first\n",
" sql = f\"INSERT INTO {table_name} (\"\n",
" template_entry = table_entries[0]\n",
" for field_name in template_entry:\n",
" db_field_name = get_db_column_name(field_name)\n",
" sql += f\"`{db_field_name}`,\"\n",
" sql = sql[:-1] # remove trailing comma\n",
" sql += \") VALUES (\"\n",
" for field_name in template_entry:\n",
" sql += f\"%s,\"\n",
" sql = sql[:-1] # remove trailing comma\n",
" sql += \")\"\n",
" \n",
" vals = [table_entry_to_tuple(entry) for entry in table_entries]\n",
" try:\n",
" cursor.executemany(sql, vals)\n",
" except Exception as e:\n",
" print(sql)\n",
" print(vals)\n",
" raise e"
]
},
{
"cell_type": "code",
"execution_count": 65,
"metadata": {},
"outputs": [],
"source": [
"def process_xdt_table(cursor, root, table_name, mappings):\n",
" table = root[table_name]\n",
" for (i, subtable_name) in tqdm(enumerate(table), desc=table_name, total=len(table)):\n",
" db_table_name = mappings[table_name][i]\n",
" #print(f\"{subtable_name} => {db_table_name}\")\n",
" \n",
" table_entries = table[subtable_name]\n",
" if db_table_name == \"CutSceneText\":\n",
" table_entries = handle_dict_table(table_entries, \"m_iEvent\", \"m_TextElement\")\n",
" table_entries = [flatten_table_entry(entry) for entry in table_entries]\n",
"\n",
" # clear the table\n",
" drop_sql = f\"DROP TABLE IF EXISTS {db_table_name}\"\n",
" cursor.execute(drop_sql)\n",
"\n",
" # create the table\n",
" table_create(cursor, db_table_name, table_entries[0])\n",
" table_populate(cursor, db_table_name, table_entries)"
]
},
{
"cell_type": "code",
"execution_count": 66,
"metadata": {},
"outputs": [],
"source": [
"def main(conn, xdt_path):\n",
" with open(\"mappings.json\", 'r') as f:\n",
" mappings = json.load(f)\n",
" with open(xdt_path, 'r') as f:\n",
" root = json.load(f)\n",
" cursor = conn.cursor()\n",
" for table_name in root:\n",
" if \"Table\" in table_name:\n",
" process_xdt_table(cursor, root, table_name, mappings)\n",
" conn.commit()"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"m_pAnimationTable: 100%|██████████| 3/3 [00:00<00:00, 9.30it/s]\n",
"m_pAvatarTable: 100%|██████████| 2/2 [00:00<00:00, 5.76it/s]\n",
"m_pChatTable: 100%|██████████| 7/7 [00:00<00:00, 9.55it/s]\n",
"m_pEmoteTable: 100%|██████████| 2/2 [00:00<00:00, 12.84it/s]\n",
"m_pGuideTable: 100%|██████████| 2/2 [00:00<00:00, 11.23it/s]\n",
"m_pInstanceTable: 100%|██████████| 3/3 [00:00<00:00, 9.99it/s]\n",
"m_pMessageTable: 100%|██████████| 1/1 [00:00<00:00, 8.64it/s]\n",
"m_pMissionTable: 100%|██████████| 4/4 [00:01<00:00, 2.57it/s]\n",
"m_pNameTable: 100%|██████████| 3/3 [00:00<00:00, 9.74it/s]\n",
"m_pNanoTable: 100%|██████████| 7/7 [00:00<00:00, 9.92it/s]\n",
"m_pNpcTable: 100%|██████████| 7/7 [00:01<00:00, 4.86it/s]\n",
"m_pShinyTable: 100%|██████████| 3/3 [00:00<00:00, 9.07it/s]\n",
"m_pSkillTable: 100%|██████████| 4/4 [00:00<00:00, 11.26it/s]\n",
"m_pConditionTable: 100%|██████████| 1/1 [00:00<00:00, 11.58it/s]\n",
"m_pTransportationTable: 100%|██████████| 7/7 [00:00<00:00, 10.35it/s]\n",
"m_pVendorTable: 100%|██████████| 1/1 [00:00<00:00, 4.68it/s]\n",
"m_pXComTable: 100%|██████████| 1/1 [00:00<00:00, 9.32it/s]\n",
"m_pBackItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.05it/s]\n",
"m_pFaceItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.85it/s]\n",
"m_pGlassItemTable: 100%|██████████| 5/5 [00:00<00:00, 8.95it/s]\n",
"m_pHatItemTable: 100%|██████████| 5/5 [00:00<00:00, 10.58it/s]\n",
"m_pHeadItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.31it/s]\n",
"m_pPantsItemTable: 100%|██████████| 5/5 [00:00<00:00, 8.16it/s]\n",
"m_pShirtsItemTable: 100%|██████████| 5/5 [00:00<00:00, 7.10it/s]\n",
"m_pShoesItemTable: 100%|██████████| 5/5 [00:00<00:00, 6.49it/s]\n",
"m_pWeaponItemTable: 100%|██████████| 5/5 [00:00<00:00, 6.26it/s]\n",
"m_pVehicleItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.41it/s]\n",
"m_pGeneralItemTable: 100%|██████████| 3/3 [00:00<00:00, 11.56it/s]\n",
"m_pChestItemTable: 100%|██████████| 3/3 [00:00<00:00, 6.83it/s]\n",
"m_pQuestItemTable: 100%|██████████| 3/3 [00:00<00:00, 11.19it/s]\n",
"m_pCreationItemTable: 100%|██████████| 1/1 [00:00<00:00, 12.50it/s]\n",
"m_pFirstUseTable: 100%|██████████| 2/2 [00:00<00:00, 10.72it/s]\n",
"m_pRulesTable: 100%|██████████| 2/2 [00:00<00:00, 8.38it/s]\n",
"m_pHelpTable: 100%|██████████| 5/5 [00:00<00:00, 9.11it/s]\n",
"m_pCutSceneTable: 100%|██████████| 1/1 [00:00<00:00, 11.51it/s]\n",
"m_pCombiningTable: 100%|██████████| 1/1 [00:00<00:00, 13.88it/s]\n",
"m_pFilterTable: 100%|██████████| 3/3 [00:00<00:00, 9.08it/s]\n",
"m_pClassTable: 100%|██████████| 4/4 [00:00<00:00, 10.94it/s]\n",
"m_pEnchantTable: 100%|██████████| 1/1 [00:00<00:00, 11.75it/s]\n",
"m_pClassSkillTable: 100%|██████████| 8/8 [00:00<00:00, 9.37it/s]\n",
"m_pSkillBookTable: 100%|██████████| 3/3 [00:00<00:00, 10.67it/s]\n"
]
}
],
"source": [
"xdt_path = \"tdata/xdt.json\"\n",
"conn = mysql.connector.connect(\n",
" host=\"localhost\",\n",
" user=\"root\",\n",
" password=\"mypassword\",\n",
" database=\"tabledata\"\n",
")\n",
"main(conn, xdt_path)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 2
}