284 lines
11 KiB
Plaintext
284 lines
11 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 59,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import json\n",
|
|
"import sys\n",
|
|
"from tqdm import tqdm\n",
|
|
"import mysql.connector"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 60,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def get_db_column_name(xdt_field_name):\n",
|
|
" # special case 1\n",
|
|
" if xdt_field_name == \"m_iitemID\":\n",
|
|
" return \"ItemID\"\n",
|
|
" \n",
|
|
" try:\n",
|
|
" # find the first uppercase character and split the string there\n",
|
|
" idx_of_first_uppercase = next(i for i, c in enumerate(xdt_field_name) if c.isupper())\n",
|
|
" except StopIteration:\n",
|
|
" # special case 2\n",
|
|
" if xdt_field_name == \"m_ibattery\":\n",
|
|
" idx_of_first_uppercase = 3\n",
|
|
" else:\n",
|
|
" print(f\"Could not find uppercase character in {xdt_field_name}\")\n",
|
|
" sys.exit(1)\n",
|
|
" prefix = xdt_field_name[:idx_of_first_uppercase]\n",
|
|
" db_field_name = xdt_field_name[idx_of_first_uppercase:]\n",
|
|
" return db_field_name"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 61,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def table_entry_to_tuple(table_entry):\n",
|
|
" vals = []\n",
|
|
" for field_name in table_entry:\n",
|
|
" field = table_entry[field_name]\n",
|
|
" vals.append(field)\n",
|
|
" return tuple(vals)\n",
|
|
"\n",
|
|
"def flatten_table_entry(table_entry):\n",
|
|
" flattened_entry = {}\n",
|
|
" for field_name in table_entry:\n",
|
|
" field = table_entry[field_name]\n",
|
|
" if type(field) == list:\n",
|
|
" for i, item in enumerate(field):\n",
|
|
" flattened_entry[f\"{field_name}{i}\"] = item\n",
|
|
" else:\n",
|
|
" flattened_entry[field_name] = field\n",
|
|
" return flattened_entry\n",
|
|
"\n",
|
|
"def handle_dict_table(table_entries, identifier_key, items_key):\n",
|
|
" new_table_entries = []\n",
|
|
" for table_entry in table_entries:\n",
|
|
" identifier = table_entry[identifier_key]\n",
|
|
" items = table_entry[items_key]\n",
|
|
" for item in items:\n",
|
|
" new_item = {}\n",
|
|
" new_item[identifier_key] = identifier # needs to be first\n",
|
|
" for field_name in item:\n",
|
|
" new_item[field_name] = item[field_name]\n",
|
|
" new_table_entries.append(new_item)\n",
|
|
" return new_table_entries\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 62,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def gen_column_sql(field_name, field_value):\n",
|
|
" field_type = type(field_value)\n",
|
|
" if field_type == int:\n",
|
|
" return f\"`{field_name}` INT,\"\n",
|
|
" elif field_type == float:\n",
|
|
" return f\"`{field_name}` FLOAT,\"\n",
|
|
" elif field_type == str:\n",
|
|
" # TODO maybe ascii vs unicode?\n",
|
|
" return f\"`{field_name}` TEXT,\"\n",
|
|
" else:\n",
|
|
" print(f\"Unknown type {field_type} for field {field_name}, skipping\")\n",
|
|
" return \"\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 63,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def table_create(cursor, table_name, xdt_template_entry):\n",
|
|
" sql = f\"CREATE TABLE {table_name} (\"\n",
|
|
" sql += \"id INT AUTO_INCREMENT PRIMARY KEY,\"\n",
|
|
" for field_name in xdt_template_entry:\n",
|
|
" db_field_name = get_db_column_name(field_name)\n",
|
|
" val = xdt_template_entry[field_name]\n",
|
|
" sql += gen_column_sql(db_field_name, val)\n",
|
|
" sql = sql[:-1] # remove trailing comma\n",
|
|
" sql += \")\"\n",
|
|
" cursor.execute(sql)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 64,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def table_populate(cursor, table_name, table_entries):\n",
|
|
" # generate the SQL first\n",
|
|
" sql = f\"INSERT INTO {table_name} (\"\n",
|
|
" template_entry = table_entries[0]\n",
|
|
" for field_name in template_entry:\n",
|
|
" db_field_name = get_db_column_name(field_name)\n",
|
|
" sql += f\"`{db_field_name}`,\"\n",
|
|
" sql = sql[:-1] # remove trailing comma\n",
|
|
" sql += \") VALUES (\"\n",
|
|
" for field_name in template_entry:\n",
|
|
" sql += f\"%s,\"\n",
|
|
" sql = sql[:-1] # remove trailing comma\n",
|
|
" sql += \")\"\n",
|
|
" \n",
|
|
" vals = [table_entry_to_tuple(entry) for entry in table_entries]\n",
|
|
" try:\n",
|
|
" cursor.executemany(sql, vals)\n",
|
|
" except Exception as e:\n",
|
|
" print(sql)\n",
|
|
" print(vals)\n",
|
|
" raise e"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 65,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def process_xdt_table(cursor, root, table_name, mappings):\n",
|
|
" table = root[table_name]\n",
|
|
" for (i, subtable_name) in tqdm(enumerate(table), desc=table_name, total=len(table)):\n",
|
|
" db_table_name = mappings[table_name][i]\n",
|
|
" #print(f\"{subtable_name} => {db_table_name}\")\n",
|
|
" \n",
|
|
" table_entries = table[subtable_name]\n",
|
|
" if db_table_name == \"CutSceneText\":\n",
|
|
" table_entries = handle_dict_table(table_entries, \"m_iEvent\", \"m_TextElement\")\n",
|
|
" table_entries = [flatten_table_entry(entry) for entry in table_entries]\n",
|
|
"\n",
|
|
" # clear the table\n",
|
|
" drop_sql = f\"DROP TABLE IF EXISTS {db_table_name}\"\n",
|
|
" cursor.execute(drop_sql)\n",
|
|
"\n",
|
|
" # create the table\n",
|
|
" table_create(cursor, db_table_name, table_entries[0])\n",
|
|
" table_populate(cursor, db_table_name, table_entries)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 66,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def main(conn, xdt_path):\n",
|
|
" with open(\"mappings.json\", 'r') as f:\n",
|
|
" mappings = json.load(f)\n",
|
|
" with open(xdt_path, 'r') as f:\n",
|
|
" root = json.load(f)\n",
|
|
" cursor = conn.cursor()\n",
|
|
" for table_name in root:\n",
|
|
" if \"Table\" in table_name:\n",
|
|
" process_xdt_table(cursor, root, table_name, mappings)\n",
|
|
" conn.commit()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 67,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"m_pAnimationTable: 100%|██████████| 3/3 [00:00<00:00, 9.30it/s]\n",
|
|
"m_pAvatarTable: 100%|██████████| 2/2 [00:00<00:00, 5.76it/s]\n",
|
|
"m_pChatTable: 100%|██████████| 7/7 [00:00<00:00, 9.55it/s]\n",
|
|
"m_pEmoteTable: 100%|██████████| 2/2 [00:00<00:00, 12.84it/s]\n",
|
|
"m_pGuideTable: 100%|██████████| 2/2 [00:00<00:00, 11.23it/s]\n",
|
|
"m_pInstanceTable: 100%|██████████| 3/3 [00:00<00:00, 9.99it/s]\n",
|
|
"m_pMessageTable: 100%|██████████| 1/1 [00:00<00:00, 8.64it/s]\n",
|
|
"m_pMissionTable: 100%|██████████| 4/4 [00:01<00:00, 2.57it/s]\n",
|
|
"m_pNameTable: 100%|██████████| 3/3 [00:00<00:00, 9.74it/s]\n",
|
|
"m_pNanoTable: 100%|██████████| 7/7 [00:00<00:00, 9.92it/s]\n",
|
|
"m_pNpcTable: 100%|██████████| 7/7 [00:01<00:00, 4.86it/s]\n",
|
|
"m_pShinyTable: 100%|██████████| 3/3 [00:00<00:00, 9.07it/s]\n",
|
|
"m_pSkillTable: 100%|██████████| 4/4 [00:00<00:00, 11.26it/s]\n",
|
|
"m_pConditionTable: 100%|██████████| 1/1 [00:00<00:00, 11.58it/s]\n",
|
|
"m_pTransportationTable: 100%|██████████| 7/7 [00:00<00:00, 10.35it/s]\n",
|
|
"m_pVendorTable: 100%|██████████| 1/1 [00:00<00:00, 4.68it/s]\n",
|
|
"m_pXComTable: 100%|██████████| 1/1 [00:00<00:00, 9.32it/s]\n",
|
|
"m_pBackItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.05it/s]\n",
|
|
"m_pFaceItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.85it/s]\n",
|
|
"m_pGlassItemTable: 100%|██████████| 5/5 [00:00<00:00, 8.95it/s]\n",
|
|
"m_pHatItemTable: 100%|██████████| 5/5 [00:00<00:00, 10.58it/s]\n",
|
|
"m_pHeadItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.31it/s]\n",
|
|
"m_pPantsItemTable: 100%|██████████| 5/5 [00:00<00:00, 8.16it/s]\n",
|
|
"m_pShirtsItemTable: 100%|██████████| 5/5 [00:00<00:00, 7.10it/s]\n",
|
|
"m_pShoesItemTable: 100%|██████████| 5/5 [00:00<00:00, 6.49it/s]\n",
|
|
"m_pWeaponItemTable: 100%|██████████| 5/5 [00:00<00:00, 6.26it/s]\n",
|
|
"m_pVehicleItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.41it/s]\n",
|
|
"m_pGeneralItemTable: 100%|██████████| 3/3 [00:00<00:00, 11.56it/s]\n",
|
|
"m_pChestItemTable: 100%|██████████| 3/3 [00:00<00:00, 6.83it/s]\n",
|
|
"m_pQuestItemTable: 100%|██████████| 3/3 [00:00<00:00, 11.19it/s]\n",
|
|
"m_pCreationItemTable: 100%|██████████| 1/1 [00:00<00:00, 12.50it/s]\n",
|
|
"m_pFirstUseTable: 100%|██████████| 2/2 [00:00<00:00, 10.72it/s]\n",
|
|
"m_pRulesTable: 100%|██████████| 2/2 [00:00<00:00, 8.38it/s]\n",
|
|
"m_pHelpTable: 100%|██████████| 5/5 [00:00<00:00, 9.11it/s]\n",
|
|
"m_pCutSceneTable: 100%|██████████| 1/1 [00:00<00:00, 11.51it/s]\n",
|
|
"m_pCombiningTable: 100%|██████████| 1/1 [00:00<00:00, 13.88it/s]\n",
|
|
"m_pFilterTable: 100%|██████████| 3/3 [00:00<00:00, 9.08it/s]\n",
|
|
"m_pClassTable: 100%|██████████| 4/4 [00:00<00:00, 10.94it/s]\n",
|
|
"m_pEnchantTable: 100%|██████████| 1/1 [00:00<00:00, 11.75it/s]\n",
|
|
"m_pClassSkillTable: 100%|██████████| 8/8 [00:00<00:00, 9.37it/s]\n",
|
|
"m_pSkillBookTable: 100%|██████████| 3/3 [00:00<00:00, 10.67it/s]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"xdt_path = \"tdata/xdt.json\"\n",
|
|
"conn = mysql.connector.connect(\n",
|
|
" host=\"localhost\",\n",
|
|
" user=\"root\",\n",
|
|
" password=\"mypassword\",\n",
|
|
" database=\"tabledata\"\n",
|
|
")\n",
|
|
"main(conn, xdt_path)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": ".venv",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.12"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|