scripts/json2xdb/prototyping.ipynb
2024-01-24 05:15:00 -05:00

284 lines
11 KiB
Plaintext

{
"cells": [
{
"cell_type": "code",
"execution_count": 59,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"import sys\n",
"from tqdm import tqdm\n",
"import mysql.connector"
]
},
{
"cell_type": "code",
"execution_count": 60,
"metadata": {},
"outputs": [],
"source": [
"def get_db_column_name(xdt_field_name):\n",
" # special case 1\n",
" if xdt_field_name == \"m_iitemID\":\n",
" return \"ItemID\"\n",
" \n",
" try:\n",
" # find the first uppercase character and split the string there\n",
" idx_of_first_uppercase = next(i for i, c in enumerate(xdt_field_name) if c.isupper())\n",
" except StopIteration:\n",
" # special case 2\n",
" if xdt_field_name == \"m_ibattery\":\n",
" idx_of_first_uppercase = 3\n",
" else:\n",
" print(f\"Could not find uppercase character in {xdt_field_name}\")\n",
" sys.exit(1)\n",
" prefix = xdt_field_name[:idx_of_first_uppercase]\n",
" db_field_name = xdt_field_name[idx_of_first_uppercase:]\n",
" return db_field_name"
]
},
{
"cell_type": "code",
"execution_count": 61,
"metadata": {},
"outputs": [],
"source": [
"def table_entry_to_tuple(table_entry):\n",
" vals = []\n",
" for field_name in table_entry:\n",
" field = table_entry[field_name]\n",
" vals.append(field)\n",
" return tuple(vals)\n",
"\n",
"def flatten_table_entry(table_entry):\n",
" flattened_entry = {}\n",
" for field_name in table_entry:\n",
" field = table_entry[field_name]\n",
" if type(field) == list:\n",
" for i, item in enumerate(field):\n",
" flattened_entry[f\"{field_name}{i}\"] = item\n",
" else:\n",
" flattened_entry[field_name] = field\n",
" return flattened_entry\n",
"\n",
"def handle_dict_table(table_entries, identifier_key, items_key):\n",
" new_table_entries = []\n",
" for table_entry in table_entries:\n",
" identifier = table_entry[identifier_key]\n",
" items = table_entry[items_key]\n",
" for item in items:\n",
" new_item = {}\n",
" new_item[identifier_key] = identifier # needs to be first\n",
" for field_name in item:\n",
" new_item[field_name] = item[field_name]\n",
" new_table_entries.append(new_item)\n",
" return new_table_entries\n"
]
},
{
"cell_type": "code",
"execution_count": 62,
"metadata": {},
"outputs": [],
"source": [
"def gen_column_sql(field_name, field_value):\n",
" field_type = type(field_value)\n",
" if field_type == int:\n",
" return f\"`{field_name}` INT,\"\n",
" elif field_type == float:\n",
" return f\"`{field_name}` FLOAT,\"\n",
" elif field_type == str:\n",
" # TODO maybe ascii vs unicode?\n",
" return f\"`{field_name}` TEXT,\"\n",
" else:\n",
" print(f\"Unknown type {field_type} for field {field_name}, skipping\")\n",
" return \"\""
]
},
{
"cell_type": "code",
"execution_count": 63,
"metadata": {},
"outputs": [],
"source": [
"def table_create(cursor, table_name, xdt_template_entry):\n",
" sql = f\"CREATE TABLE {table_name} (\"\n",
" sql += \"id INT AUTO_INCREMENT PRIMARY KEY,\"\n",
" for field_name in xdt_template_entry:\n",
" db_field_name = get_db_column_name(field_name)\n",
" val = xdt_template_entry[field_name]\n",
" sql += gen_column_sql(db_field_name, val)\n",
" sql = sql[:-1] # remove trailing comma\n",
" sql += \")\"\n",
" cursor.execute(sql)"
]
},
{
"cell_type": "code",
"execution_count": 64,
"metadata": {},
"outputs": [],
"source": [
"def table_populate(cursor, table_name, table_entries):\n",
" # generate the SQL first\n",
" sql = f\"INSERT INTO {table_name} (\"\n",
" template_entry = table_entries[0]\n",
" for field_name in template_entry:\n",
" db_field_name = get_db_column_name(field_name)\n",
" sql += f\"`{db_field_name}`,\"\n",
" sql = sql[:-1] # remove trailing comma\n",
" sql += \") VALUES (\"\n",
" for field_name in template_entry:\n",
" sql += f\"%s,\"\n",
" sql = sql[:-1] # remove trailing comma\n",
" sql += \")\"\n",
" \n",
" vals = [table_entry_to_tuple(entry) for entry in table_entries]\n",
" try:\n",
" cursor.executemany(sql, vals)\n",
" except Exception as e:\n",
" print(sql)\n",
" print(vals)\n",
" raise e"
]
},
{
"cell_type": "code",
"execution_count": 65,
"metadata": {},
"outputs": [],
"source": [
"def process_xdt_table(cursor, root, table_name, mappings):\n",
" table = root[table_name]\n",
" for (i, subtable_name) in tqdm(enumerate(table), desc=table_name, total=len(table)):\n",
" db_table_name = mappings[table_name][i]\n",
" #print(f\"{subtable_name} => {db_table_name}\")\n",
" \n",
" table_entries = table[subtable_name]\n",
" if db_table_name == \"CutSceneText\":\n",
" table_entries = handle_dict_table(table_entries, \"m_iEvent\", \"m_TextElement\")\n",
" table_entries = [flatten_table_entry(entry) for entry in table_entries]\n",
"\n",
" # clear the table\n",
" drop_sql = f\"DROP TABLE IF EXISTS {db_table_name}\"\n",
" cursor.execute(drop_sql)\n",
"\n",
" # create the table\n",
" table_create(cursor, db_table_name, table_entries[0])\n",
" table_populate(cursor, db_table_name, table_entries)"
]
},
{
"cell_type": "code",
"execution_count": 66,
"metadata": {},
"outputs": [],
"source": [
"def main(conn, xdt_path):\n",
" with open(\"mappings.json\", 'r') as f:\n",
" mappings = json.load(f)\n",
" with open(xdt_path, 'r') as f:\n",
" root = json.load(f)\n",
" cursor = conn.cursor()\n",
" for table_name in root:\n",
" if \"Table\" in table_name:\n",
" process_xdt_table(cursor, root, table_name, mappings)\n",
" conn.commit()"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"m_pAnimationTable: 100%|██████████| 3/3 [00:00<00:00, 9.30it/s]\n",
"m_pAvatarTable: 100%|██████████| 2/2 [00:00<00:00, 5.76it/s]\n",
"m_pChatTable: 100%|██████████| 7/7 [00:00<00:00, 9.55it/s]\n",
"m_pEmoteTable: 100%|██████████| 2/2 [00:00<00:00, 12.84it/s]\n",
"m_pGuideTable: 100%|██████████| 2/2 [00:00<00:00, 11.23it/s]\n",
"m_pInstanceTable: 100%|██████████| 3/3 [00:00<00:00, 9.99it/s]\n",
"m_pMessageTable: 100%|██████████| 1/1 [00:00<00:00, 8.64it/s]\n",
"m_pMissionTable: 100%|██████████| 4/4 [00:01<00:00, 2.57it/s]\n",
"m_pNameTable: 100%|██████████| 3/3 [00:00<00:00, 9.74it/s]\n",
"m_pNanoTable: 100%|██████████| 7/7 [00:00<00:00, 9.92it/s]\n",
"m_pNpcTable: 100%|██████████| 7/7 [00:01<00:00, 4.86it/s]\n",
"m_pShinyTable: 100%|██████████| 3/3 [00:00<00:00, 9.07it/s]\n",
"m_pSkillTable: 100%|██████████| 4/4 [00:00<00:00, 11.26it/s]\n",
"m_pConditionTable: 100%|██████████| 1/1 [00:00<00:00, 11.58it/s]\n",
"m_pTransportationTable: 100%|██████████| 7/7 [00:00<00:00, 10.35it/s]\n",
"m_pVendorTable: 100%|██████████| 1/1 [00:00<00:00, 4.68it/s]\n",
"m_pXComTable: 100%|██████████| 1/1 [00:00<00:00, 9.32it/s]\n",
"m_pBackItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.05it/s]\n",
"m_pFaceItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.85it/s]\n",
"m_pGlassItemTable: 100%|██████████| 5/5 [00:00<00:00, 8.95it/s]\n",
"m_pHatItemTable: 100%|██████████| 5/5 [00:00<00:00, 10.58it/s]\n",
"m_pHeadItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.31it/s]\n",
"m_pPantsItemTable: 100%|██████████| 5/5 [00:00<00:00, 8.16it/s]\n",
"m_pShirtsItemTable: 100%|██████████| 5/5 [00:00<00:00, 7.10it/s]\n",
"m_pShoesItemTable: 100%|██████████| 5/5 [00:00<00:00, 6.49it/s]\n",
"m_pWeaponItemTable: 100%|██████████| 5/5 [00:00<00:00, 6.26it/s]\n",
"m_pVehicleItemTable: 100%|██████████| 5/5 [00:00<00:00, 9.41it/s]\n",
"m_pGeneralItemTable: 100%|██████████| 3/3 [00:00<00:00, 11.56it/s]\n",
"m_pChestItemTable: 100%|██████████| 3/3 [00:00<00:00, 6.83it/s]\n",
"m_pQuestItemTable: 100%|██████████| 3/3 [00:00<00:00, 11.19it/s]\n",
"m_pCreationItemTable: 100%|██████████| 1/1 [00:00<00:00, 12.50it/s]\n",
"m_pFirstUseTable: 100%|██████████| 2/2 [00:00<00:00, 10.72it/s]\n",
"m_pRulesTable: 100%|██████████| 2/2 [00:00<00:00, 8.38it/s]\n",
"m_pHelpTable: 100%|██████████| 5/5 [00:00<00:00, 9.11it/s]\n",
"m_pCutSceneTable: 100%|██████████| 1/1 [00:00<00:00, 11.51it/s]\n",
"m_pCombiningTable: 100%|██████████| 1/1 [00:00<00:00, 13.88it/s]\n",
"m_pFilterTable: 100%|██████████| 3/3 [00:00<00:00, 9.08it/s]\n",
"m_pClassTable: 100%|██████████| 4/4 [00:00<00:00, 10.94it/s]\n",
"m_pEnchantTable: 100%|██████████| 1/1 [00:00<00:00, 11.75it/s]\n",
"m_pClassSkillTable: 100%|██████████| 8/8 [00:00<00:00, 9.37it/s]\n",
"m_pSkillBookTable: 100%|██████████| 3/3 [00:00<00:00, 10.67it/s]\n"
]
}
],
"source": [
"xdt_path = \"tdata/xdt.json\"\n",
"conn = mysql.connector.connect(\n",
" host=\"localhost\",\n",
" user=\"root\",\n",
" password=\"mypassword\",\n",
" database=\"tabledata\"\n",
")\n",
"main(conn, xdt_path)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 2
}