{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import sys\n",
    "from tqdm import tqdm\n",
    "import mysql.connector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_db_column_name(xdt_field_name):\n",
    "    # special case 1\n",
    "    if xdt_field_name == \"m_iitemID\":\n",
    "        return \"ItemID\"\n",
    "    \n",
    "    try:\n",
    "        # find the first uppercase character and split the string there\n",
    "        idx_of_first_uppercase = next(i for i, c in enumerate(xdt_field_name) if c.isupper())\n",
    "    except StopIteration:\n",
    "        # special case 2\n",
    "        if xdt_field_name == \"m_ibattery\":\n",
    "            idx_of_first_uppercase = 3\n",
    "        else:\n",
    "            print(f\"Could not find uppercase character in {xdt_field_name}\")\n",
    "            sys.exit(1)\n",
    "    prefix = xdt_field_name[:idx_of_first_uppercase]\n",
    "    db_field_name = xdt_field_name[idx_of_first_uppercase:]\n",
    "    return db_field_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def table_entry_to_tuple(table_entry):\n",
    "    vals = []\n",
    "    for field_name in table_entry:\n",
    "        field = table_entry[field_name]\n",
    "        vals.append(field)\n",
    "    return tuple(vals)\n",
    "\n",
    "def flatten_table_entry(table_entry):\n",
    "    flattened_entry = {}\n",
    "    for field_name in table_entry:\n",
    "        field = table_entry[field_name]\n",
    "        if type(field) == list:\n",
    "            for i, item in enumerate(field):\n",
    "                flattened_entry[f\"{field_name}{i}\"] = item\n",
    "        else:\n",
    "            flattened_entry[field_name] = field\n",
    "    return flattened_entry\n",
    "\n",
    "def handle_dict_table(table_entries, identifier_key, items_key):\n",
    "    new_table_entries = []\n",
    "    for table_entry in table_entries:\n",
    "        identifier = table_entry[identifier_key]\n",
    "        items = table_entry[items_key]\n",
    "        for item in items:\n",
    "            new_item = {}\n",
    "            new_item[identifier_key] = identifier # needs to be first\n",
    "            for field_name in item:\n",
    "                new_item[field_name] = item[field_name]\n",
    "            new_table_entries.append(new_item)\n",
    "    return new_table_entries\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_column_sql(field_name, field_value):\n",
    "    field_type = type(field_value)\n",
    "    if field_type == int:\n",
    "        return f\"`{field_name}` INT,\"\n",
    "    elif field_type == float:\n",
    "        return f\"`{field_name}` FLOAT,\"\n",
    "    elif field_type == str:\n",
    "        # TODO maybe ascii vs unicode?\n",
    "        return f\"`{field_name}` TEXT,\"\n",
    "    else:\n",
    "        print(f\"Unknown type {field_type} for field {field_name}, skipping\")\n",
    "        return \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def table_create(cursor, table_name, xdt_template_entry):\n",
    "    sql = f\"CREATE TABLE {table_name} (\"\n",
    "    sql += \"id INT AUTO_INCREMENT PRIMARY KEY,\"\n",
    "    for field_name in xdt_template_entry:\n",
    "        db_field_name = get_db_column_name(field_name)\n",
    "        val = xdt_template_entry[field_name]\n",
    "        sql += gen_column_sql(db_field_name, val)\n",
    "    sql = sql[:-1] # remove trailing comma\n",
    "    sql += \")\"\n",
    "    cursor.execute(sql)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def table_populate(cursor, table_name, table_entries):\n",
    "    # generate the SQL first\n",
    "    sql = f\"INSERT INTO {table_name} (\"\n",
    "    template_entry = table_entries[0]\n",
    "    for field_name in template_entry:\n",
    "        db_field_name = get_db_column_name(field_name)\n",
    "        sql += f\"`{db_field_name}`,\"\n",
    "    sql = sql[:-1] # remove trailing comma\n",
    "    sql += \") VALUES (\"\n",
    "    for field_name in template_entry:\n",
    "        sql += f\"%s,\"\n",
    "    sql = sql[:-1] # remove trailing comma\n",
    "    sql += \")\"\n",
    "    \n",
    "    vals = [table_entry_to_tuple(entry) for entry in table_entries]\n",
    "    try:\n",
    "        cursor.executemany(sql, vals)\n",
    "    except Exception as e:\n",
    "        print(sql)\n",
    "        print(vals)\n",
    "        raise e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_xdt_table(cursor, root, table_name, mappings):\n",
    "    table = root[table_name]\n",
    "    for (i, subtable_name) in tqdm(enumerate(table), desc=table_name, total=len(table)):\n",
    "        db_table_name = mappings[table_name][i]\n",
    "        #print(f\"{subtable_name} => {db_table_name}\")\n",
    "        \n",
    "        table_entries = table[subtable_name]\n",
    "        if db_table_name == \"CutSceneText\":\n",
    "            table_entries = handle_dict_table(table_entries, \"m_iEvent\", \"m_TextElement\")\n",
    "        table_entries = [flatten_table_entry(entry) for entry in table_entries]\n",
    "\n",
    "        # clear the table\n",
    "        drop_sql = f\"DROP TABLE IF EXISTS {db_table_name}\"\n",
    "        cursor.execute(drop_sql)\n",
    "\n",
    "        # create the table\n",
    "        table_create(cursor, db_table_name, table_entries[0])\n",
    "        table_populate(cursor, db_table_name, table_entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(conn, xdt_path):\n",
    "    with open(\"mappings.json\", 'r') as f:\n",
    "        mappings = json.load(f)\n",
    "    with open(xdt_path, 'r') as f:\n",
    "        root = json.load(f)\n",
    "    cursor = conn.cursor()\n",
    "    for table_name in root:\n",
    "        if \"Table\" in table_name:\n",
    "            process_xdt_table(cursor, root, table_name, mappings)\n",
    "    conn.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "m_pAnimationTable: 100%|██████████| 3/3 [00:00<00:00,  9.30it/s]\n",
      "m_pAvatarTable: 100%|██████████| 2/2 [00:00<00:00,  5.76it/s]\n",
      "m_pChatTable: 100%|██████████| 7/7 [00:00<00:00,  9.55it/s]\n",
      "m_pEmoteTable: 100%|██████████| 2/2 [00:00<00:00, 12.84it/s]\n",
      "m_pGuideTable: 100%|██████████| 2/2 [00:00<00:00, 11.23it/s]\n",
      "m_pInstanceTable: 100%|██████████| 3/3 [00:00<00:00,  9.99it/s]\n",
      "m_pMessageTable: 100%|██████████| 1/1 [00:00<00:00,  8.64it/s]\n",
      "m_pMissionTable: 100%|██████████| 4/4 [00:01<00:00,  2.57it/s]\n",
      "m_pNameTable: 100%|██████████| 3/3 [00:00<00:00,  9.74it/s]\n",
      "m_pNanoTable: 100%|██████████| 7/7 [00:00<00:00,  9.92it/s]\n",
      "m_pNpcTable: 100%|██████████| 7/7 [00:01<00:00,  4.86it/s]\n",
      "m_pShinyTable: 100%|██████████| 3/3 [00:00<00:00,  9.07it/s]\n",
      "m_pSkillTable: 100%|██████████| 4/4 [00:00<00:00, 11.26it/s]\n",
      "m_pConditionTable: 100%|██████████| 1/1 [00:00<00:00, 11.58it/s]\n",
      "m_pTransportationTable: 100%|██████████| 7/7 [00:00<00:00, 10.35it/s]\n",
      "m_pVendorTable: 100%|██████████| 1/1 [00:00<00:00,  4.68it/s]\n",
      "m_pXComTable: 100%|██████████| 1/1 [00:00<00:00,  9.32it/s]\n",
      "m_pBackItemTable: 100%|██████████| 5/5 [00:00<00:00,  9.05it/s]\n",
      "m_pFaceItemTable: 100%|██████████| 5/5 [00:00<00:00,  9.85it/s]\n",
      "m_pGlassItemTable: 100%|██████████| 5/5 [00:00<00:00,  8.95it/s]\n",
      "m_pHatItemTable: 100%|██████████| 5/5 [00:00<00:00, 10.58it/s]\n",
      "m_pHeadItemTable: 100%|██████████| 5/5 [00:00<00:00,  9.31it/s]\n",
      "m_pPantsItemTable: 100%|██████████| 5/5 [00:00<00:00,  8.16it/s]\n",
      "m_pShirtsItemTable: 100%|██████████| 5/5 [00:00<00:00,  7.10it/s]\n",
      "m_pShoesItemTable: 100%|██████████| 5/5 [00:00<00:00,  6.49it/s]\n",
      "m_pWeaponItemTable: 100%|██████████| 5/5 [00:00<00:00,  6.26it/s]\n",
      "m_pVehicleItemTable: 100%|██████████| 5/5 [00:00<00:00,  9.41it/s]\n",
      "m_pGeneralItemTable: 100%|██████████| 3/3 [00:00<00:00, 11.56it/s]\n",
      "m_pChestItemTable: 100%|██████████| 3/3 [00:00<00:00,  6.83it/s]\n",
      "m_pQuestItemTable: 100%|██████████| 3/3 [00:00<00:00, 11.19it/s]\n",
      "m_pCreationItemTable: 100%|██████████| 1/1 [00:00<00:00, 12.50it/s]\n",
      "m_pFirstUseTable: 100%|██████████| 2/2 [00:00<00:00, 10.72it/s]\n",
      "m_pRulesTable: 100%|██████████| 2/2 [00:00<00:00,  8.38it/s]\n",
      "m_pHelpTable: 100%|██████████| 5/5 [00:00<00:00,  9.11it/s]\n",
      "m_pCutSceneTable: 100%|██████████| 1/1 [00:00<00:00, 11.51it/s]\n",
      "m_pCombiningTable: 100%|██████████| 1/1 [00:00<00:00, 13.88it/s]\n",
      "m_pFilterTable: 100%|██████████| 3/3 [00:00<00:00,  9.08it/s]\n",
      "m_pClassTable: 100%|██████████| 4/4 [00:00<00:00, 10.94it/s]\n",
      "m_pEnchantTable: 100%|██████████| 1/1 [00:00<00:00, 11.75it/s]\n",
      "m_pClassSkillTable: 100%|██████████| 8/8 [00:00<00:00,  9.37it/s]\n",
      "m_pSkillBookTable: 100%|██████████| 3/3 [00:00<00:00, 10.67it/s]\n"
     ]
    }
   ],
   "source": [
    "xdt_path = \"tdata/xdt.json\"\n",
    "conn = mysql.connector.connect(\n",
    "    host=\"localhost\",\n",
    "    user=\"root\",\n",
    "    password=\"mypassword\",\n",
    "    database=\"tabledata\"\n",
    ")\n",
    "main(conn, xdt_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}