Schemas, split fields, and padding

json2xdb/apply_schema.ipynb

@@ -0,0 +1,130 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"xdt.json\", \"r\") as f:\n",
+    "    xdt = json.load(f)\n",
+    "with open(\"mappings.json\", \"r\") as f:\n",
+    "    mappings = json.load(f)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def apply_schema(schema, entry):\n",
+    "    fixed_entry = {}\n",
+    "    padding = 0\n",
+    "    for field_name in schema:\n",
+    "        if field_name is None:\n",
+    "            fixed_entry[f\"m_iPadding{padding}\"] = 0\n",
+    "            continue\n",
+    "\n",
+    "        if field_name in entry:\n",
+    "            fixed_entry[field_name] = entry[field_name]\n",
+    "        else:\n",
+    "            print(\"Missing field: {}\".format(field_name))\n",
+    "    return fixed_entry"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['m_iNpcNumber', 'm_iDifficulty', 'm_iNpcName', 'm_iComment', 'm_iTeam', 'm_iNpcLevel', 'm_iNpcType', 'm_iHNpc', 'm_iHNpcNum', 'm_iNpcStyle', 'm_iLegStyle', 'm_iAiType', 'm_iHP', 'm_iHPRegen', 'm_iDropType', 'm_iRegenTime', 'm_iHeight', 'm_iRadius', 'm_fScale', 'm_iPower', 'm_iAccuracy', 'm_iProtection', 'm_iDodge', 'm_iWalkSpeed', 'm_iRunSpeed', 'm_fAnimationSpeed', 'm_fWalkAnimationSpeed', 'm_fRunAnimationSpeed', 'm_iSwimSpeed', 'm_iJumpHeight', 'm_iJumpDistance', 'm_iSightRange', 'm_iIdleRange', 'm_iCombatRange', 'm_iAtkRange', 'm_iAtkAngle', 'm_iEffectArea', 'm_iTargetMode', 'm_iTargetNumber', 'm_iInitalTime', 'm_iDeliverTime', 'm_iDelayTime', 'm_iDurationTime', 'm_iMegaType', 'm_iMegaTypeProb', 'm_iMegaAni', 'm_iMegaString', 'm_iCorruptionType', 'm_iCorruptionTypeProb', 'm_iCorruptionString', 'm_iActiveSkill1', 'm_iActiveSkill1Prob', 'm_iActiveSkill1Ani', 'm_iActiveSkill1String', 'm_iActiveSkill2', 'm_iActiveSkill2Prob', 'm_iActiveSkill2Ani', 'm_iActiveSkill2String', 'm_iSupportSkill', 'm_iSupportSkillAni', 'm_iSupportSkillString', 'm_iPassiveBuff', 'm_iMesh', 'm_iIcon1', 'm_iMapIcon', 'm_iEffect', 'm_iSound', 'm_iBarkerType', 'm_iBarkerNumber', 'm_iServiceNumber']\n",
+      "{'m_iNpcNumber': 3, 'm_iDifficulty': 0, 'm_iNpcName': 3, 'm_iComment': 3, 'm_iTeam': 2, 'm_iNpcLevel': 8, 'm_iNpcType': 0, 'm_iHNpc': 0, 'm_iHNpcNum': 0, 'm_iNpcStyle': 0, 'm_iAiType': 2, 'm_iHP': 1685, 'm_iHPRegen': 0, 'm_iDropType': 20, 'm_iRegenTime': 300000000, 'm_iHeight': 240, 'm_iRadius': 100, 'm_fScale': 1.8600000143051147, 'm_iPower': -145, 'm_iAccuracy': 45, 'm_iProtection': 215, 'm_iDodge': 235, 'm_iRunSpeed': 500, 'm_iSwimSpeed': 0, 'm_iJumpHeight': 0, 'm_iJumpDistance': 0, 'm_iSightRange': 1000, 'm_iIdleRange': 300, 'm_iCombatRange': 4000, 'm_iAtkRange': 1100, 'm_iAtkAngle': 100, 'm_iAtkRate': 0, 'm_iEffectArea': 100, 'm_iTargetMode': 1, 'm_iTargetNumber': 1, 'm_iInitalTime': 15, 'm_iDeliverTime': 0, 'm_iDelayTime': 12, 'm_iDurationTime': 0, 'm_iMegaType': 176, 'm_iMegaTypeProb': 100000, 'm_iCorruptionType': 180, 'm_iCorruptionTypeProb': 100000, 'm_iActiveSkill1': 0, 'm_iActiveSkill1Prob': 0, 'm_iActiveSkill2': 0, 'm_iActiveSkill2Prob': 0, 'm_iActiveSkill3': 0, 'm_iActiveSkill3Prob': 0, 'm_iSupportSkill': 0, 'm_iPassiveBuff': 160, 'm_iNeck': 0, 'm_iMesh': 343, 'm_iTexture': 0, 'm_iTexture2': 0, 'm_iIcon1': 342, 'm_iEffect': 66, 'm_iSound': 2, 'm_fAnimationSpeed': 1.0, 'm_iWalkSpeed': 300, 'm_fWalkAnimationSpeed': 0.9399999976158142, 'm_fRunAnimationSpeed': 1.0, 'm_iMapIcon': 19, 'm_iLegStyle': 1, 'm_iBarkerType': 0, 'm_iMegaAni': 0, 'm_iActiveSkill1Ani': 1, 'm_iActiveSkill2Ani': 2, 'm_iSupportSkillAni': 3, 'm_iMegaString': 0, 'm_iCorruptionString': 0, 'm_iActiveSkill1String': 0, 'm_iActiveSkill2String': 0, 'm_iSupportSkillString': 0, 'm_iBarkerNumber': 0, 'm_iServiceNumber': 0}\n",
+      "{'m_iNpcNumber': 3, 'm_iDifficulty': 0, 'm_iNpcName': 3, 'm_iComment': 3, 'm_iTeam': 2, 'm_iNpcLevel': 8, 'm_iNpcType': 0, 'm_iHNpc': 0, 'm_iHNpcNum': 0, 'm_iNpcStyle': 0, 'm_iLegStyle': 1, 'm_iAiType': 2, 'm_iHP': 1685, 'm_iHPRegen': 0, 'm_iDropType': 20, 'm_iRegenTime': 300000000, 'm_iHeight': 240, 'm_iRadius': 100, 'm_fScale': 1.8600000143051147, 'm_iPower': -145, 'm_iAccuracy': 45, 'm_iProtection': 215, 'm_iDodge': 235, 'm_iWalkSpeed': 300, 'm_iRunSpeed': 500, 'm_fAnimationSpeed': 1.0, 'm_fWalkAnimationSpeed': 0.9399999976158142, 'm_fRunAnimationSpeed': 1.0, 'm_iSwimSpeed': 0, 'm_iJumpHeight': 0, 'm_iJumpDistance': 0, 'm_iSightRange': 1000, 'm_iIdleRange': 300, 'm_iCombatRange': 4000, 'm_iAtkRange': 1100, 'm_iAtkAngle': 100, 'm_iEffectArea': 100, 'm_iTargetMode': 1, 'm_iTargetNumber': 1, 'm_iInitalTime': 15, 'm_iDeliverTime': 0, 'm_iDelayTime': 12, 'm_iDurationTime': 0, 'm_iMegaType': 176, 'm_iMegaTypeProb': 100000, 'm_iMegaAni': 0, 'm_iMegaString': 0, 'm_iCorruptionType': 180, 'm_iCorruptionTypeProb': 100000, 'm_iCorruptionString': 0, 'm_iActiveSkill1': 0, 'm_iActiveSkill1Prob': 0, 'm_iActiveSkill1Ani': 1, 'm_iActiveSkill1String': 0, 'm_iActiveSkill2': 0, 'm_iActiveSkill2Prob': 0, 'm_iActiveSkill2Ani': 2, 'm_iActiveSkill2String': 0, 'm_iSupportSkill': 0, 'm_iSupportSkillAni': 3, 'm_iSupportSkillString': 0, 'm_iPassiveBuff': 160, 'm_iMesh': 343, 'm_iIcon1': 342, 'm_iMapIcon': 19, 'm_iEffect': 66, 'm_iSound': 2, 'm_iBarkerType': 0, 'm_iBarkerNumber': 0, 'm_iServiceNumber': 0}\n"
+     ]
+    }
+   ],
+   "source": [
+    "table_name = \"m_pNpcTable\"\n",
+    "subtable_name = \"m_pNpcData\"\n",
+    "db_table_name = mappings[table_name][subtable_name]\n",
+    "test_entry = xdt[table_name][subtable_name][3]\n",
+    "with open(f\"schema/{db_table_name}.json\", 'r') as f:\n",
+    "    schema = json.load(f)\n",
+    "print(schema)\n",
+    "print(test_entry)\n",
+    "fixed_entry = apply_schema(schema, test_entry)\n",
+    "print(fixed_entry)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['m_iNpcNumber', 'm_iDifficulty', 'm_iNpcName', 'm_iComment', 'm_iTeam', 'm_iNpcLevel', 'm_iNpcType', 'm_iHNpc', 'm_iHNpcNum', 'm_iNpcStyle', 'm_iLegStyle', 'm_iAiType', 'm_iHP', 'm_iHPRegen', 'm_iDropType', 'm_iRegenTime', 'm_iHeight', 'm_iRadius', 'm_fScale', 'm_iPower', 'm_iAccuracy', 'm_iProtection', 'm_iDodge', 'm_iWalkSpeed', 'm_iRunSpeed', 'm_fAnimationSpeed', 'm_fWalkAnimationSpeed', 'm_fRunAnimationSpeed', 'm_iSwimSpeed', 'm_iJumpHeight', 'm_iJumpDistance', 'm_iSightRange', 'm_iIdleRange', 'm_iCombatRange', 'm_iAtkRange', 'm_iAtkAngle', 'm_iEffectArea', 'm_iTargetMode', 'm_iTargetNumber', 'm_iInitalTime', 'm_iDeliverTime', 'm_iDelayTime', 'm_iDurationTime', 'm_iMegaType', 'm_iMegaTypeProb', 'm_iMegaAni', 'm_iMegaString', 'm_iCorruptionType', 'm_iCorruptionTypeProb', 'm_iCorruptionString', 'm_iActiveSkill1', 'm_iActiveSkill1Prob', 'm_iActiveSkill1Ani', 'm_iActiveSkill1String', 'm_iActiveSkill2', 'm_iActiveSkill2Prob', 'm_iActiveSkill2Ani', 'm_iActiveSkill2String', 'm_iSupportSkill', 'm_iSupportSkillAni', 'm_iSupportSkillString', 'm_iPassiveBuff', 'm_iMesh', 'm_iIcon1', 'm_iMapIcon', 'm_iEffect', 'm_iSound', 'm_iBarkerType', 'm_iBarkerNumber', 'm_iServiceNumber']\n",
+      "{'m_iNpcNumber': 3, 'm_iDifficulty': 0, 'm_iNpcName': 3, 'm_iComment': 3, 'm_iTeam': 2, 'm_iNpcLevel': 8, 'm_iNpcType': 0, 'm_iHNpc': 0, 'm_iHNpcNum': 0, 'm_iNpcStyle': 0, 'm_iAiType': 2, 'm_iHP': 1685, 'm_iHPRegen': 0, 'm_iDropType': 20, 'm_iRegenTime': 300000000, 'm_iHeight': 240, 'm_iRadius': 100, 'm_fScale': 1.8600000143051147, 'm_iPower': -145, 'm_iAccuracy': 45, 'm_iProtection': 215, 'm_iDodge': 235, 'm_iRunSpeed': 500, 'm_iSwimSpeed': 0, 'm_iJumpHeight': 0, 'm_iJumpDistance': 0, 'm_iSightRange': 1000, 'm_iIdleRange': 300, 'm_iCombatRange': 4000, 'm_iAtkRange': 1100, 'm_iAtkAngle': 100, 'm_iAtkRate': 0, 'm_iEffectArea': 100, 'm_iTargetMode': 1, 'm_iTargetNumber': 1, 'm_iInitalTime': 15, 'm_iDeliverTime': 0, 'm_iDelayTime': 12, 'm_iDurationTime': 0, 'm_iMegaType': 176, 'm_iMegaTypeProb': 100000, 'm_iCorruptionType': 180, 'm_iCorruptionTypeProb': 100000, 'm_iActiveSkill1': 0, 'm_iActiveSkill1Prob': 0, 'm_iActiveSkill2': 0, 'm_iActiveSkill2Prob': 0, 'm_iActiveSkill3': 0, 'm_iActiveSkill3Prob': 0, 'm_iSupportSkill': 0, 'm_iPassiveBuff': 160, 'm_iNeck': 0, 'm_iMesh': 343, 'm_iTexture': 0, 'm_iTexture2': 0, 'm_iIcon1': 342, 'm_iEffect': 66, 'm_iSound': 2, 'm_fAnimationSpeed': 1.0, 'm_iWalkSpeed': 300, 'm_fWalkAnimationSpeed': 0.9399999976158142, 'm_fRunAnimationSpeed': 1.0, 'm_iMapIcon': 19, 'm_iLegStyle': 1, 'm_iBarkerType': 0, 'm_iMegaAni': 0, 'm_iActiveSkill1Ani': 1, 'm_iActiveSkill2Ani': 2, 'm_iSupportSkillAni': 3, 'm_iMegaString': 0, 'm_iCorruptionString': 0, 'm_iActiveSkill1String': 0, 'm_iActiveSkill2String': 0, 'm_iSupportSkillString': 0, 'm_iBarkerNumber': 0, 'm_iServiceNumber': 0}\n",
+      "{'m_iNpcNumber': 3, 'm_iDifficulty': 0, 'm_iNpcName': 3, 'm_iComment': 3, 'm_iTeam': 2, 'm_iNpcLevel': 8, 'm_iNpcType': 0, 'm_iHNpc': 0, 'm_iHNpcNum': 0, 'm_iNpcStyle': 0, 'm_iLegStyle': 1, 'm_iAiType': 2, 'm_iHP': 1685, 'm_iHPRegen': 0, 'm_iDropType': 20, 'm_iRegenTime': 300000000, 'm_iHeight': 240, 'm_iRadius': 100, 'm_fScale': 1.8600000143051147, 'm_iPower': -145, 'm_iAccuracy': 45, 'm_iProtection': 215, 'm_iDodge': 235, 'm_iWalkSpeed': 300, 'm_iRunSpeed': 500, 'm_fAnimationSpeed': 1.0, 'm_fWalkAnimationSpeed': 0.9399999976158142, 'm_fRunAnimationSpeed': 1.0, 'm_iSwimSpeed': 0, 'm_iJumpHeight': 0, 'm_iJumpDistance': 0, 'm_iSightRange': 1000, 'm_iIdleRange': 300, 'm_iCombatRange': 4000, 'm_iAtkRange': 1100, 'm_iAtkAngle': 100, 'm_iEffectArea': 100, 'm_iTargetMode': 1, 'm_iTargetNumber': 1, 'm_iInitalTime': 15, 'm_iDeliverTime': 0, 'm_iDelayTime': 12, 'm_iDurationTime': 0, 'm_iMegaType': 176, 'm_iMegaTypeProb': 100000, 'm_iMegaAni': 0, 'm_iMegaString': 0, 'm_iCorruptionType': 180, 'm_iCorruptionTypeProb': 100000, 'm_iCorruptionString': 0, 'm_iActiveSkill1': 0, 'm_iActiveSkill1Prob': 0, 'm_iActiveSkill1Ani': 1, 'm_iActiveSkill1String': 0, 'm_iActiveSkill2': 0, 'm_iActiveSkill2Prob': 0, 'm_iActiveSkill2Ani': 2, 'm_iActiveSkill2String': 0, 'm_iSupportSkill': 0, 'm_iSupportSkillAni': 3, 'm_iSupportSkillString': 0, 'm_iPassiveBuff': 160, 'm_iMesh': 343, 'm_iIcon1': 342, 'm_iMapIcon': 19, 'm_iEffect': 66, 'm_iSound': 2, 'm_iBarkerType': 0, 'm_iBarkerNumber': 0, 'm_iServiceNumber': 0}\n"
+     ]
+    }
+   ],
+   "source": [
+    "table_name = \"m_pInstanceTable\"\n",
+    "subtable_name = \"m_pInstanceData\"\n",
+    "db_table_name = mappings[table_name][subtable_name]\n",
+    "test_entry = xdt[table_name][subtable_name][3]\n",
+    "with open(f\"schema/{db_table_name}.json\", 'r') as f:\n",
+    "    schema = json.load(f)\n",
+    "print(schema)\n",
+    "print(test_entry)\n",
+    "fixed_entry = apply_schema(schema, test_entry)\n",
+    "print(fixed_entry)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

json2xdb/json2xdb.py

@@ -5,14 +5,14 @@ from tqdm import tqdm
 import mysql.connector
 
 SPLIT_FIELDS = {
-    "m_iMissionRewardItemID": ("m_iMissionRewardItemID", "m_iMissionRewarItemType"),
+    "m_iMissionRewardItem": ("m_iMissionRewardItemID", "m_iMissionRewarItemType"),
-    "m_iMissionRewardItemID2": ("m_iMissionRewardItemID2", "m_iMissionRewardItemType2"),
+    "m_iMissionRewardItem2": ("m_iMissionRewardItemID2", "m_iMissionRewardItemType2"),
-    "m_iCSTItemID": ("m_iCSTItemID", "m_iCSTItemNumNeeded"),
+    "m_iCSTItem": ("m_iCSTItemID", "m_iCSTItemNumNeeded"),
-    "m_iCSUEnemyID": ("m_iCSUEnemyID", "m_iCSUNumToKill"),
+    "m_iCSUEnemy": ("m_iCSUEnemyID", "m_iCSUNumToKill"),
-    "m_iCSUItemID": ("m_iCSUItemID", "m_iCSUItemNumNeeded"),
+    "m_iCSUItem": ("m_iCSUItemID", "m_iCSUItemNumNeeded"),
-    "m_iSTItemID": ("m_iSTItemID", "m_iSTItemNumNeeded", "m_iSTItemDropRate"),
+    "m_iSTItem": ("m_iSTItemID", "m_iSTItemNumNeeded", "m_iSTItemDropRate"),
-    "m_iSUItem": ("m_iSUItem", "m_iSUInstancename"),
+    "m_iSUItem_": ("m_iSUItem", "m_iSUInstancename"),
-    "m_iFItemID": ("m_iFItemID", "m_iFItemNumNeeded"),
+    "m_iFItem": ("m_iFItemID", "m_iFItemNumNeeded"),
 }
 
 # %%
@@ -67,6 +67,32 @@ def handle_dict_table(table_entries, identifier_key, items_key):
             new_table_entries.append(new_item)
     return new_table_entries
 
+def apply_schema(schema, entry):
+    fixed_entry = {}
+    padding = 0
+    for field_name in schema:
+        if field_name is None:
+            fixed_entry[f"m_iPadding{padding}"] = 0
+            padding += 1
+            continue
+
+        if field_name in entry:
+            fixed_entry[field_name] = entry[field_name]
+        elif field_name in SPLIT_FIELDS:
+            split_field_names = SPLIT_FIELDS[field_name]
+            interleaved_arr_len = len(entry[split_field_names[0]])
+            val = []
+            for i in range(interleaved_arr_len):
+                for split_field_name in split_field_names:
+                    val.append(entry[split_field_name][i])
+            for l in range(len(val)):
+                n = l % len(split_field_names)
+                i = l // len(split_field_names)
+                new_field_name = f"{split_field_names[n]}{i}"
+                fixed_entry[new_field_name] = val[l]
+        else:
+            print("Missing field: {}".format(field_name))
+    return fixed_entry
 
 # %%
 def gen_column_sql(field_name, field_value):
@@ -125,11 +151,14 @@ def process_xdt_table(cursor, root, table_name, mappings):
             print(f"No mapping found for {table_name}.{subtable_name}")
             raise Exception()
         db_table_name = mappings[table_name][subtable_name]
+        with open(f"schema/{db_table_name}.json", 'r') as f:
+            schema = json.load(f)
         #print(f"{subtable_name} => {db_table_name}")
         
         table_entries = table[subtable_name]
         if db_table_name == "CutSceneText":
             table_entries = handle_dict_table(table_entries, "m_iEvent", "m_TextElement")
+        table_entries = [apply_schema(schema, entry) for entry in table_entries]
         table_entries = [flatten_table_entry(entry) for entry in table_entries]
 
         # clear the table

json2xdb/schema/MissionField.json

@@ -45,7 +45,7 @@
     "m_iSUReward",
     "m_iSUOutgoingMission",
     "m_iSUOutgoingTask",
-    "m_iSUItem",
+    "m_iSUItem_",
     "m_iSUMessageType",
     "m_iSUMessagetextID",
     "m_iSUMessageSendNPC",