Schemas, split fields, and padding

This commit is contained in:
Gent Semaj 2024-01-24 17:47:06 -05:00
parent 349e7b6e07
commit 27143ccb01
Signed by untrusted user: ycc
GPG Key ID: 2D76C57BF6BEADC4
3 changed files with 168 additions and 9 deletions

130
json2xdb/apply_schema.ipynb Normal file
View File

@ -0,0 +1,130 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import json"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"with open(\"xdt.json\", \"r\") as f:\n",
" xdt = json.load(f)\n",
"with open(\"mappings.json\", \"r\") as f:\n",
" mappings = json.load(f)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"def apply_schema(schema, entry):\n",
" fixed_entry = {}\n",
" padding = 0\n",
" for field_name in schema:\n",
" if field_name is None:\n",
" fixed_entry[f\"m_iPadding{padding}\"] = 0\n",
" continue\n",
"\n",
" if field_name in entry:\n",
" fixed_entry[field_name] = entry[field_name]\n",
" else:\n",
" print(\"Missing field: {}\".format(field_name))\n",
" return fixed_entry"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['m_iNpcNumber', 'm_iDifficulty', 'm_iNpcName', 'm_iComment', 'm_iTeam', 'm_iNpcLevel', 'm_iNpcType', 'm_iHNpc', 'm_iHNpcNum', 'm_iNpcStyle', 'm_iLegStyle', 'm_iAiType', 'm_iHP', 'm_iHPRegen', 'm_iDropType', 'm_iRegenTime', 'm_iHeight', 'm_iRadius', 'm_fScale', 'm_iPower', 'm_iAccuracy', 'm_iProtection', 'm_iDodge', 'm_iWalkSpeed', 'm_iRunSpeed', 'm_fAnimationSpeed', 'm_fWalkAnimationSpeed', 'm_fRunAnimationSpeed', 'm_iSwimSpeed', 'm_iJumpHeight', 'm_iJumpDistance', 'm_iSightRange', 'm_iIdleRange', 'm_iCombatRange', 'm_iAtkRange', 'm_iAtkAngle', 'm_iEffectArea', 'm_iTargetMode', 'm_iTargetNumber', 'm_iInitalTime', 'm_iDeliverTime', 'm_iDelayTime', 'm_iDurationTime', 'm_iMegaType', 'm_iMegaTypeProb', 'm_iMegaAni', 'm_iMegaString', 'm_iCorruptionType', 'm_iCorruptionTypeProb', 'm_iCorruptionString', 'm_iActiveSkill1', 'm_iActiveSkill1Prob', 'm_iActiveSkill1Ani', 'm_iActiveSkill1String', 'm_iActiveSkill2', 'm_iActiveSkill2Prob', 'm_iActiveSkill2Ani', 'm_iActiveSkill2String', 'm_iSupportSkill', 'm_iSupportSkillAni', 'm_iSupportSkillString', 'm_iPassiveBuff', 'm_iMesh', 'm_iIcon1', 'm_iMapIcon', 'm_iEffect', 'm_iSound', 'm_iBarkerType', 'm_iBarkerNumber', 'm_iServiceNumber']\n",
"{'m_iNpcNumber': 3, 'm_iDifficulty': 0, 'm_iNpcName': 3, 'm_iComment': 3, 'm_iTeam': 2, 'm_iNpcLevel': 8, 'm_iNpcType': 0, 'm_iHNpc': 0, 'm_iHNpcNum': 0, 'm_iNpcStyle': 0, 'm_iAiType': 2, 'm_iHP': 1685, 'm_iHPRegen': 0, 'm_iDropType': 20, 'm_iRegenTime': 300000000, 'm_iHeight': 240, 'm_iRadius': 100, 'm_fScale': 1.8600000143051147, 'm_iPower': -145, 'm_iAccuracy': 45, 'm_iProtection': 215, 'm_iDodge': 235, 'm_iRunSpeed': 500, 'm_iSwimSpeed': 0, 'm_iJumpHeight': 0, 'm_iJumpDistance': 0, 'm_iSightRange': 1000, 'm_iIdleRange': 300, 'm_iCombatRange': 4000, 'm_iAtkRange': 1100, 'm_iAtkAngle': 100, 'm_iAtkRate': 0, 'm_iEffectArea': 100, 'm_iTargetMode': 1, 'm_iTargetNumber': 1, 'm_iInitalTime': 15, 'm_iDeliverTime': 0, 'm_iDelayTime': 12, 'm_iDurationTime': 0, 'm_iMegaType': 176, 'm_iMegaTypeProb': 100000, 'm_iCorruptionType': 180, 'm_iCorruptionTypeProb': 100000, 'm_iActiveSkill1': 0, 'm_iActiveSkill1Prob': 0, 'm_iActiveSkill2': 0, 'm_iActiveSkill2Prob': 0, 'm_iActiveSkill3': 0, 'm_iActiveSkill3Prob': 0, 'm_iSupportSkill': 0, 'm_iPassiveBuff': 160, 'm_iNeck': 0, 'm_iMesh': 343, 'm_iTexture': 0, 'm_iTexture2': 0, 'm_iIcon1': 342, 'm_iEffect': 66, 'm_iSound': 2, 'm_fAnimationSpeed': 1.0, 'm_iWalkSpeed': 300, 'm_fWalkAnimationSpeed': 0.9399999976158142, 'm_fRunAnimationSpeed': 1.0, 'm_iMapIcon': 19, 'm_iLegStyle': 1, 'm_iBarkerType': 0, 'm_iMegaAni': 0, 'm_iActiveSkill1Ani': 1, 'm_iActiveSkill2Ani': 2, 'm_iSupportSkillAni': 3, 'm_iMegaString': 0, 'm_iCorruptionString': 0, 'm_iActiveSkill1String': 0, 'm_iActiveSkill2String': 0, 'm_iSupportSkillString': 0, 'm_iBarkerNumber': 0, 'm_iServiceNumber': 0}\n",
"{'m_iNpcNumber': 3, 'm_iDifficulty': 0, 'm_iNpcName': 3, 'm_iComment': 3, 'm_iTeam': 2, 'm_iNpcLevel': 8, 'm_iNpcType': 0, 'm_iHNpc': 0, 'm_iHNpcNum': 0, 'm_iNpcStyle': 0, 'm_iLegStyle': 1, 'm_iAiType': 2, 'm_iHP': 1685, 'm_iHPRegen': 0, 'm_iDropType': 20, 'm_iRegenTime': 300000000, 'm_iHeight': 240, 'm_iRadius': 100, 'm_fScale': 1.8600000143051147, 'm_iPower': -145, 'm_iAccuracy': 45, 'm_iProtection': 215, 'm_iDodge': 235, 'm_iWalkSpeed': 300, 'm_iRunSpeed': 500, 'm_fAnimationSpeed': 1.0, 'm_fWalkAnimationSpeed': 0.9399999976158142, 'm_fRunAnimationSpeed': 1.0, 'm_iSwimSpeed': 0, 'm_iJumpHeight': 0, 'm_iJumpDistance': 0, 'm_iSightRange': 1000, 'm_iIdleRange': 300, 'm_iCombatRange': 4000, 'm_iAtkRange': 1100, 'm_iAtkAngle': 100, 'm_iEffectArea': 100, 'm_iTargetMode': 1, 'm_iTargetNumber': 1, 'm_iInitalTime': 15, 'm_iDeliverTime': 0, 'm_iDelayTime': 12, 'm_iDurationTime': 0, 'm_iMegaType': 176, 'm_iMegaTypeProb': 100000, 'm_iMegaAni': 0, 'm_iMegaString': 0, 'm_iCorruptionType': 180, 'm_iCorruptionTypeProb': 100000, 'm_iCorruptionString': 0, 'm_iActiveSkill1': 0, 'm_iActiveSkill1Prob': 0, 'm_iActiveSkill1Ani': 1, 'm_iActiveSkill1String': 0, 'm_iActiveSkill2': 0, 'm_iActiveSkill2Prob': 0, 'm_iActiveSkill2Ani': 2, 'm_iActiveSkill2String': 0, 'm_iSupportSkill': 0, 'm_iSupportSkillAni': 3, 'm_iSupportSkillString': 0, 'm_iPassiveBuff': 160, 'm_iMesh': 343, 'm_iIcon1': 342, 'm_iMapIcon': 19, 'm_iEffect': 66, 'm_iSound': 2, 'm_iBarkerType': 0, 'm_iBarkerNumber': 0, 'm_iServiceNumber': 0}\n"
]
}
],
"source": [
"table_name = \"m_pNpcTable\"\n",
"subtable_name = \"m_pNpcData\"\n",
"db_table_name = mappings[table_name][subtable_name]\n",
"test_entry = xdt[table_name][subtable_name][3]\n",
"with open(f\"schema/{db_table_name}.json\", 'r') as f:\n",
" schema = json.load(f)\n",
"print(schema)\n",
"print(test_entry)\n",
"fixed_entry = apply_schema(schema, test_entry)\n",
"print(fixed_entry)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['m_iNpcNumber', 'm_iDifficulty', 'm_iNpcName', 'm_iComment', 'm_iTeam', 'm_iNpcLevel', 'm_iNpcType', 'm_iHNpc', 'm_iHNpcNum', 'm_iNpcStyle', 'm_iLegStyle', 'm_iAiType', 'm_iHP', 'm_iHPRegen', 'm_iDropType', 'm_iRegenTime', 'm_iHeight', 'm_iRadius', 'm_fScale', 'm_iPower', 'm_iAccuracy', 'm_iProtection', 'm_iDodge', 'm_iWalkSpeed', 'm_iRunSpeed', 'm_fAnimationSpeed', 'm_fWalkAnimationSpeed', 'm_fRunAnimationSpeed', 'm_iSwimSpeed', 'm_iJumpHeight', 'm_iJumpDistance', 'm_iSightRange', 'm_iIdleRange', 'm_iCombatRange', 'm_iAtkRange', 'm_iAtkAngle', 'm_iEffectArea', 'm_iTargetMode', 'm_iTargetNumber', 'm_iInitalTime', 'm_iDeliverTime', 'm_iDelayTime', 'm_iDurationTime', 'm_iMegaType', 'm_iMegaTypeProb', 'm_iMegaAni', 'm_iMegaString', 'm_iCorruptionType', 'm_iCorruptionTypeProb', 'm_iCorruptionString', 'm_iActiveSkill1', 'm_iActiveSkill1Prob', 'm_iActiveSkill1Ani', 'm_iActiveSkill1String', 'm_iActiveSkill2', 'm_iActiveSkill2Prob', 'm_iActiveSkill2Ani', 'm_iActiveSkill2String', 'm_iSupportSkill', 'm_iSupportSkillAni', 'm_iSupportSkillString', 'm_iPassiveBuff', 'm_iMesh', 'm_iIcon1', 'm_iMapIcon', 'm_iEffect', 'm_iSound', 'm_iBarkerType', 'm_iBarkerNumber', 'm_iServiceNumber']\n",
"{'m_iNpcNumber': 3, 'm_iDifficulty': 0, 'm_iNpcName': 3, 'm_iComment': 3, 'm_iTeam': 2, 'm_iNpcLevel': 8, 'm_iNpcType': 0, 'm_iHNpc': 0, 'm_iHNpcNum': 0, 'm_iNpcStyle': 0, 'm_iAiType': 2, 'm_iHP': 1685, 'm_iHPRegen': 0, 'm_iDropType': 20, 'm_iRegenTime': 300000000, 'm_iHeight': 240, 'm_iRadius': 100, 'm_fScale': 1.8600000143051147, 'm_iPower': -145, 'm_iAccuracy': 45, 'm_iProtection': 215, 'm_iDodge': 235, 'm_iRunSpeed': 500, 'm_iSwimSpeed': 0, 'm_iJumpHeight': 0, 'm_iJumpDistance': 0, 'm_iSightRange': 1000, 'm_iIdleRange': 300, 'm_iCombatRange': 4000, 'm_iAtkRange': 1100, 'm_iAtkAngle': 100, 'm_iAtkRate': 0, 'm_iEffectArea': 100, 'm_iTargetMode': 1, 'm_iTargetNumber': 1, 'm_iInitalTime': 15, 'm_iDeliverTime': 0, 'm_iDelayTime': 12, 'm_iDurationTime': 0, 'm_iMegaType': 176, 'm_iMegaTypeProb': 100000, 'm_iCorruptionType': 180, 'm_iCorruptionTypeProb': 100000, 'm_iActiveSkill1': 0, 'm_iActiveSkill1Prob': 0, 'm_iActiveSkill2': 0, 'm_iActiveSkill2Prob': 0, 'm_iActiveSkill3': 0, 'm_iActiveSkill3Prob': 0, 'm_iSupportSkill': 0, 'm_iPassiveBuff': 160, 'm_iNeck': 0, 'm_iMesh': 343, 'm_iTexture': 0, 'm_iTexture2': 0, 'm_iIcon1': 342, 'm_iEffect': 66, 'm_iSound': 2, 'm_fAnimationSpeed': 1.0, 'm_iWalkSpeed': 300, 'm_fWalkAnimationSpeed': 0.9399999976158142, 'm_fRunAnimationSpeed': 1.0, 'm_iMapIcon': 19, 'm_iLegStyle': 1, 'm_iBarkerType': 0, 'm_iMegaAni': 0, 'm_iActiveSkill1Ani': 1, 'm_iActiveSkill2Ani': 2, 'm_iSupportSkillAni': 3, 'm_iMegaString': 0, 'm_iCorruptionString': 0, 'm_iActiveSkill1String': 0, 'm_iActiveSkill2String': 0, 'm_iSupportSkillString': 0, 'm_iBarkerNumber': 0, 'm_iServiceNumber': 0}\n",
"{'m_iNpcNumber': 3, 'm_iDifficulty': 0, 'm_iNpcName': 3, 'm_iComment': 3, 'm_iTeam': 2, 'm_iNpcLevel': 8, 'm_iNpcType': 0, 'm_iHNpc': 0, 'm_iHNpcNum': 0, 'm_iNpcStyle': 0, 'm_iLegStyle': 1, 'm_iAiType': 2, 'm_iHP': 1685, 'm_iHPRegen': 0, 'm_iDropType': 20, 'm_iRegenTime': 300000000, 'm_iHeight': 240, 'm_iRadius': 100, 'm_fScale': 1.8600000143051147, 'm_iPower': -145, 'm_iAccuracy': 45, 'm_iProtection': 215, 'm_iDodge': 235, 'm_iWalkSpeed': 300, 'm_iRunSpeed': 500, 'm_fAnimationSpeed': 1.0, 'm_fWalkAnimationSpeed': 0.9399999976158142, 'm_fRunAnimationSpeed': 1.0, 'm_iSwimSpeed': 0, 'm_iJumpHeight': 0, 'm_iJumpDistance': 0, 'm_iSightRange': 1000, 'm_iIdleRange': 300, 'm_iCombatRange': 4000, 'm_iAtkRange': 1100, 'm_iAtkAngle': 100, 'm_iEffectArea': 100, 'm_iTargetMode': 1, 'm_iTargetNumber': 1, 'm_iInitalTime': 15, 'm_iDeliverTime': 0, 'm_iDelayTime': 12, 'm_iDurationTime': 0, 'm_iMegaType': 176, 'm_iMegaTypeProb': 100000, 'm_iMegaAni': 0, 'm_iMegaString': 0, 'm_iCorruptionType': 180, 'm_iCorruptionTypeProb': 100000, 'm_iCorruptionString': 0, 'm_iActiveSkill1': 0, 'm_iActiveSkill1Prob': 0, 'm_iActiveSkill1Ani': 1, 'm_iActiveSkill1String': 0, 'm_iActiveSkill2': 0, 'm_iActiveSkill2Prob': 0, 'm_iActiveSkill2Ani': 2, 'm_iActiveSkill2String': 0, 'm_iSupportSkill': 0, 'm_iSupportSkillAni': 3, 'm_iSupportSkillString': 0, 'm_iPassiveBuff': 160, 'm_iMesh': 343, 'm_iIcon1': 342, 'm_iMapIcon': 19, 'm_iEffect': 66, 'm_iSound': 2, 'm_iBarkerType': 0, 'm_iBarkerNumber': 0, 'm_iServiceNumber': 0}\n"
]
}
],
"source": [
"table_name = \"m_pInstanceTable\"\n",
"subtable_name = \"m_pInstanceData\"\n",
"db_table_name = mappings[table_name][subtable_name]\n",
"test_entry = xdt[table_name][subtable_name][3]\n",
"with open(f\"schema/{db_table_name}.json\", 'r') as f:\n",
" schema = json.load(f)\n",
"print(schema)\n",
"print(test_entry)\n",
"fixed_entry = apply_schema(schema, test_entry)\n",
"print(fixed_entry)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

View File

@ -5,14 +5,14 @@ from tqdm import tqdm
import mysql.connector import mysql.connector
SPLIT_FIELDS = { SPLIT_FIELDS = {
"m_iMissionRewardItemID": ("m_iMissionRewardItemID", "m_iMissionRewarItemType"), "m_iMissionRewardItem": ("m_iMissionRewardItemID", "m_iMissionRewarItemType"),
"m_iMissionRewardItemID2": ("m_iMissionRewardItemID2", "m_iMissionRewardItemType2"), "m_iMissionRewardItem2": ("m_iMissionRewardItemID2", "m_iMissionRewardItemType2"),
"m_iCSTItemID": ("m_iCSTItemID", "m_iCSTItemNumNeeded"), "m_iCSTItem": ("m_iCSTItemID", "m_iCSTItemNumNeeded"),
"m_iCSUEnemyID": ("m_iCSUEnemyID", "m_iCSUNumToKill"), "m_iCSUEnemy": ("m_iCSUEnemyID", "m_iCSUNumToKill"),
"m_iCSUItemID": ("m_iCSUItemID", "m_iCSUItemNumNeeded"), "m_iCSUItem": ("m_iCSUItemID", "m_iCSUItemNumNeeded"),
"m_iSTItemID": ("m_iSTItemID", "m_iSTItemNumNeeded", "m_iSTItemDropRate"), "m_iSTItem": ("m_iSTItemID", "m_iSTItemNumNeeded", "m_iSTItemDropRate"),
"m_iSUItem": ("m_iSUItem", "m_iSUInstancename"), "m_iSUItem_": ("m_iSUItem", "m_iSUInstancename"),
"m_iFItemID": ("m_iFItemID", "m_iFItemNumNeeded"), "m_iFItem": ("m_iFItemID", "m_iFItemNumNeeded"),
} }
# %% # %%
@ -67,6 +67,32 @@ def handle_dict_table(table_entries, identifier_key, items_key):
new_table_entries.append(new_item) new_table_entries.append(new_item)
return new_table_entries return new_table_entries
def apply_schema(schema, entry):
fixed_entry = {}
padding = 0
for field_name in schema:
if field_name is None:
fixed_entry[f"m_iPadding{padding}"] = 0
padding += 1
continue
if field_name in entry:
fixed_entry[field_name] = entry[field_name]
elif field_name in SPLIT_FIELDS:
split_field_names = SPLIT_FIELDS[field_name]
interleaved_arr_len = len(entry[split_field_names[0]])
val = []
for i in range(interleaved_arr_len):
for split_field_name in split_field_names:
val.append(entry[split_field_name][i])
for l in range(len(val)):
n = l % len(split_field_names)
i = l // len(split_field_names)
new_field_name = f"{split_field_names[n]}{i}"
fixed_entry[new_field_name] = val[l]
else:
print("Missing field: {}".format(field_name))
return fixed_entry
# %% # %%
def gen_column_sql(field_name, field_value): def gen_column_sql(field_name, field_value):
@ -125,11 +151,14 @@ def process_xdt_table(cursor, root, table_name, mappings):
print(f"No mapping found for {table_name}.{subtable_name}") print(f"No mapping found for {table_name}.{subtable_name}")
raise Exception() raise Exception()
db_table_name = mappings[table_name][subtable_name] db_table_name = mappings[table_name][subtable_name]
with open(f"schema/{db_table_name}.json", 'r') as f:
schema = json.load(f)
#print(f"{subtable_name} => {db_table_name}") #print(f"{subtable_name} => {db_table_name}")
table_entries = table[subtable_name] table_entries = table[subtable_name]
if db_table_name == "CutSceneText": if db_table_name == "CutSceneText":
table_entries = handle_dict_table(table_entries, "m_iEvent", "m_TextElement") table_entries = handle_dict_table(table_entries, "m_iEvent", "m_TextElement")
table_entries = [apply_schema(schema, entry) for entry in table_entries]
table_entries = [flatten_table_entry(entry) for entry in table_entries] table_entries = [flatten_table_entry(entry) for entry in table_entries]
# clear the table # clear the table

View File

@ -45,7 +45,7 @@
"m_iSUReward", "m_iSUReward",
"m_iSUOutgoingMission", "m_iSUOutgoingMission",
"m_iSUOutgoingTask", "m_iSUOutgoingTask",
"m_iSUItem", "m_iSUItem_",
"m_iSUMessageType", "m_iSUMessageType",
"m_iSUMessagetextID", "m_iSUMessagetextID",
"m_iSUMessageSendNPC", "m_iSUMessageSendNPC",