From 27143ccb01ecd090ff423b13f135fb5a97d90ce7 Mon Sep 17 00:00:00 2001 From: Gent Semaj Date: Wed, 24 Jan 2024 17:47:06 -0500 Subject: [PATCH] Schemas, split fields, and padding --- json2xdb/apply_schema.ipynb | 130 ++++++++++++++++++++++++++++++ json2xdb/json2xdb.py | 45 +++++++++-- json2xdb/schema/MissionField.json | 2 +- 3 files changed, 168 insertions(+), 9 deletions(-) create mode 100644 json2xdb/apply_schema.ipynb diff --git a/json2xdb/apply_schema.ipynb b/json2xdb/apply_schema.ipynb new file mode 100644 index 0000000..7fcb0df --- /dev/null +++ b/json2xdb/apply_schema.ipynb @@ -0,0 +1,130 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"xdt.json\", \"r\") as f:\n", + " xdt = json.load(f)\n", + "with open(\"mappings.json\", \"r\") as f:\n", + " mappings = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def apply_schema(schema, entry):\n", + " fixed_entry = {}\n", + " padding = 0\n", + " for field_name in schema:\n", + " if field_name is None:\n", + " fixed_entry[f\"m_iPadding{padding}\"] = 0\n", + " continue\n", + "\n", + " if field_name in entry:\n", + " fixed_entry[field_name] = entry[field_name]\n", + " else:\n", + " print(\"Missing field: {}\".format(field_name))\n", + " return fixed_entry" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['m_iNpcNumber', 'm_iDifficulty', 'm_iNpcName', 'm_iComment', 'm_iTeam', 'm_iNpcLevel', 'm_iNpcType', 'm_iHNpc', 'm_iHNpcNum', 'm_iNpcStyle', 'm_iLegStyle', 'm_iAiType', 'm_iHP', 'm_iHPRegen', 'm_iDropType', 'm_iRegenTime', 'm_iHeight', 'm_iRadius', 'm_fScale', 'm_iPower', 'm_iAccuracy', 'm_iProtection', 'm_iDodge', 'm_iWalkSpeed', 'm_iRunSpeed', 'm_fAnimationSpeed', 'm_fWalkAnimationSpeed', 'm_fRunAnimationSpeed', 'm_iSwimSpeed', 'm_iJumpHeight', 'm_iJumpDistance', 'm_iSightRange', 'm_iIdleRange', 'm_iCombatRange', 'm_iAtkRange', 'm_iAtkAngle', 'm_iEffectArea', 'm_iTargetMode', 'm_iTargetNumber', 'm_iInitalTime', 'm_iDeliverTime', 'm_iDelayTime', 'm_iDurationTime', 'm_iMegaType', 'm_iMegaTypeProb', 'm_iMegaAni', 'm_iMegaString', 'm_iCorruptionType', 'm_iCorruptionTypeProb', 'm_iCorruptionString', 'm_iActiveSkill1', 'm_iActiveSkill1Prob', 'm_iActiveSkill1Ani', 'm_iActiveSkill1String', 'm_iActiveSkill2', 'm_iActiveSkill2Prob', 'm_iActiveSkill2Ani', 'm_iActiveSkill2String', 'm_iSupportSkill', 'm_iSupportSkillAni', 'm_iSupportSkillString', 'm_iPassiveBuff', 'm_iMesh', 'm_iIcon1', 'm_iMapIcon', 'm_iEffect', 'm_iSound', 'm_iBarkerType', 'm_iBarkerNumber', 'm_iServiceNumber']\n", + "{'m_iNpcNumber': 3, 'm_iDifficulty': 0, 'm_iNpcName': 3, 'm_iComment': 3, 'm_iTeam': 2, 'm_iNpcLevel': 8, 'm_iNpcType': 0, 'm_iHNpc': 0, 'm_iHNpcNum': 0, 'm_iNpcStyle': 0, 'm_iAiType': 2, 'm_iHP': 1685, 'm_iHPRegen': 0, 'm_iDropType': 20, 'm_iRegenTime': 300000000, 'm_iHeight': 240, 'm_iRadius': 100, 'm_fScale': 1.8600000143051147, 'm_iPower': -145, 'm_iAccuracy': 45, 'm_iProtection': 215, 'm_iDodge': 235, 'm_iRunSpeed': 500, 'm_iSwimSpeed': 0, 'm_iJumpHeight': 0, 'm_iJumpDistance': 0, 'm_iSightRange': 1000, 'm_iIdleRange': 300, 'm_iCombatRange': 4000, 'm_iAtkRange': 1100, 'm_iAtkAngle': 100, 'm_iAtkRate': 0, 'm_iEffectArea': 100, 'm_iTargetMode': 1, 'm_iTargetNumber': 1, 'm_iInitalTime': 15, 'm_iDeliverTime': 0, 'm_iDelayTime': 12, 'm_iDurationTime': 0, 'm_iMegaType': 176, 'm_iMegaTypeProb': 100000, 'm_iCorruptionType': 180, 'm_iCorruptionTypeProb': 100000, 'm_iActiveSkill1': 0, 'm_iActiveSkill1Prob': 0, 'm_iActiveSkill2': 0, 'm_iActiveSkill2Prob': 0, 'm_iActiveSkill3': 0, 'm_iActiveSkill3Prob': 0, 'm_iSupportSkill': 0, 'm_iPassiveBuff': 160, 'm_iNeck': 0, 'm_iMesh': 343, 'm_iTexture': 0, 'm_iTexture2': 0, 'm_iIcon1': 342, 'm_iEffect': 66, 'm_iSound': 2, 'm_fAnimationSpeed': 1.0, 'm_iWalkSpeed': 300, 'm_fWalkAnimationSpeed': 0.9399999976158142, 'm_fRunAnimationSpeed': 1.0, 'm_iMapIcon': 19, 'm_iLegStyle': 1, 'm_iBarkerType': 0, 'm_iMegaAni': 0, 'm_iActiveSkill1Ani': 1, 'm_iActiveSkill2Ani': 2, 'm_iSupportSkillAni': 3, 'm_iMegaString': 0, 'm_iCorruptionString': 0, 'm_iActiveSkill1String': 0, 'm_iActiveSkill2String': 0, 'm_iSupportSkillString': 0, 'm_iBarkerNumber': 0, 'm_iServiceNumber': 0}\n", + "{'m_iNpcNumber': 3, 'm_iDifficulty': 0, 'm_iNpcName': 3, 'm_iComment': 3, 'm_iTeam': 2, 'm_iNpcLevel': 8, 'm_iNpcType': 0, 'm_iHNpc': 0, 'm_iHNpcNum': 0, 'm_iNpcStyle': 0, 'm_iLegStyle': 1, 'm_iAiType': 2, 'm_iHP': 1685, 'm_iHPRegen': 0, 'm_iDropType': 20, 'm_iRegenTime': 300000000, 'm_iHeight': 240, 'm_iRadius': 100, 'm_fScale': 1.8600000143051147, 'm_iPower': -145, 'm_iAccuracy': 45, 'm_iProtection': 215, 'm_iDodge': 235, 'm_iWalkSpeed': 300, 'm_iRunSpeed': 500, 'm_fAnimationSpeed': 1.0, 'm_fWalkAnimationSpeed': 0.9399999976158142, 'm_fRunAnimationSpeed': 1.0, 'm_iSwimSpeed': 0, 'm_iJumpHeight': 0, 'm_iJumpDistance': 0, 'm_iSightRange': 1000, 'm_iIdleRange': 300, 'm_iCombatRange': 4000, 'm_iAtkRange': 1100, 'm_iAtkAngle': 100, 'm_iEffectArea': 100, 'm_iTargetMode': 1, 'm_iTargetNumber': 1, 'm_iInitalTime': 15, 'm_iDeliverTime': 0, 'm_iDelayTime': 12, 'm_iDurationTime': 0, 'm_iMegaType': 176, 'm_iMegaTypeProb': 100000, 'm_iMegaAni': 0, 'm_iMegaString': 0, 'm_iCorruptionType': 180, 'm_iCorruptionTypeProb': 100000, 'm_iCorruptionString': 0, 'm_iActiveSkill1': 0, 'm_iActiveSkill1Prob': 0, 'm_iActiveSkill1Ani': 1, 'm_iActiveSkill1String': 0, 'm_iActiveSkill2': 0, 'm_iActiveSkill2Prob': 0, 'm_iActiveSkill2Ani': 2, 'm_iActiveSkill2String': 0, 'm_iSupportSkill': 0, 'm_iSupportSkillAni': 3, 'm_iSupportSkillString': 0, 'm_iPassiveBuff': 160, 'm_iMesh': 343, 'm_iIcon1': 342, 'm_iMapIcon': 19, 'm_iEffect': 66, 'm_iSound': 2, 'm_iBarkerType': 0, 'm_iBarkerNumber': 0, 'm_iServiceNumber': 0}\n" + ] + } + ], + "source": [ + "table_name = \"m_pNpcTable\"\n", + "subtable_name = \"m_pNpcData\"\n", + "db_table_name = mappings[table_name][subtable_name]\n", + "test_entry = xdt[table_name][subtable_name][3]\n", + "with open(f\"schema/{db_table_name}.json\", 'r') as f:\n", + " schema = json.load(f)\n", + "print(schema)\n", + "print(test_entry)\n", + "fixed_entry = apply_schema(schema, test_entry)\n", + "print(fixed_entry)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['m_iNpcNumber', 'm_iDifficulty', 'm_iNpcName', 'm_iComment', 'm_iTeam', 'm_iNpcLevel', 'm_iNpcType', 'm_iHNpc', 'm_iHNpcNum', 'm_iNpcStyle', 'm_iLegStyle', 'm_iAiType', 'm_iHP', 'm_iHPRegen', 'm_iDropType', 'm_iRegenTime', 'm_iHeight', 'm_iRadius', 'm_fScale', 'm_iPower', 'm_iAccuracy', 'm_iProtection', 'm_iDodge', 'm_iWalkSpeed', 'm_iRunSpeed', 'm_fAnimationSpeed', 'm_fWalkAnimationSpeed', 'm_fRunAnimationSpeed', 'm_iSwimSpeed', 'm_iJumpHeight', 'm_iJumpDistance', 'm_iSightRange', 'm_iIdleRange', 'm_iCombatRange', 'm_iAtkRange', 'm_iAtkAngle', 'm_iEffectArea', 'm_iTargetMode', 'm_iTargetNumber', 'm_iInitalTime', 'm_iDeliverTime', 'm_iDelayTime', 'm_iDurationTime', 'm_iMegaType', 'm_iMegaTypeProb', 'm_iMegaAni', 'm_iMegaString', 'm_iCorruptionType', 'm_iCorruptionTypeProb', 'm_iCorruptionString', 'm_iActiveSkill1', 'm_iActiveSkill1Prob', 'm_iActiveSkill1Ani', 'm_iActiveSkill1String', 'm_iActiveSkill2', 'm_iActiveSkill2Prob', 'm_iActiveSkill2Ani', 'm_iActiveSkill2String', 'm_iSupportSkill', 'm_iSupportSkillAni', 'm_iSupportSkillString', 'm_iPassiveBuff', 'm_iMesh', 'm_iIcon1', 'm_iMapIcon', 'm_iEffect', 'm_iSound', 'm_iBarkerType', 'm_iBarkerNumber', 'm_iServiceNumber']\n", + "{'m_iNpcNumber': 3, 'm_iDifficulty': 0, 'm_iNpcName': 3, 'm_iComment': 3, 'm_iTeam': 2, 'm_iNpcLevel': 8, 'm_iNpcType': 0, 'm_iHNpc': 0, 'm_iHNpcNum': 0, 'm_iNpcStyle': 0, 'm_iAiType': 2, 'm_iHP': 1685, 'm_iHPRegen': 0, 'm_iDropType': 20, 'm_iRegenTime': 300000000, 'm_iHeight': 240, 'm_iRadius': 100, 'm_fScale': 1.8600000143051147, 'm_iPower': -145, 'm_iAccuracy': 45, 'm_iProtection': 215, 'm_iDodge': 235, 'm_iRunSpeed': 500, 'm_iSwimSpeed': 0, 'm_iJumpHeight': 0, 'm_iJumpDistance': 0, 'm_iSightRange': 1000, 'm_iIdleRange': 300, 'm_iCombatRange': 4000, 'm_iAtkRange': 1100, 'm_iAtkAngle': 100, 'm_iAtkRate': 0, 'm_iEffectArea': 100, 'm_iTargetMode': 1, 'm_iTargetNumber': 1, 'm_iInitalTime': 15, 'm_iDeliverTime': 0, 'm_iDelayTime': 12, 'm_iDurationTime': 0, 'm_iMegaType': 176, 'm_iMegaTypeProb': 100000, 'm_iCorruptionType': 180, 'm_iCorruptionTypeProb': 100000, 'm_iActiveSkill1': 0, 'm_iActiveSkill1Prob': 0, 'm_iActiveSkill2': 0, 'm_iActiveSkill2Prob': 0, 'm_iActiveSkill3': 0, 'm_iActiveSkill3Prob': 0, 'm_iSupportSkill': 0, 'm_iPassiveBuff': 160, 'm_iNeck': 0, 'm_iMesh': 343, 'm_iTexture': 0, 'm_iTexture2': 0, 'm_iIcon1': 342, 'm_iEffect': 66, 'm_iSound': 2, 'm_fAnimationSpeed': 1.0, 'm_iWalkSpeed': 300, 'm_fWalkAnimationSpeed': 0.9399999976158142, 'm_fRunAnimationSpeed': 1.0, 'm_iMapIcon': 19, 'm_iLegStyle': 1, 'm_iBarkerType': 0, 'm_iMegaAni': 0, 'm_iActiveSkill1Ani': 1, 'm_iActiveSkill2Ani': 2, 'm_iSupportSkillAni': 3, 'm_iMegaString': 0, 'm_iCorruptionString': 0, 'm_iActiveSkill1String': 0, 'm_iActiveSkill2String': 0, 'm_iSupportSkillString': 0, 'm_iBarkerNumber': 0, 'm_iServiceNumber': 0}\n", + "{'m_iNpcNumber': 3, 'm_iDifficulty': 0, 'm_iNpcName': 3, 'm_iComment': 3, 'm_iTeam': 2, 'm_iNpcLevel': 8, 'm_iNpcType': 0, 'm_iHNpc': 0, 'm_iHNpcNum': 0, 'm_iNpcStyle': 0, 'm_iLegStyle': 1, 'm_iAiType': 2, 'm_iHP': 1685, 'm_iHPRegen': 0, 'm_iDropType': 20, 'm_iRegenTime': 300000000, 'm_iHeight': 240, 'm_iRadius': 100, 'm_fScale': 1.8600000143051147, 'm_iPower': -145, 'm_iAccuracy': 45, 'm_iProtection': 215, 'm_iDodge': 235, 'm_iWalkSpeed': 300, 'm_iRunSpeed': 500, 'm_fAnimationSpeed': 1.0, 'm_fWalkAnimationSpeed': 0.9399999976158142, 'm_fRunAnimationSpeed': 1.0, 'm_iSwimSpeed': 0, 'm_iJumpHeight': 0, 'm_iJumpDistance': 0, 'm_iSightRange': 1000, 'm_iIdleRange': 300, 'm_iCombatRange': 4000, 'm_iAtkRange': 1100, 'm_iAtkAngle': 100, 'm_iEffectArea': 100, 'm_iTargetMode': 1, 'm_iTargetNumber': 1, 'm_iInitalTime': 15, 'm_iDeliverTime': 0, 'm_iDelayTime': 12, 'm_iDurationTime': 0, 'm_iMegaType': 176, 'm_iMegaTypeProb': 100000, 'm_iMegaAni': 0, 'm_iMegaString': 0, 'm_iCorruptionType': 180, 'm_iCorruptionTypeProb': 100000, 'm_iCorruptionString': 0, 'm_iActiveSkill1': 0, 'm_iActiveSkill1Prob': 0, 'm_iActiveSkill1Ani': 1, 'm_iActiveSkill1String': 0, 'm_iActiveSkill2': 0, 'm_iActiveSkill2Prob': 0, 'm_iActiveSkill2Ani': 2, 'm_iActiveSkill2String': 0, 'm_iSupportSkill': 0, 'm_iSupportSkillAni': 3, 'm_iSupportSkillString': 0, 'm_iPassiveBuff': 160, 'm_iMesh': 343, 'm_iIcon1': 342, 'm_iMapIcon': 19, 'm_iEffect': 66, 'm_iSound': 2, 'm_iBarkerType': 0, 'm_iBarkerNumber': 0, 'm_iServiceNumber': 0}\n" + ] + } + ], + "source": [ + "table_name = \"m_pInstanceTable\"\n", + "subtable_name = \"m_pInstanceData\"\n", + "db_table_name = mappings[table_name][subtable_name]\n", + "test_entry = xdt[table_name][subtable_name][3]\n", + "with open(f\"schema/{db_table_name}.json\", 'r') as f:\n", + " schema = json.load(f)\n", + "print(schema)\n", + "print(test_entry)\n", + "fixed_entry = apply_schema(schema, test_entry)\n", + "print(fixed_entry)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/json2xdb/json2xdb.py b/json2xdb/json2xdb.py index 01b10ce..57e4418 100644 --- a/json2xdb/json2xdb.py +++ b/json2xdb/json2xdb.py @@ -5,14 +5,14 @@ from tqdm import tqdm import mysql.connector SPLIT_FIELDS = { - "m_iMissionRewardItemID": ("m_iMissionRewardItemID", "m_iMissionRewarItemType"), - "m_iMissionRewardItemID2": ("m_iMissionRewardItemID2", "m_iMissionRewardItemType2"), - "m_iCSTItemID": ("m_iCSTItemID", "m_iCSTItemNumNeeded"), - "m_iCSUEnemyID": ("m_iCSUEnemyID", "m_iCSUNumToKill"), - "m_iCSUItemID": ("m_iCSUItemID", "m_iCSUItemNumNeeded"), - "m_iSTItemID": ("m_iSTItemID", "m_iSTItemNumNeeded", "m_iSTItemDropRate"), - "m_iSUItem": ("m_iSUItem", "m_iSUInstancename"), - "m_iFItemID": ("m_iFItemID", "m_iFItemNumNeeded"), + "m_iMissionRewardItem": ("m_iMissionRewardItemID", "m_iMissionRewarItemType"), + "m_iMissionRewardItem2": ("m_iMissionRewardItemID2", "m_iMissionRewardItemType2"), + "m_iCSTItem": ("m_iCSTItemID", "m_iCSTItemNumNeeded"), + "m_iCSUEnemy": ("m_iCSUEnemyID", "m_iCSUNumToKill"), + "m_iCSUItem": ("m_iCSUItemID", "m_iCSUItemNumNeeded"), + "m_iSTItem": ("m_iSTItemID", "m_iSTItemNumNeeded", "m_iSTItemDropRate"), + "m_iSUItem_": ("m_iSUItem", "m_iSUInstancename"), + "m_iFItem": ("m_iFItemID", "m_iFItemNumNeeded"), } # %% @@ -67,6 +67,32 @@ def handle_dict_table(table_entries, identifier_key, items_key): new_table_entries.append(new_item) return new_table_entries +def apply_schema(schema, entry): + fixed_entry = {} + padding = 0 + for field_name in schema: + if field_name is None: + fixed_entry[f"m_iPadding{padding}"] = 0 + padding += 1 + continue + + if field_name in entry: + fixed_entry[field_name] = entry[field_name] + elif field_name in SPLIT_FIELDS: + split_field_names = SPLIT_FIELDS[field_name] + interleaved_arr_len = len(entry[split_field_names[0]]) + val = [] + for i in range(interleaved_arr_len): + for split_field_name in split_field_names: + val.append(entry[split_field_name][i]) + for l in range(len(val)): + n = l % len(split_field_names) + i = l // len(split_field_names) + new_field_name = f"{split_field_names[n]}{i}" + fixed_entry[new_field_name] = val[l] + else: + print("Missing field: {}".format(field_name)) + return fixed_entry # %% def gen_column_sql(field_name, field_value): @@ -125,11 +151,14 @@ def process_xdt_table(cursor, root, table_name, mappings): print(f"No mapping found for {table_name}.{subtable_name}") raise Exception() db_table_name = mappings[table_name][subtable_name] + with open(f"schema/{db_table_name}.json", 'r') as f: + schema = json.load(f) #print(f"{subtable_name} => {db_table_name}") table_entries = table[subtable_name] if db_table_name == "CutSceneText": table_entries = handle_dict_table(table_entries, "m_iEvent", "m_TextElement") + table_entries = [apply_schema(schema, entry) for entry in table_entries] table_entries = [flatten_table_entry(entry) for entry in table_entries] # clear the table diff --git a/json2xdb/schema/MissionField.json b/json2xdb/schema/MissionField.json index a8e4bce..4d9415a 100644 --- a/json2xdb/schema/MissionField.json +++ b/json2xdb/schema/MissionField.json @@ -45,7 +45,7 @@ "m_iSUReward", "m_iSUOutgoingMission", "m_iSUOutgoingTask", - "m_iSUItem", + "m_iSUItem_", "m_iSUMessageType", "m_iSUMessagetextID", "m_iSUMessageSendNPC",