# ##### BEGIN GPL LICENSE BLOCK #####
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; either version 2
#  of the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####


bl_info = {
    "name": "Mass Effect Import",
    "description": "Mass Effect Import",
    "author": "PotatoFire777",
    "version": (0, 2, 0),
    "blender": (3, 0, 0),
    "location": "PROPERTIES",
    "warning": "",
    "wiki_url": "",
    "tracker_url": "",
    "category": "Import-Export"
}

import os
import re
import bpy
import re
import math
from pathlib import Path
import random
import time
import json

###############################################
#####              Dependecy              #####
###############################################
## GLTF import addon
## This will also reload project (so ... yeh)
#from addon_utils import check, enable
#bpy.ops.wm.read_factory_settings(use_empty=True)
#for addon in ("io_export_dxf", "io_scene_gltf2"):
#    default, enabled = check(addon)
#    if not enabled:
#        enable(addon, default_set=True, persistent=True)

###############################################
#####              Globals                #####
###############################################
GLOBAL_matLibrary = ""
GLOBAL_importTextures = False
GLOBAL_importLights = False
GLOBAL_mergePointLight = False
GLOBAL_mergeDistance = ""
GLOBAL_directory = "./"
GLOBAL_local = True
GLOBAL_mapFile = ""
GLOBAL_lightFile = ""
GLOBAL_primaryPathsDict = {}
GLOBAL_useCache = True
GLOBAL_missingCache = []
GLOBAL_genShader = ""
GLOBAL_complexMaterial = False

###############################################
#####              Main                   #####
###############################################

## save processing time when doing a bunch of imports
## requires cache to be rebuilt anytime change contens
## in the directory
class BuildCache(bpy.types.Operator):
    bl_idname = "me.buildcache"
    bl_label = "Build Index"
    
    def execute(self, context):
        ## declare globals we use
        global GLOBAL_directory
        global GLOBAL_primaryPathsDict

        CollectVariables(context)
    
        ## Build file cache save as txt file of all paths (requires static directory content)
        for root, dirs, files in os.walk(GLOBAL_directory):
            for dir in dirs:
                ## "root" of package name
                dirSplit = dir.split('_')
                
                ## only process BioA and BioP
                if not (dirSplit[0] == "BioA" or dirSplit[0] == "BioP"):
                    continue

                ## not enough params in split
                if len(dirSplit) == 1:
                    continue

                rootPackage = dirSplit[1]

                ## add path to "rootPackage" name
                if rootPackage in GLOBAL_primaryPathsDict:
                    paths = GLOBAL_primaryPathsDict[rootPackage]
                else:
                    paths = []

                paths.append(os.path.join(root, dir))
                GLOBAL_primaryPathsDict[rootPackage] = paths


        ## save file
        try:
            f = open(os.path.join(GLOBAL_directory, "index.txt"), 'w')
            f.seek(0)
            f.write(json.dumps(GLOBAL_primaryPathsDict))
            f.truncate()
            f.close()
        except:
            ShowMessageBox("Failed to save index file", "File Error", 'ERROR')

        return {'FINISHED'}

def LoadCache(context):
    ## declare globals we use
    global GLOBAL_directory
    global GLOBAL_primaryPathsDict
    global GLOBAL_missingCache

    ## load file
    afile = Path(os.path.join(GLOBAL_directory, "index.txt"))
    if afile.is_file():
        try:
            f = open(os.path.join(GLOBAL_directory, "index.txt"), 'r')
            data = f.read()
            GLOBAL_primaryPathsDict = json.loads(data)
            f.close()
        except:
            ShowMessageBox("Failed to load index file", "File Error", 'ERROR')

    ## load file
    afile = Path(os.path.join(GLOBAL_directory, "missingIndex.txt"))
    if afile.is_file():
        try:
            f = open(os.path.join(GLOBAL_directory, "missingIndex.txt"), 'r')
            data = f.read()
            GLOBAL_missingCache = json.loads(data)
            f.close()
        except:
            ShowMessageBox("Failed to load missing index file", "File Error", 'ERROR')

class PostImportFix(bpy.types.Operator):
    bl_idname = "me.postimport"
    bl_label = "Post Import Fixes"
    
    def execute(self, context):
        ## Deselect everything
        bpy.context.view_layer.objects.active = None
        for obj in bpy.context.scene.objects:
            obj.select_set(False)

        ### Select all meshes
        sel_objs = [obj for obj in bpy.context.scene.objects if obj.type == 'MESH']

        ## Apply transform (must be after import else does not seem to work :( )
        ## Required for full map scaling to work
        ## DONT APPLY SCALE, it will fuck up the normals
        for obj in sel_objs:
            obj.select_set(True)            
            bpy.ops.object.transform_apply(location = True, scale = False, rotation = True) 
            bpy.context.view_layer.objects.active = None
            obj.select_set(False)


        ## Scale -1 * y or x it depends on level so...leave this for person to do
        ##bpy.context.view_layer.objects.active = obj
        ##bpy.context.view_layer.objects.active.scale = (-1 * bpy.context.view_layer.objects.active.scale.x, bpy.context.view_layer.objects.active.scale.y, bpy.context.view_layer.objects.active.scale.z)

        return {'FINISHED'}

class ImportProcess(bpy.types.Operator):
    bl_idname = "me.import"
    bl_label = "Import"

    ## Current model being processed
    currentModel = ""; ##set(bpy.data.objects)

    ## We need to pick and arbitrary origin else this process will not work
    ## dont clear (in one project if do multi import this needs to stay fixed)
    originPosX = 0   
    originPosY = 0
    originPosZ = 0
    originScaleX = 0
    originScaleY = 0
    originScaleZ = 0
    originRotX = 0
    originRotY = 0
    originRotZ = 0

    ## Track a previously used directories (so speed up processing)
    modelInclude = set(['StaticMesh3'])
    matInclude = set(['Material3', 'MaterialInstanceConstant'])
    imgInclude = set(['Texture2D'])

    ## temp info
    currentModel = ""
    previousPaths = set()
    previousFoundData = {}
    previousImportedModels = {}
    previousImportedTextures = {}
    modelResult = ""
    
    def execute(self, context):
        ## start tracking run time
        start_time = time.time()

        ## declare globals we use
        global GLOBAL_matLibrary
        global GLOBAL_importTexture
        global GLOBAL_importLights
        global GLOBAL_mergePointLight
        global GLOBAL_mergeDistance
        global GLOBAL_directory
        global GLOBAL_local
        global GLOBAL_mapFile
        global GLOBAL_lightFile
        global GLOBAL_primaryPathsDict
        global GLOBAL_useCache
        global GLOBAL_genShader
        global GLOBAL_complexMaterial
        global GLOBAL_missingCache

        CollectVariables(context)

        ## using cache load it (else it will be built as we go)
        if GLOBAL_useCache:
            LoadCache(context)

        ## clear info for reuse (we cannot know if previously imported models still exist)
        self.currentModel = ""
        self.previousPaths = set()
        self.previousFoundData = {}
        self.previousImportedModels = {}
        self.previousImportedTextures = {}
        self.modelResult = ""
        
        if GLOBAL_complexMaterial:
            GLOBAL_genShader = CreateGenShaderGroup("Gen Shader")

        self.LoadModels()
        self.LoadMaterials()
        self.LoadLights()

        if GLOBAL_useCache:
            try:
                f = open(os.path.join(GLOBAL_directory, "missingIndex.txt"), 'w')
                f.seek(0)
                f.write(json.dumps(GLOBAL_missingCache))
                f.truncate()
                f.close()
            except:
                ShowMessageBox("Failed save missing index file", "File Error", 'ERROR')

        print("Total Run Time ::: --- %s seconds ---" % (time.time() - start_time))
        return {'FINISHED'}

    def LoadModels(self):
        ## declare globals we use
        global GLOBAL_mapFile

        ## start tracking run time
        start_time = time.time()
    
        ## Load map file
        try:
            f = open(GLOBAL_mapFile, 'r')
        except ValueError as err:
            ShowMessageBox("Failed to open map file < " + GLOBAL_mapFile + " >", "File Error", 'ERROR')
            print(err.args)
            return

        ## How many imported obejcts
        modelNumber = 0

        colectionMaster = bpy.context.scene.collection

        collectionModel = bpy.data.collections.new("Models")
        bpy.context.scene.collection.children.link(collectionModel)

        collectionVol = bpy.data.collections.new("Volumetrics")
        bpy.context.scene.collection.children.link(collectionVol)

        ## Process map line by line
        ## Load model from name
        ## Place in the world
        ## Attempt to load material or setup the default material
        for line in f:
            ## If matched we know we have a model
            self.modelResult = line.split(",")

            if self.modelResult:
                ## Skip malformed data/header
                if self.modelResult[0] == "ID":
                    continue
                if self.modelResult[0] == "":
                    continue
                if self.modelResult[0] == ",":
                    continue

                ## check correct column count
                if len(self.modelResult) < 12:
                    ShowMessageBox("CSV map file should have 12 columns", "CSV Mismatch", 'ERROR')

                ## Check if already imported this model (copy it rather than re-import, so fast, much speed)
                modelName = self.modelResult[1]
                package = self.modelResult[11]
                self.currentModel = self.LocateAlreadyImportedModel(modelName)

                ## Import model
                if (self.currentModel == ""):
                    self.LoadModel(modelName, package)
                
                if (self.currentModel == ""):
                    continue
                
                ## New name
                self.currentModel.name = self.modelResult[0] + "_" + modelName; ## self.currentModel.name
                
                ## Put in collection
                if "Volumetric" in modelName or "SphereMesh" in modelName:
                    collectionVol.objects.link(self.currentModel)
                    colectionMaster.objects.unlink(self.currentModel)
                else:
                    collectionModel.objects.link(self.currentModel)
                    colectionMaster.objects.unlink(self.currentModel)
                
                ## track number of models imported
                modelNumber = modelNumber + 1
                bNegativeScaleY = False
                
                ## Not initial model import
                if modelNumber > 1:
                    ## Need to update location
                    x = self.modelResult[2]
                    y = self.modelResult[3]
                    z = self.modelResult[4]
                    self.UpdateLocation(x, y, z)
                    
                    ## Need to update scale
                    x = self.modelResult[8]
                    y = self.modelResult[9]
                    if float(y) > 0:
                        bNegativeScaleY = True

                    z = self.modelResult[10]
                    self.UpdateScale(x, y, z)
                    
                    ## Need to update rotation
                    x = self.modelResult[5]
                    y = self.modelResult[6]
                    if bNegativeScaleY:
                        y = str(-float(y))

                    z = self.modelResult[7]
                    self.UpdateRotation(x, y, z)
                    
                    self.UpdateMaterial()

                ## Initial model import so gather origin data
                elif modelNumber == 1:            
                    ## Need to update location
                    x = self.modelResult[2]
                    y = self.modelResult[3]
                    z = self.modelResult[4]
                    self.UpdateLocationOrigin(x, y, z)
                    
                    ## Need to update scale
                    x = self.modelResult[8]
                    y = self.modelResult[9]
                    if float(y) > 0:
                        bNegativeScaleY = True

                    z = self.modelResult[10]
                    self.UpdateScale(x, y, z)
                    
                    ## Need to update rotation
                    x = self.modelResult[5]
                    y = self.modelResult[6]
                    if bNegativeScaleY:
                        y = str(-float(y))

                    z = self.modelResult[7]
                    self.UpdateRotation(x, y, z)
                    
                    self.UpdateMaterial()

        print("Model Import Run Time ::: --- %s seconds ---" % (time.time() - start_time))

    def LoadMaterials(self):
        ## declare globals we use
        global GLOBAL_matLibrary

        if GLOBAL_matLibrary != "":
            self.LoadMaterialsFromReferenceFile()

        print ("LOAD MATS")
        self.InitAllUnMatchedMaterials()

    def LoadLights(self):
        ## declare globals we use
        global GLOBAL_mergeDistance
        global GLOBAL_importLights
        global GLOBAL_mergePointLight
        global GLOBAL_lightFile

        importedLights = False;

        if (GLOBAL_importLights):
            try:
                ## Load light file
                f = open(GLOBAL_lightFile, 'r')
            except:
                ShowMessageBox("Failed to open lighting file < " + GLOBAL_lightFile + " >", "File Error", 'ERROR')
                return
               
            lightCount = 0;
            importedLights = True

            collectionLight = bpy.data.collections.new("Lights")
            bpy.context.scene.collection.children.link(collectionLight)

            collectionLightMerged = bpy.data.collections.new("LightsMerged")
            bpy.context.scene.collection.children.link(collectionLightMerged)

            collectionLightOld = bpy.data.collections.new("LightsOld")
            bpy.context.scene.collection.children.link(collectionLightOld)

            ## Process lights line by line
            for line in f:
                ## If matched we know we have a model
                lightResult = line.split(",")
                
                if lightResult:
                    ## Skip malformed data/header
                    if lightResult[0] == "ID":
                        continue
                    if lightResult[0] == "":
                        continue
                    if lightResult[0] == ",":
                        continue

                    ## check correct column count
                    if len(lightResult) < 16:
                        ShowMessageBox("CSV lighting file should have 16 columns", "CSV Mismatch", 'ERROR')
                    
                    lightType = lightResult[1]
                    
                    if lightType == "SpotLightComponent":
                        lightType = "SPOT"
                    elif lightType == "PointLightComponent":
                        lightType = "POINT"
                    else:
                        lightType = "";
                        
                    if lightType != "":
                        ## Light needs unique name (has random dup issues so add random)
                        lightCount = lightCount + 1
                        x = lightResult[2] ## Put x pos in name (for my own testing purpose)
                        lightName = "light_" + str(lightCount) + "_" + x + "_" + str(random.randint(1,100000))
                        
                        ## Create light datablock
                        light_data = bpy.data.lights.new(name="light-data", type=lightType)
                        
                        ## Set the radius (blender/ue3 have different meanings here)
                        rad = float(lightResult[11])
                        rad = rad/10
                        
                        ## Set the brightness
                        bright = float(lightResult[12])
                        light_data.energy = bright * rad
                        
                        ## Set the RGBA
                        r = float(lightResult[13])/255
                        g = float(lightResult[14])/255
                        b = float(lightResult[15])/255
                        ## a = lightResult[16]
                        light_data.color = (r, g, b)

                        ## set some default values for now
                        if lightType == "SPOT":
                            light_data.spot_blend = 0.75
                            light_data.spot_size = 1.570796 ## 90 deg

                        ## Create new object, pass the light data 
                        self.currentModel = bpy.data.objects.new(name=lightName, object_data=light_data)

                        ## Link object to collection in context
                        ##bpy.context.collection.objects.link(self.currentModel)
                        collectionLight.objects.link(self.currentModel)
                    
                    if (self.currentModel == ""):
                        continue

                    ##collectionLight.objects.link(self.currentModel)
                    ##colectionMaster.objects.unlink(self.currentModel)

                    ## Need to update location
                    x = lightResult[2]
                    y = lightResult[3]
                    z = lightResult[4]
                    self.UpdateLocation(x, y, z)
                    
                    ## Skip light scale/rotation for now
                    ## Need to update scale
                    ##x = lightResult[8]
                    ##y = lightResult[9]
                    ##z = lightResult[10]
                    ##self.UpdateScale(x, y, z)
                    
                    ## Need to update rotation
                    ##x = lightResult[5]
                    ##y = lightResult[6]
                    ##z = lightResult[7]
                    ##self.UpdateRotation(x, y, z)
            
            ## Try to reduce number of lights
            ## Find lights (POINT) close to each other
            ## Attempt to combine data and create a single light
            ## As a note this process would need to be done before any transforms can be applied
            ## This seems like a good idea but .....
            ## could place lights inside models, could cause inapproriate shadows
            ## could cause muted lighting, etc....
            ## May be useful for scenes with 1000+ lights but for scenes with 0-500, does not seem worth it...
            lightsUsed = set();
            if importedLights and GLOBAL_mergePointLight:
                print("ATTEMPTING TO MERGE LIGHTS")
                print(collectionLight.all_objects)
                lightCount = 0;
                for objPrimary in collectionLight.all_objects:
                    primaryLoc = objPrimary.location
                    primaryType = objPrimary.data.type

                    ## Skip spot lights for now (because rotation matters, how can you merge rotation??)
                    if primaryType != "POINT":
                        continue
                    
                    ## Light already part of a merged set so skip it (dont double merge)
                    if objPrimary in lightsUsed:
                        continue
                    
                    lightsToMerge = set();

                    for objSecondary in collectionLight.all_objects:
                        ## Not same as this
                        if objPrimary == objSecondary:
                            continue

                        ## Light already part of a merged set so skip it (dont double merge)
                        if objSecondary in lightsUsed:
                            continue
                        
                        ## Skip light if is different type
                        if primaryType != objSecondary.data.type:
                            continue
                        
                        ## Check distance
                        secondaryLoc = objSecondary.location
                        distance = (primaryLoc - secondaryLoc).length
                        ##print("DISTANCE : " + str(distance))
                        if (distance <= GLOBAL_mergeDistance):
                            ## Add light to list, (dont add primary it is the starting point)
                            lightsToMerge.add(objSecondary)
                            lightsUsed.add(objSecondary)

                    print("FOUND THESE LIGHTS TO MERGE : ")

                    ## Did not find lights to merge so continue loop
                    if len(lightsToMerge) == 0:
                        continue

                    ## Mark main light as used
                    lightsUsed.add(objPrimary)
                    
                    newLoc = primaryLoc
                    x = primaryLoc[0]
                    y = primaryLoc[1]
                    z = primaryLoc[2]
                    newColor = objPrimary.data.color
                    newBrightness = objPrimary.data.energy
                    ## Loop over lights being merged
                    for light in lightsToMerge:         
                        ## Get data
                        secondaryLoc = light.location
                        secondaryColor = light.data.color
                        secondaryBrightness = light.data.energy
                        
                        ## Calculate merged data
                        newBrightness = newBrightness + secondaryBrightness
                        newColor[0] = newColor[0] + secondaryColor[0]
                        newColor[1] = newColor[1] + secondaryColor[1]
                        newColor[2] = newColor[2] + secondaryColor[2]
                        newLoc = newLoc + secondaryLoc
                        x = x + secondaryLoc[0]
                        y = y + secondaryLoc[1]
                        z = z + secondaryLoc[2]
                        
                        ## We want to move the used lights to a collection (so can be hidden easily)
                        collectionLight.objects.unlink(light)
                        collectionLightOld.objects.link(light)
                        
                    count = (len(lightsToMerge) + 1)
                    newBrightness = newBrightness / count
                    newBrightness = newBrightness * 1.125 ## Increase by 1/8 due to lights moving away from surfaces
                    newColor[0] = newColor[0] / count
                    newColor[1] = newColor[1] / count
                    newColor[2] = newColor[2] / count
                    newLoc = newLoc / count
                    x = x / count
                    y = y / count
                    z = z / count
                    newLoc[0] = x
                    newLoc[1] = y
                    newLoc[2] = z
                        
                    lightCount = lightCount + 1
                    lightName = "light_merged_" + str(lightCount) + "_" + str(random.randint(1,100000))
                    
                    ## Create light datablock
                    light_data = bpy.data.lights.new(name="light-data", type="POINT")
                    light_data.energy = newBrightness
                    light_data.color = newColor

                    ## Create new object, pass the light data 
                    self.currentModel = bpy.data.objects.new(name=lightName, object_data=light_data)

                    ## Set position (skip rot/scale for now)
                    self.currentModel.location = newLoc

                    ## Link object to collection in context
                    collectionLightMerged.objects.link(self.currentModel)

    ###############################################
    #####             Search Util             #####
    ###############################################

    def LocateAlreadyImportedModel(self, fileName):
        if fileName in self.previousImportedModels.keys():
            oldModel = self.previousImportedModels[fileName];
            newModel = oldModel.copy()
            newModel.data = oldModel.data.copy()
            bpy.context.collection.objects.link(newModel)
            return newModel;

        return "";

    def LocateAlreadyImportedTexture(self, fileName):
        if fileName in self.previousImportedTextures.keys():
            texture = self.previousImportedTextures[fileName];
            return texture;

        return "";
       
    ## Find all folders that match the package "root"
    ## We prefer directories that match the package "root" 
    ##  (higher chance of getting package specific items)
    ## BioA_CitHub_110 will check all folders with CitHub i.e. package "root"
    def LocatePrimaryPaths(self, sourcePackage):
        ## declare globals we use
        global GLOBAL_directory
        global GLOBAL_useCache
        global GLOBAL_primaryPathsDict

        primaryPaths = set()
        
        # BioA_CitHub_110 -> CitHub
        sourcePackageSplit = sourcePackage.split('_')
        
        ## only process BioA and BioP
        if not (sourcePackageSplit[0] == "BioA" or sourcePackageSplit[0] == "BioP"):
            return ""

        ## not enough params in split
        if len(sourcePackageSplit) == 1:
            return ""

        mainPackage = sourcePackageSplit[1]

        ## use cache (may be from loaded index or built on the fly)
        if mainPackage in GLOBAL_primaryPathsDict:
            primaryPaths = GLOBAL_primaryPathsDict[mainPackage]
            return primaryPaths

        ## prefer directories that match the mainPackage 
        ##  (higher chance of getting package specific items)
        for root, dirs, files in os.walk(GLOBAL_directory):
            dirs = [d for d in dirs if mainPackage in d]
            for dir in dirs:
                primaryPaths.add(os.path.join(root, dir))

        ## if using cache index this is already built don't waste time updating it
        if not GLOBAL_useCache:
            GLOBAL_primaryPathsDict[mainPackage] = primaryPaths

        return primaryPaths

    ## Locate file and return path to file
    def LocateFileOnDisk(self, fileName, fileExtension, dirIncludes, package):
        ## declare globals we use
        global GLOBAL_directory
        global GLOBAL_useCache
        global GLOBAL_primaryPathsDict
        global GLOBAL_missingCache

        fileName = fileName.strip()
        
        ## empty name obviously cannot import, skip them
        if fileName == "":
            return "";
        
        ## name is to short, skip them
        if len(fileName) < 4:
            return "";
        
        ## dummy_material is what materials that have a missing name are called, skip them
        if "dummy_material" in fileName:
            return "";

        ## materials with _REF are pulled from reference file, skip them
        if fileName[-4] == "_REF":
            return "";

        ## Check previous imports
        fileNameFull = fileName + "." + fileExtension
        if fileNameFull in self.previousFoundData.keys():
            return self.previousFoundData[fileNameFull]

        ## skip files we could not find
        if fileNameFull in GLOBAL_missingCache:
            return ""

        ## Try previously used path (faster to reuse, trying to save processing time)
        if len(self.previousPaths) != 0:
            for previousPath in self.previousPaths:
                for dirInclude in dirIncludes:
                    previousPath = os.path.join(previousPath, dirInclude)
                    potentialFile = os.path.join(previousPath, fileNameFull)
                    afile = Path(potentialFile)
                    if afile.is_file():
                        path_to_file = potentialFile
                        if GLOBAL_local:
                            path_to_file = path_to_file[2:]
                        else:
                            path_to_file = path_to_file

                        print("FOUND PATH: " + path_to_file)
                        self.previousFoundData[fileNameFull] = path_to_file
                        return path_to_file

        ## First check primary package "root"
        primaryPaths = self.LocatePrimaryPaths(package)
        for primaryPath in primaryPaths:
            ## Else check all, Dirs and sub dirs
            for root, dirs, files in os.walk(primaryPath):
                ## Keep based on includes
                dirs = [d for d in dirs if d in dirIncludes]
                
                for dir in dirs:
                    for root2, dirs2, files2 in os.walk(os.path.join(root, dir)):
                        ## All files in dir
                        for afile in files2:
                            ## normal processing of files.... see below
                            ## Must have file extension
                            if not afile.endswith('.' + fileExtension):
                                continue;

                            if fileNameFull != os.path.basename(afile):
                                continue;

                            ## Cache
                            head, tail = os.path.split(root2)
                            self.previousPaths.add(head)
                            path_to_file = os.path.join(root2, afile)

                            if GLOBAL_local:
                                path_to_file = path_to_file[2:]
                            else:
                                path_to_file = path_to_file

                            print("FOUND PATH: " + path_to_file)
                            self.previousFoundData[fileNameFull] = path_to_file
                            return path_to_file

        ## Else check all, Dirs and sub dirs
        for root, dirs, files in os.walk(GLOBAL_directory):
            ## Keep based on includes
            dirs = [d for d in dirs if d in dirIncludes]
            
            ## All files in dir
            for afile in files:
                ## Must have file extension
                if not afile.endswith('.' + fileExtension):
                    continue;

                if fileNameFull != os.path.basename(afile):
                    continue;

                ## Cache
                head, tail = os.path.split(root)
                self.previousPaths.add(head)
                path_to_file = os.path.join(root, afile)

                if GLOBAL_local:
                    path_to_file = path_to_file[2:]
                else:
                    path_to_file = path_to_file

                print("FOUND PATH: " + path_to_file)
                self.previousFoundData[fileNameFull] = path_to_file
                return path_to_file

        ## cache missing files so can skip later
        GLOBAL_missingCache.append(fileNameFull)
        print("FAILED TO FIND PATH FOR : " + fileNameFull)
        return ""


    ###############################################
    #####             Materials               #####
    ###############################################

    ## Attempt to combine materials of same name
    def UpdateMaterial(self):
        for slot in self.currentModel.material_slots:
            matched = re.match("^.*(\.\d*)$", slot.name)
            if bool(matched):
                self.replace_material(self.currentModel, slot.name, slot.name[0:-4])

     ## Material Replacement Method
    def replace_material(self, object,old_material,new_material):
        """
        replace a material in blender.
        params:
            object - object for which we are replacing the material
            old_material - The old material name as a string
            new_material - The new material name as a string
        """
        ob = object
        hasMats = False
        
        try:
            om = bpy.data.materials[old_material]
            nm = bpy.data.materials[new_material]
            hasMats = True
        except:
            hasMats = False
        
        if hasMats:
            materialnum = len(ob.data.materials)

            for i in range(0,materialnum):
                # test if the old material is in the list of used materials and replace it
                if ob.data.materials[i] == om:
                    ob.data.materials[i] = nm

    ## Initialize materials with default node setup, if they were not autoloaded
    def InitAllUnMatchedMaterials(self):
        ## declare globals we use
        global GLOBAL_importTextures
        global GLOBAL_genShader
        global GLOBAL_complexMaterial

        # gap between nodes
        spacing = 100
        spacing2 = 300
        spacing3 = 600
          
        ## Clean unused materials created during import (will speed searching up 10x)
        for material in bpy.data.materials:
            if not material.users:
                bpy.data.materials.remove(material)  
        
        ## Setup materials with BSDF two images (diff and normal map)
        for mat in bpy.data.materials:      
            mat.use_nodes = True
            
            ## Skip reference material (ie ones we pulled from file)
            if mat.name[-4:] == "_REF":
                continue;
            
            #Clear the Nodes
            tree = mat.node_tree
            nodes = mat.node_tree.nodes
            bsdf = ""
            output = ""

            if GLOBAL_complexMaterial:
                for node in nodes:
                    if node.type != 'OUTPUT_MATERIAL': # skip the material output node as we'll need it later
                        nodes.remove(node)
                    else:
                        output = node
            else:
                for node in nodes:
                    if node.type != 'OUTPUT_MATERIAL':  # skip the material output/bsdf node as we'll need it later
                        if node.type != 'BSDF_PRINCIPLED':
                            nodes.remove(node)
                        else:
                            bsdf = node

            ## when using gen shader create, more todo here
            if GLOBAL_complexMaterial:
                shader = GLOBAL_genShader
                genNode = mat.node_tree.nodes.new("ShaderNodeGroup")
                genNode.node_tree = bpy.data.node_groups['Gen Shader']
                genNode.location = (-200, 200)
                mat.node_tree.links.new(genNode.outputs["BSDF"], output.inputs["Surface"])

            ## Locate .mat file
            if GLOBAL_importTextures and IsBlender3X():
                ## find first object that uses this (this limits imports to the same package, need a better solution)
                exampleObj = ""
                for obj in bpy.data.objects:
                    for slot in obj.material_slots:
                        if slot.material == mat:
                            exampleObj = obj
                            break
                    if exampleObj != "":
                        break

                ## NEED TO GET THE OBJECT SO CAN SEE THE PACKAGE
                if exampleObj == "":
                    package = ""
                else:
                    package = exampleObj["Package"]

                ## mat and props.txt file
                path = self.LocateFileOnDisk(mat.name, "mat", self.matInclude, package)
                pathProps = self.LocateFileOnDisk(mat.name + ".props", "txt", self.matInclude, package)
                
                diffY = 400
                specY = 0
                emisY = -125
                opacY = -175
                normY = -250

                if path != "":
                    ## Load map file
                    try:
                        f = open(path, 'r')
                    except:
                        ShowMessageBox("Failed to open material file < " + path + " >", "File Error", 'ERROR')
                    
                    ## Attempt to load material
                    for line in f:
                        ## If matched we know we have a model
                        textureResult = line.split("=")

                        if textureResult:
                            if textureResult[0] == "Diffuse":
                                img = self.LoadImage(textureResult[1], package)
                                if img != None:
                                    ## Nodes
                                    texImage = mat.node_tree.nodes.new("ShaderNodeTexImage")

                                    ## Location
                                    texImage.location = (-400, diffY)

                                    ## Link img
                                    texImage.image = img

                                    ## ignore alpha channel
                                    texImage.image.alpha_mode = 'NONE'

                                    ## Connect Everything
                                    if GLOBAL_complexMaterial:
                                        mat.node_tree.links.new(texImage.outputs[0], genNode.inputs["Diffuse"])
                                    else:
                                        mat.node_tree.links.new(texImage.outputs[0], bsdf.inputs[0])
                                
                            if textureResult[0] == "Normal":
                                img = self.LoadImage(textureResult[1], package)
                                if img != None:
                                    ## Nodes
                                    normTex = mat.node_tree.nodes.new("ShaderNodeTexImage")

                                    ## Location
                                    normTex.location = (-800, normY)

                                    ## Link img and set non-color
                                    normTex.image = img
                                    normTex.image.colorspace_settings.name = 'Non-Color'

                                    ## Connect Everything
                                    if GLOBAL_complexMaterial:
                                        mat.node_tree.links.new(normTex.outputs[0], genNode.inputs["Normal"])
                                    else:
                                        ## nodes
                                        normCurve = mat.node_tree.nodes.new("ShaderNodeRGBCurve")
                                        normMap = mat.node_tree.nodes.new("ShaderNodeNormalMap")

                                        ## Location
                                        normCurve.location = (-500, normY)
                                        normMap.location = (-200, normY)

                                        ## Setup 'RGB Curve'
                                        normCurve.mapping.curves[1].points[0].location = (0,1)
                                        normCurve.mapping.curves[1].points[1].location = (1,0)

                                        mat.node_tree.links.new(normTex.outputs[0], normCurve.inputs[1])
                                        mat.node_tree.links.new(normCurve.outputs[0], normMap.inputs[1])
                                        mat.node_tree.links.new(normMap.outputs[0], bsdf.inputs["Normal"])

                            if textureResult[0] == "Specular":
                                img = self.LoadImage(textureResult[1], package)
                                if img != None:
                                    ## Nodes
                                    specImage = mat.node_tree.nodes.new("ShaderNodeTexImage")

                                    ## Location
                                    specImage.location = (-1000, specY)

                                    ## Link img
                                    specImage.image = img
                                    specImage.image.colorspace_settings.name = 'Non-Color'

                                    ## Connect Everything
                                    if GLOBAL_complexMaterial:
                                        mat.node_tree.links.new(specImage.outputs[0], genNode.inputs["Spec/Mask"])
                                    else:
                                        ## Nodes
                                        specSep = mat.node_tree.nodes.new("ShaderNodeSeparateColor")

                                        ## Location
                                        specSep.location = (-600, specY)

                                        ## Msk3
                                        ## ME2 Example = ParameterName = MSK3 Red- Spec Green - Emis/Relfc Blue -Sppwr
                                        mat.node_tree.links.new(specImage.outputs[0], specSep.inputs[0])
                                        mat.node_tree.links.new(specSep.outputs[0], bsdf.inputs[7])
                                        ## mat.node_tree.links.new(specSep.outputs[0], bsdf.inputs[9])
                                        mat.node_tree.links.new(specSep.outputs[2], bsdf.inputs[8])
                                
                            if textureResult[0] == "Opacity":
                                img = self.LoadImage(textureResult[1], package)
                                if img != None:
                                    ## Nodes
                                    opacImage = mat.node_tree.nodes.new("ShaderNodeTexImage")

                                    ## Location
                                    opacImage.location = (-1200, opacY)

                                    ## Link img
                                    opacImage.image = img
                                    opacImage.image.colorspace_settings.name = 'Non-Color'

                                    ## Connect Everything
                                    if GLOBAL_complexMaterial:
                                        temp = 0
                                        ##mat.node_tree.links.new(opacImage.outputs[0], genNode.inputs["Diffuse"])
                                    else:
                                        mat.node_tree.links.new(opacImage.outputs[0], bsdf.inputs[21])

                            if textureResult[0] == "Emissive":
                                img = self.LoadImage(textureResult[1], package)
                                if img != None:
                                    ## Nodes
                                    emisTex = mat.node_tree.nodes.new("ShaderNodeTexImage")

                                    ## Location
                                    emisTex.location = (-1400, emisY)

                                    ## Link img
                                    emisTex.image = img

                                    ## Connect Everything
                                    if GLOBAL_complexMaterial:
                                        ##mat.node_tree.links.new(emisTex.outputs[0], genNode.inputs["Emission Color"])
                                        mat.node_tree.links.new(emisTex.outputs[0], genNode.inputs["Emission Strength"])
                                    else:
                                        ##mat.node_tree.links.new(emisTex.outputs[0], bsdf.inputs[19])
                                        mat.node_tree.links.new(emisTex.outputs[0], bsdf.inputs[20])

    def LoadImage(self, imageName, package):
        img = self.LocateAlreadyImportedTexture(imageName);
        if (img != ""):
            return img

        path = self.LocateFileOnDisk(imageName, "tga", self.imgInclude, package)
        if path != "":
            img = bpy.data.images.load(path, check_existing=True)
            self.previousImportedTextures[imageName] = img
            return img

        return None 

    ## Attempt to match and replace materials with akready created materials
    def LoadMaterialsFromReferenceFile(self):
        ## declare globals we use
        global GLOBAL_matLibrary

        ## Import all reference materials
        path = GLOBAL_matLibrary
        with bpy.data.libraries.load(path) as (data_from, data_to):
            data_to.materials = data_from.materials

        ## Attempt to match material to a reference an repalce it
        for obj in bpy.data.objects:
            material_slots = obj.material_slots
            for mat in material_slots:
                currentMatName = mat.name
                
                split = currentMatName.split(".")
                currentMatNameSimplified = split[0]
                
                ## Reference materials end with _REF
                referenceMatName = currentMatNameSimplified + "_REF"
                
                self.replace_material(obj, currentMatName, referenceMatName)


    ###############################################
    #####              Model Utils            #####
    ###############################################

    ## Set current model location origin point
    def UpdateLocationOrigin(self, x, y, z):
        ## Get origin location
        self.originPosX = self.ConverToMeters(self.modelResult[2])
        self.originPosY = self.ConverToMeters(self.modelResult[3])
        self.originPosZ = self.ConverToMeters(self.modelResult[4])
        
        self.currentModel.location = (0, 0, 0)
        
    ## Set current model location
    def UpdateLocation(self, x, y, z):
        x = self.ConverToMeters(x) - self.originPosX
        y = self.ConverToMeters(y) - self.originPosY
        z = self.ConverToMeters(z) - self.originPosZ
        self.currentModel.location = (x, y, z)

    ## Set current model scale
    def UpdateScale(self, x, y, z):
        x = float(x)
        y = float(y)
        z = float(z)
        
        y = y * (-1) ## Fix scale issue

        self.currentModel.scale = (x, y, z)
        
    ## Set current model rotation
    def UpdateRotation(self, x, y, z):
        x = float(x)
        y = float(y)
        z = float(z)

        x = x * (-1) ## Fix rot issue

        self.currentModel.rotation_mode = 'XYZ'
        self.currentModel.rotation_euler = (y*math.pi/180, x*math.pi/180, z*math.pi/180)

    ## Convert imported float to meters
    def ConverToMeters(self, propertyValue):
        return float(propertyValue)/100;

    ## Search for model file location
    def LoadModel(self, modelName, package):
        old_objs = set(bpy.data.objects)
        
        ##global modelInclude
        self.currentModel = ""
        
        path = self.LocateFileOnDisk(modelName, "gltf", self.modelInclude, package)
        
        if path != "":
            bpy.ops.import_scene.gltf(filepath = path)
                
            self.currentModel = set(bpy.data.objects) - old_objs
            
            if isinstance(self.currentModel, set): 
                obj = list(self.currentModel)
                if (obj):
                    self.currentModel = obj[0]
                    self.currentModel["Package"] = package
                    self.previousImportedModels[modelName] = self.currentModel
                    return
            if isinstance(self.currentModel, object):
                self.currentModel = obj[0]
                self.currentModel["Package"] = package
                self.previousImportedModels[modelName] = self.currentModel
                return
            else:
                return


###############################################
#####              UTILS                  #####
###############################################

def ShowMessageBox(message = "", title = "Message Box", icon = 'INFO'):

    def draw(self, context):
        self.layout.label(text=message)

    print (icon + " :: " + title + " :: " + message);
    bpy.context.window_manager.popup_menu(draw, title = title, icon = icon)
    
def CollectVariables(context):
    ## declare globals we use
    global GLOBAL_matLibrary
    global GLOBAL_importTextures
    global GLOBAL_complexMaterial
    global GLOBAL_importLights
    global GLOBAL_mergePointLight
    global GLOBAL_mergeDistance
    global GLOBAL_directory
    global GLOBAL_local
    global GLOBAL_mapFile
    global GLOBAL_lightFile
    global GLOBAL_primaryPathsDict

    ## Get props
    scene = context.scene.me_tool

    ## Open map file, check provided, else search local
    GLOBAL_mapFile = scene.mapPath
    GLOBAL_lightFile = scene.lightPath
    if (GLOBAL_mapFile == ""):
        GLOBAL_mapFile = "map.csv"
        files = [f for f in os.listdir('.') if os.path.isfile(f)]
        for f in files:
            if f[0:4] == "map_":
                GLOBAL_mapFile = f
                
    if (GLOBAL_lightFile == ""):
        GLOBAL_lightFile = "lighting.csv"
        files = [f for f in os.listdir('.') if os.path.isfile(f)]
        for f in files:
            if f[0:9] == "lighting_":
                GLOBAL_lightFile = f

    library = scene.libraryPath
    GLOBAL_matLibrary = scene.matLibPath
    GLOBAL_importTextures = scene.importTextures
    GLOBAL_complexMaterial = scene.complexMaterial
    GLOBAL_importLights = scene.importLights
    GLOBAL_mergePointLight = scene.mergeLights
    GLOBAL_mergeDistance = scene.mergeLightsDistance
    GLOBAL_useCache = scene.useCache

    if (library == ""):
        GLOBAL_directory = "./"
        GLOBAL_local = True
    else:
        GLOBAL_directory = library
        GLOBAL_local = False
        
    if (GLOBAL_mapFile == ""):
        GLOBAL_mapFile = os.path.join("./", GLOBAL_mapFile)
        
    if (GLOBAL_lightFile == ""):
        GLOBAL_lightFile = os.path.join("./", GLOBAL_lightFile)
    
    ## verify .csv file add extension if needed
    if GLOBAL_mapFile[-4:] != ".csv":
        GLOBAL_mapFile = GLOBAL_mapFile + ".csv";

    ## verify .csv file add extension if needed    
    if GLOBAL_lightFile[-4:] != ".csv":
        GLOBAL_lightFile = GLOBAL_lightFile + ".csv";

def CreateGenShaderGroup(groupName):
    bpy.context.scene.use_nodes = True

    group = bpy.data.node_groups.new(groupName, "ShaderNodeTree")

    ## input
    groupIn = group.nodes.new("NodeGroupInput")
    groupIn.location = (-200, 0)

    ## inputs
    if IsBlender3X():
        group.inputs.new("NodeSocketColor", "Diffuse")
        group.inputs.new("NodeSocketColor", "Spec/Mask")
        group.inputs.new("NodeSocketColor", "Normal")
        group.inputs.new("NodeSocketColor", "Emission Color")
        group.inputs.new("NodeSocketFloat", "Emission Strength")

        group.inputs[0].default_value = (1,1,1,1)
        group.inputs[1].default_value = (1,1,1,1)
        group.inputs[2].default_value = (1,1,1,1)
        group.inputs[3].default_value = (1,1,1,1)
        group.inputs[4].default_value = 0
    elif IsBlender4X():
        group.interface.new_socket(name="Diffuse", in_out="INPUT", socket_type="NodeSocketColor")
        group.interface.new_socket(name="Spec/Mask", in_out="INPUT", socket_type="NodeSocketColor")
        group.interface.new_socket(name="Normal", in_out="INPUT", socket_type="NodeSocketColor")
        group.interface.new_socket(name="Emission Color", in_out="INPUT", socket_type="NodeSocketColor")
        group.interface.new_socket(name="Emission Strength", in_out="INPUT", socket_type="NodeSocketFloat")

        #groupIn.inputs[0].default_value = (1,1,1,1)
        #groupIn.inputs[1].default_value = (1,1,1,1)
        #groupIn.inputs[2].default_value = (1,1,1,1)
        #groupIn.inputs[3].default_value = (1,1,1,1)
        #groupIn.inputs[4].default_value = 0

    ## bsdf
    bsdf = group.nodes.new("ShaderNodeBsdfPrincipled")
    bsdf.location = (800, 0)

    ## multiply
    multi = group.nodes.new("ShaderNodeMix")
    multi.data_type = "RGBA"
    multi.blend_type = "MULTIPLY"
    multi.inputs[0].default_value = 1
    multi.location = (500, 200)

    ## separate color
    separate = group.nodes.new("ShaderNodeSeparateColor")
    separate.location = (300, -200)

    ## invert
    invert = group.nodes.new("ShaderNodeInvert")
    invert.location = (500, -300)

    ## rgb curve
    rgbCurve = group.nodes.new("ShaderNodeRGBCurve")
    rgbCurve.mapping.curves[1].points[0].location = (0,1)
    rgbCurve.mapping.curves[1].points[1].location = (1,0)
    rgbCurve.location = (200, -600)

    ## normal
    normal = group.nodes.new('ShaderNodeNormalMap')
    normal.location = (500, -600)

    ## output
    groupOut = group.nodes.new("NodeGroupOutput")
    groupOut.location = (1000, 0)
    if IsBlender3X():
        group.outputs.new("NodeSocketShader", "BSDF")
    elif IsBlender4X():
        group.interface.new_socket(name="BSDF", in_out="OUTPUT", socket_type="NodeSocketShader")

    if IsBlender3X():
        ## connections
        group.links.new(normal.outputs[0], bsdf.inputs[22])
        group.links.new(rgbCurve.outputs[0], normal.inputs[1])
        group.links.new(groupOut.inputs["BSDF"], bsdf.outputs[0])
        group.links.new(rgbCurve.inputs[1], groupIn.outputs["Normal"])
        group.links.new(multi.outputs[2], bsdf.inputs[0])
        group.links.new(groupIn.outputs["Diffuse"], multi.inputs[6])
        group.links.new(groupIn.outputs["Diffuse"], multi.inputs[7])
        group.links.new(groupIn.outputs["Spec/Mask"], separate.inputs[0])
        group.links.new(bsdf.inputs[20], groupIn.outputs["Emission Strength"])
        group.links.new(bsdf.inputs[19], groupIn.outputs["Emission Color"])
        group.links.new(separate.outputs[0], bsdf.inputs[6])
        group.links.new(separate.outputs[1], bsdf.inputs[7])
        group.links.new(separate.outputs[2], invert.inputs[1])
        group.links.new(invert.outputs[0], bsdf.inputs[9])
    #elif IsBlender4X():
        ## connections

    return group

def IsBlender3X():
    ver = bpy.app.version
    if (4, 0, 0) > ver and (3, 0, 0) <= ver:
        return True
    return False

def IsBlender4X():
    ver = bpy.app.version
    if (5, 0, 0) > ver and (4, 0, 0) <= ver:
        return True
    return False

###############################################
#####              My Addon               #####
###############################################

class MyProperties(bpy.types.PropertyGroup):
    ## UI props
    mergeLightsDistance: bpy.props.FloatProperty(
        name='Merge Distance',
        description='When merging lights <Merge Lights>, if they are this close together merge them',
        default=5.0
    )

    mergeLights: bpy.props.BoolProperty(
        name='Merge Lights',
        description='When there are to many lights <Import Data> we can merge them if they are within the <Merge Distance>',
        default=False
    )
    
    libraryPath: bpy.props.StringProperty(
        name='Library Path',
        description='Full path to the exported models example ( C:\Folder\ME2 )',
        default=""
    )
    
    matLibPath: bpy.props.StringProperty(
        name='Material Library Path',
        description='Full path to the blender file with materials example ( C:\Folder\File.blend )',
        default=""
    )
    
    mapPath: bpy.props.StringProperty(
        name='Map Path',
        description='Either name of csv or full path to the csv example ( BioA_CitHub_models.csv )',
        default=""
    )
    
    lightPath: bpy.props.StringProperty(
        name='Light Path',
        description='Either name of csv or full path to the csv example ( BioA_CitHub_lights.csv )',
        default=""
    )
    
    importTextures: bpy.props.BoolProperty(
        name='Import Textures',
        description='Search through material/texture data and attempt to setup default material and assign textures',
        default=False
    )

    complexMaterial: bpy.props.BoolProperty(
        name='Gen Shader - Outlet - [ ALPHA ]',
        description='Use the generic shader node',
        default=False
    )
    
    importLights: bpy.props.BoolProperty(
        name='Import Lights',
        description='Import lights from the Light csv <Import Data>',
        default=False
    )

    useCache: bpy.props.BoolProperty(
        name='Use Index',
        description='Use the index file (if found). This should increase performance since we dont need to scan the disk for which folders exist',
        default=True
    )
     
class ME_ImportPanel(bpy.types.Panel):
    bl_label = "ME Import"
    bl_idname = "OBJECT_PT_custom_panel"
    bl_category = "ME Import"
    bl_space_type = "PROPERTIES"
    bl_region_type = "WINDOW"
    bl_context = 'world'
    def draw(self, context):
        layout = self.layout
        me_tool = context.scene.me_tool
        
        layout.label(text="")
        layout.label(text="Build Index")
        box = layout.box()
        row = box.row()
        row.operator(BuildCache.bl_idname, icon='PLAY')
        
        layout.label(text="Import Data")
        box = layout.box()
        row = box.row()
        row.prop(me_tool, "libraryPath", text="Library")

        row = box.row()
        row.prop(me_tool, "mapPath", text="Map CSV")
        row = box.row()
        row.prop(me_tool, "lightPath", text="Lights CSV")
        
        layout.label(text="Materials")
        box = layout.box()
        row = box.row()
        row.prop(me_tool, "matLibPath", text="Source")
        
        layout.label(text="Textures (3.X only)")
        box = layout.box()
        row = box.row()
        row.prop(me_tool, "importTextures")
        row = box.row()
        row.prop(me_tool, "complexMaterial")
        
        layout.label(text="Lights")
        box = layout.box()
        row = box.row()
        row.prop(me_tool, "importLights")
        row = box.row()
        row.prop(me_tool, "mergeLights", text="Point light merge")
        row = box.row()
        row.prop(me_tool, "mergeLightsDistance")
        
        layout.label(text="Settings")
        box = layout.box()
        row = box.row()
        row.prop(me_tool, "useCache")

        layout.label(text="")
        layout.label(text="Import")
        row = layout.row()
        row.operator(ImportProcess.bl_idname, icon='IMPORT')
        row = layout.row()
        row.operator(PostImportFix.bl_idname, icon='FILE_REFRESH')
    
classes = (
    MyProperties,
    ImportProcess,
    PostImportFix,
    BuildCache,
    ME_ImportPanel,
)

def register():
    for cls in classes:
        bpy.utils.register_class(cls)
    bpy.types.Scene.me_tool = bpy.props.PointerProperty(type=MyProperties)

def unregister():
    del bpy.types.Scene.me_tool
    for cls in reversed(classes):
        bpy.utils.unregister_class(cls)

if __name__ == "__main__":
    register()