paramecio2fm/paramecio2/libraries/db/dbadmin.py

#!/usr/bin/env python3

"""
Paramecio2fm is a series of wrappers for Flask, mako and others and construct a simple headless cms.

Copyright (C) 2023 Antonio de la Rosa Caballero

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.
"""

import argparse
import os,traceback
import sys, inspect
import shutil
import re
from datetime import date
from pathlib import Path
from colorama import init, Fore, Back, Style
from importlib import import_module, reload
from paramecio2.libraries.db.webmodel import WebModel

sys.path.insert(0, os.path.realpath('.'))

try:
    from settings import config
except:
    #print('You need a settings directory with a paramecio2 configuration')
    #sys.exit(1)
    pass

def start():
    """Function for create and update mysql tables using webmodel classes and fields how source. 
    """
    
    connection=WebModel.connection()
    
    #connection.connect_to_db(WebModel.connections['default'])
    
    parser = argparse.ArgumentParser(description='A tool for create tables in databases using models from Cromosoma')

    parser.add_argument('--model', help='Model python path', required=True)
    
    parser.add_argument('--config', help='The config file', required=False)
    
    args = parser.parse_args()
    
    init()
    
    #Import config
    
    config_file='config'
    
    if args.config!=None:
        config_file=args.config
    
    try:
    
        config=import_module('settings.'+config_file)
        
    except:
        e = sys.exc_info()[0]
        v = sys.exc_info()[1]
            
        print(Fore.WHITE+Back.RED+Style.BRIGHT+"Config file not found: %s %s" % (e, v))
        
        exit(1)
    
    #print(WebModel.connections)
    
    if '/' in args.model:
        
        args.model=args.model.replace('/', '.')[:-3] #.replace('.py', '')
    
    try:
    
        model=import_module(args.model)
        
        for name, obj in inspect.getmembers(sys.modules[model.__name__]):
            if inspect.isclass(obj):
                if obj.__module__==args.model and hasattr(obj, 'webmodel'):
                    
                    WebModel.model[name.lower()]=obj(connection)
                    
                    
                    #WebModel.modelobj
        
    except:
        """
        e = sys.exc_info()[0]
        v = sys.exc_info()[1]
            
        print(Fore.WHITE+Back.RED+Style.BRIGHT +"Error, file with model not found: %s %s" % (e, v))
        """
        print("Exception in user code:")
        print("-"*60)
        traceback.print_exc(file=sys.stdout)
        print("-"*60)
        
        exit(1)
    
    #load the table of databases
    
    cursor=connection.query("show tables")
    
    table_exists=[]
    
    for row in cursor:
        table=list(row.values())[0]
        
        if table in WebModel.model:
            table_exists.append(table)
        
        #If don't want order
        #set([1,2,3,4]) - set([2,5])
        
    tables=list(WebModel.model.keys())
    
    #Array diff ordered
    
    new_tables=[x for x in tables if x not in table_exists]
    
    # Get foreignkeys
            
    # SELECT * FROM information_schema.TABLE_CONSTRAINTS WHERE CONSTRAINT_SCHEMA='catalogdev_db' AND information_schema.TABLE_CONSTRAINTS.CONSTRAINT_TYPE = 'FOREIGN KEY' ;
    
    foreignkey_fields={}
    
    #| CONSTRAINT_CATALOG | CONSTRAINT_SCHEMA | CONSTRAINT_NAME                         | TABLE_SCHEMA  | TABLE_NAME        | CONSTRAINT_TYPE |
    #+--------------------+-------------------+-----------------------------------------+---------------+-------------------+-----------------+
    #| def                | catalogdev_db     | product_id_attributesIDX                | catalogdev_db | attributes        | FOREIGN KEY     |

    
    #WebModel.connections
    
    db_name=WebModel.connections['default']['db']
    
    with connection.query('SELECT * FROM information_schema.TABLE_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=%s AND information_schema.TABLE_CONSTRAINTS.CONSTRAINT_TYPE = %s', [db_name, 'FOREIGN KEY']) as cursor:
        
        for row in cursor:
            if not row['TABLE_NAME'] in foreignkey_fields:
                foreignkey_fields[row['TABLE_NAME']]=[]
            
            foreignkey_fields[row['TABLE_NAME']]=row['CONSTRAINT_NAME'].replace('_{}IDX'.format(row['TABLE_NAME']), '')
        
        pass
    
    
    #If don't want order
    #new_tables=set(tables)-set(table_exists)
    
    #Need order new_tables
    
    changes=0
    
    #Create new tables
    
    if len(new_tables)>0:
        print(Style.BRIGHT+"Creating new tables...")
        
        changes+=1
        
        for table in new_tables:
            print(Style.NORMAL+"--Creating table "+table+"...")
            connection.query(WebModel.model[table].create_table())

        for table in new_tables:
            
            print("--Adding indexes and constraints for the new table "+table)

            for k_field, index in WebModel.arr_sql_index[table].items():
                print("---Added index to "+k_field)
                connection.query(index)
                
            for k_set, index_set in WebModel.arr_sql_set_index[table].items():
                
                if index_set!="":
                    connection.query(index_set)
                    print("---Added constraint to "+k_set)
                
            print("--Adding uniques elements for the new table")

            for k_field, unique_set in WebModel.arr_sql_unique[table].items():
                
                if unique_set!="":
                    connection.query(unique_set)
                    print("---Added unique to "+unique_set)

    #See if changes exists
    
    #Check if created tables are modified.
    
    try:
    
        model_old=import_module('backups.'+args.model)
        
        for name, obj in inspect.getmembers(sys.modules[model_old.__name__]):
            if inspect.isclass(obj):
                if obj.__module__=='backups.'+args.model and hasattr(obj, 'webmodel'):
                    
                    WebModel.model['old_'+name.lower()]=obj(connection)
        
        print(Style.BRIGHT+"Checking old versions of model for find changes...")
        
        for table in tables:
            
            #print(table)
            
            table_fields={table: {}}
            
            #  Field           | Type         | Null | Key | Default | Extra          |
            #| id              | int(11)      | NO   | PRI | NULL    | auto_increment |
            
            with connection.query('describe %s' % table) as cursor:
                #all_fields=cursor.fetchall()
                #print(all_fields)
                
                for row in cursor:
                    table_fields[table][row['Field']]={'type': row['Type'], 'key': row['Key']}
                    pass
            #print(table_fields)
        
        #connection.query("")
            #Check if new table
            
            #fields_to_add, fields_to_modify, fields_to_add_index, fields_to_add_constraint, fields_to_add_unique, fields_to_delete_index, fields_to_delete_unique, fields_to_delete_constraint, fields_to_delete
            
            fields_to_add=[]
            fields_to_modify=[]
            fields_to_add_index=[]
            fields_to_add_constraint=[]
            fields_to_add_unique=[]
            fields_to_delete_index=[]
            fields_to_delete_unique=[]
            fields_to_delete_constraint=[]
            fields_to_delete=[]
            
            old_table='old_'+table
            
            if not old_table in WebModel.model:
                WebModel.model[old_table]=WebModel.model[table]
            
            for f, v in WebModel.model[table].fields.items():
                
                #if not f in WebModel.model[old_table].fields:
                if not f in table_fields[table]:
                    
                    fields_to_add.append(f)
                    
                    #Add index
                    
                    if v.indexed==True:
                        
                        fields_to_add_index.append(f)
                        
                        changes+=1
                    
                    #Add unique
                    
                    if v.unique==True:
                        
                        fields_to_add_unique.append(f)
                        
                        changes+=1
                    
                    #Add constraint
                    
                    if v.foreignkey==True:
                        
                        fields_to_add_constraint.append(f)
                        
                        changes+=1
                    
                    changes+=1
                
                #If exists field in old webmodel and new
                
                else:
                    
                    v_old=WebModel.model[old_table].fields[f]
                    
                    if v.get_type_sql()!=v_old.get_type_sql():
                        
                        fields_to_modify.append(f)
                        
                        changes+=1
                
                    #Add index
                    
                    #if v.indexed==True and v_old.indexed==False:
                    if v.indexed==True and table_fields[table][f]['key']!='MUL':
                        
                        fields_to_add_index.append(f)
                        
                        changes+=1
                    
                    #if v.indexed==False and v_old.indexed==True:
                    if v.indexed==False and table_fields[table][f]['key']=='MUL' and v.foreignkey==False:
                        
                        fields_to_delete_index.append(f)
                        
                        changes+=1
                    
                    #Add unique
                    
                    #if v.unique==True and v_old.unique==False:
                    if v.unique==True and table_fields[table][f]['key']!='UNI':
                        
                        fields_to_add_unique.append(f)
                        
                        changes+=1
                        
                    #if v.unique==False and v_old.unique==True:
                    if v.unique==False and table_fields[table][f]['key']=='UNI':
                        
                        fields_to_delete_unique.append(f)
                        
                        changes+=1
                    
                    #Add constraint
                    
                    #if v.foreignkey==True and v_old.foreignkey==False:
                    if v.foreignkey==True and table_fields[table][f]['key']!='MUL':
                        
                        fields_to_add_constraint.append(f)
                        
                        changes+=1
                        
                    #if v.foreignkey==False and v_old.foreignkey==True:
                    if v.foreignkey==False and table_fields[table][f]['key']=='MUL':
                        
                        if table in foreignkey_fields:
                        
                            if f in foreignkey_fields[table]:
                            
                                fields_to_delete_constraint.append(f)
                                
                                changes+=1
            
            # Clean fields
            
            #for f, v in WebModel.model[old_table].fields.items():
            
            for f, v in table_fields[table].items():
                
                if not f in WebModel.model[table].fields:
                    
                    #Add constraint
                    
                    #if v.foreignkey==True:
                    
                    if table in foreignkey_fields:
                    
                        if f in foreignkey_fields[table]:
                            
                            fields_to_delete_constraint.append(f)
                            
                            changes+=1
                    
                    fields_to_delete.append(f)
                    
                    changes+=1
            
            WebModel.model[table].update_table(fields_to_add, fields_to_modify, fields_to_add_index, fields_to_add_constraint, fields_to_add_unique, fields_to_delete_index, fields_to_delete_unique, fields_to_delete_constraint, fields_to_delete)
                
            #for field_update in arr_update:
                
            
        #Make a for in fields, if the field not exist in old model, create, if is not same type, recreate. If no have index now, delete index, if is a new index, create, same thing with uniques
        
        #for field in WebModel.model
        
    except ImportError:
        
        pass
    
    except:
        
        print("Exception in user code:")
        print("-"*60)
        traceback.print_exc(file=sys.stdout)
        print("-"*60)
        
        exit(1)
    
    original_file_path=args.model.replace('.', '/')+'.py'
    
    backup_path='backups/'+original_file_path
    
    if changes>0:
        print(Style.BRIGHT+"Creating backup of the model. WARNING: DON'T DELETE BACKUPS DIRECTORY IF YOU WANT MAKE CHANGES IN THE FUTURE WITHOUT MODIFY DIRECTLY THE DATABASE")
        
        create_backup(original_file_path, backup_path)
        
    else:
        if not os.path.isfile(backup_path):
            create_backup(original_file_path, backup_path)
    
    # Execute script
    
    arr_script_model=args.model.split('.')
    
    arr_script_model.pop()
    
    script_model='.'.join(arr_script_model)+'.scripts.install'
    
    script_py=script_model.replace('.', '/')+'.py'

    if os.path.isfile(script_py):
        
        locked_file='/'.join(arr_script_model)+'/scripts/locked'
        
        if not os.path.isfile(locked_file):

            script_install=import_module(script_model)
            
            script_install.run()
            
            f=open(locked_file, 'w')
            
            f.write('OK')
            
            f.close()
    
    
    connection.close()
    
    #script_model=args.model+''
    
    print(Style.BRIGHT+"All tasks finished")
        
def create_backup(original_file_path, file_path):
    
    #Create copy
    
    path=os.path.dirname(file_path)
    
    p=Path(path)
        
    if not p.is_dir():
        p.mkdir(0o755, True)
        with open(path+'/__init__.py', 'w') as f:
            f.write("#!/usr/bin/env python3\n")
        
    #Create path
        
    if os.path.isfile(file_path):
        today = date.today()
        shutil.copy(file_path, file_path+'.'+today.strftime("%Y%M%d%H%M%S"))
    
    new_file=""
    
    f=open(original_file_path)
    
    for line in f:
        """
        new_line=line.replace("model[\"", "model[\"old_")
        new_line=new_line.replace("model['", "model['old_")
        
        new_line=new_line.replace("WebModel(\"", "WebModel(\"old_")
        new_line=new_line.replace("WebModel('", "WebModel('old_")
        """
        new_file+=line
    
    f.close()
    
    f=open(file_path, 'w')
    
    f.write(new_file)
    
    f.close()