Added gevent server for tasks
This commit is contained in:
parent
a9762c1a11
commit
0e0131334a
5 changed files with 449 additions and 8 deletions
225
servers/launcher.py
Normal file
225
servers/launcher.py
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import sys, os
|
||||
|
||||
sys.path.insert(0, os.path.realpath(os.path.dirname(__file__))+'/../../../')
|
||||
os.chdir(os.path.realpath(os.path.dirname(__file__))+'/../../../')
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from paramecio2.libraries.db.webmodel import WebModel
|
||||
from modules.pastafari2.models import tasks
|
||||
from modules.pastafari2.models.pastafari2 import ServerDbTask
|
||||
from modules.pastafari2.libraries.task import Task
|
||||
from settings import config
|
||||
from modules.pastafari2.libraries.configtask import config_task
|
||||
from multiprocessing.pool import Pool
|
||||
import importlib
|
||||
from paramecio2.libraries.db.sqlalchemy import SqlClass
|
||||
SqlClass.disable_pool=True
|
||||
|
||||
num_tasks=10
|
||||
|
||||
if hasattr(config, 'pastafari_num_tasks'):
|
||||
num_tasks=config.pastafari_num_tasks
|
||||
|
||||
def start(cli_args=None):
|
||||
|
||||
parser = argparse.ArgumentParser(description='A daemon used for make a task in a server.The results are saved in a sql database using task class')
|
||||
parser.add_argument('--task_id', help='The task to execute', required=True)
|
||||
|
||||
args = parser.parse_args(cli_args)
|
||||
|
||||
task_id=int(args.task_id)
|
||||
|
||||
conn=WebModel.connection()
|
||||
|
||||
task_model=tasks.Task(conn)
|
||||
|
||||
logtask=tasks.LogTask(conn)
|
||||
|
||||
arr_task=task_model.select_a_row(task_id)
|
||||
|
||||
if arr_task:
|
||||
|
||||
if (arr_task['user']!='' or arr_task['password']!='') and arr_task['path']!='':
|
||||
|
||||
config_task.remote_user=arr_task['user']
|
||||
config_task.remote_password=arr_task['password']
|
||||
config_task.remote_path=arr_task['path']
|
||||
|
||||
task_model.create_forms()
|
||||
task_model.reset_require()
|
||||
|
||||
task_model.set_conditions('WHERE id=%s', [task_id])
|
||||
|
||||
task_model.update({'password': ''})
|
||||
|
||||
"""
|
||||
if not commands_to_execute:
|
||||
print('Error: no task files')
|
||||
exit(1)
|
||||
"""
|
||||
server_model=ServerDbTask(conn)
|
||||
|
||||
if not arr_task['is_parent']:
|
||||
|
||||
#task=generate_task(arr_task, conn, task_id)
|
||||
|
||||
remote_user=arr_task['user']
|
||||
remote_password=arr_task['password']
|
||||
private_key=arr_task['ssh_key_priv']
|
||||
password_key=arr_task.get('ssh_key_password', '')
|
||||
|
||||
taskmod=importlib.import_module(arr_task.get('path', ''))
|
||||
|
||||
ssh_task=taskmod.ServerTask(arr_task['server'], conn, remote_user=remote_user, remote_password=remote_password, private_key=private_key, password_key=password_key, remote_path='pastafari2', task_id=task_id, data=json.loads(arr_task['data']))
|
||||
|
||||
ssh_task.exec()
|
||||
|
||||
|
||||
else:
|
||||
|
||||
# Select the servers and make all tasks asynchronous
|
||||
|
||||
#server_model.set_conditions(arr_task['where_sql_server'], [])
|
||||
|
||||
#server_model.yes_reset_conditions=False
|
||||
|
||||
c=task_model.set_conditions('WHERE parent_id=%s', [task_id]).select_count()
|
||||
#with conn.query('select count(*) from task where parent_id=%s', [task_id]) as cursor:
|
||||
|
||||
|
||||
#Update task with number of servers
|
||||
|
||||
task_model.set_conditions('WHERE id=%s', [task_id])
|
||||
|
||||
task_model.reset_require()
|
||||
|
||||
task_model.valid_fields=['num_servers']
|
||||
|
||||
task_model.update({'num_servers': c})
|
||||
|
||||
z=0
|
||||
|
||||
while z<c:
|
||||
|
||||
# Set the num of pools
|
||||
|
||||
#server_model.set_limit([z, num_tasks])
|
||||
|
||||
arr_tasks=task_model.set_conditions('WHERE parent_id=%s', [task_id]).set_limit([z, num_tasks]).select_to_array()
|
||||
|
||||
num_pools=len(arr_tasks)
|
||||
|
||||
arr_task_exec=[]
|
||||
|
||||
with Pool(processes=num_pools) as pool:
|
||||
|
||||
for one_task in arr_tasks:
|
||||
arr_task_exec.append(one_task)
|
||||
|
||||
pool.map(execute_task, arr_task_exec)
|
||||
|
||||
#for x in range(num_pools)
|
||||
#pool.
|
||||
pass
|
||||
|
||||
z+=num_tasks
|
||||
|
||||
pass
|
||||
|
||||
# Task done
|
||||
|
||||
task_model.set_conditions('WHERE id=%s', [task_id])
|
||||
|
||||
task_model.reset_require()
|
||||
|
||||
task_model.valid_fields=['status']
|
||||
|
||||
task_model.update({'num_servers': 1, 'status': 1})
|
||||
|
||||
conn.close()
|
||||
|
||||
exit(0)
|
||||
|
||||
def execute_task(arr_task):
|
||||
|
||||
conn=WebModel.connection()
|
||||
|
||||
remote_user=arr_task['user']
|
||||
remote_password=arr_task['password']
|
||||
private_key=arr_task['ssh_key_priv']
|
||||
password_key=arr_task.get('ssh_key_password', '')
|
||||
task_id=arr_task['id']
|
||||
|
||||
taskmod=importlib.import_module(arr_task.get('path', ''))
|
||||
|
||||
ssh_task=taskmod.ServerTask(arr_task['server'], conn, remote_user=remote_user, remote_password=remote_password, private_key=private_key, password_key=password_key, remote_path='pastafari2', task_id=task_id, data=json.loads(arr_task['data']))
|
||||
|
||||
ssh_task.exec()
|
||||
|
||||
conn.close()
|
||||
|
||||
del ssh_task
|
||||
|
||||
def execute_multitask(arr_task_exec=[]):
|
||||
|
||||
conn=WebModel.connection()
|
||||
|
||||
arr_task=arr_task_exec[0]
|
||||
task_id=arr_task_exec[1]
|
||||
server=arr_task_exec[2]
|
||||
os_server=arr_task_exec[3]
|
||||
|
||||
task=generate_task(arr_task, conn, task_id)
|
||||
|
||||
task.server=server
|
||||
task.os_server=os_server
|
||||
|
||||
task.exec()
|
||||
|
||||
conn.close()
|
||||
|
||||
del task
|
||||
|
||||
def generate_task(arr_task, conn, task_id):
|
||||
|
||||
|
||||
#taskssh.config=taskset.get('config', taskssh.config)
|
||||
#arr_task={k: v for k, v in arr_task.items() if v is not "" and v is not "[]"}
|
||||
|
||||
taskmod=importlib.import_module(arr_task.get('path', ''))
|
||||
|
||||
taskssh=taskmod.TaskServer(arr_task['server'], conn, task_id)
|
||||
|
||||
"""
|
||||
if arr_task['name_task']!='':
|
||||
taskssh.name_task=arr_task['name_task']
|
||||
|
||||
if arr_task['codename_task']!='':
|
||||
taskssh.codename_task=arr_task['codename_task']
|
||||
|
||||
if arr_task['description_task']!='':
|
||||
taskssh.description_task=arr_task['description_task']
|
||||
|
||||
|
||||
if arr_task['files']!='[]':
|
||||
taskssh.files=json.loads(arr_task['files'])
|
||||
|
||||
if arr_task['commands_to_execute']!='[]':
|
||||
taskssh.commands_to_execute=json.loads(arr_task['commands_to_execute'])
|
||||
|
||||
if arr_task['delete_files']!='[]':
|
||||
taskssh.delete_files=json.loads(arr_task['delete_files'])
|
||||
|
||||
if arr_task['delete_directories']!='[]':
|
||||
taskssh.delete_directories=json.loads(arr_task['delete_directories'])
|
||||
"""
|
||||
|
||||
taskssh.data=json.loads(arr_task['data'])
|
||||
|
||||
return taskssh
|
||||
|
||||
if __name__=='__main__':
|
||||
start()
|
||||
Loading…
Add table
Add a link
Reference in a new issue