/gnss_collector/engine/engine.py

# stdlilb python
import asyncio
import concurrent.futures
import datetime
import functools
import itertools
import logging
import multiprocessing
import multiprocessing as mp
import os
import socket
import sys 
import time
import traceback
from asyncio import shield, wait_for
from datetime import timedelta, datetime
from dataclasses import (field, dataclass)
from functools import reduce
from ipaddress import ip_address
from pathlib import Path
from tasktools.taskloop import TaskLoop
from typing import List, Dict, Union, overload, Any, Tuple
from collections.abc import Iterable

# contrib 
from rethinkdb import RethinkDB
from rich import print

# contrib: share exceptions as message
from tblib import pickling_support
pickling_support.install()

# contrib @dpineda
from networktools.time import timestamp, now
from networktools.colorprint import gprint, bprint, rprint
from networktools.library import my_random_string
from networktools.library import check_type
from networktools.library import (pattern_value,
                                  fill_pattern, context_split,
                                  gns_loads, gns_dumps)
from networktools.messages import MSGException, MessageLog

from networktools.time import gps_time, now
from basic_queuetools.queue import read_queue_gen
from basic_logtools.filelog import LogFile as FileLog
from crb import RingBuffer, Data 
from dataprotocols import BaseProtocol, Gsof, Eryo
from data_rdb import Rethink_DBS
from orm_collector.manager import SessionCollector, object_as_dict

# Tasktools
from tasktools.taskloop import coromask, renew, simple_fargs, simple_fargs_out
from tasktools.scheduler import TaskScheduler
# GSOF Protocol
from .steps import (CollectSteps as CSteps, DBSteps, Logger, ORMSteps,
                    ControlActions)
# DBS Rethinkdb
# same module
from .subscribe import SubscribeData
from .message import MessageManager

# from .async_mongo import AsyncMongoDB

rdb = RethinkDB()

def rdbnow():
    return rdb.iso8601(now().isoformat())

# base settings
try:
    from .conf.settings import COMMANDS, groups, dirs

except:
    from conf.settings import COMMANDS, groups, dirs

DATA_KEYS = {"DT_GEN":"dt_gen","DELTA_TIME":"latency"}
import time

def load_stations(server_name, datadb, log_path='~/log'):
    print("Obteniendo estaciones....", server_name, datadb)
    dbmanager = SessionCollector(
        log_path=log_path, 
        active='true', 
        server=server_name, 
        **datadb)
    u = [st for st in dbmanager.get_station_data(server=server_name)]
    print("STATIONS READED")
    print(u)
    dbmanager.close()
    return u


def load_databases(datadb, log_path='~/log'):
    print("Obteniendo datadb lista")
    dbmanager = SessionCollector(
        log_path=log_path, 
        **datadb)
    u = dbmanager.get_dbdata_data()
    print("Resultado...", u)
    dbmanager.close()
    return u


def active_server(server_name, datadb, log_path='~/log'):
    print("Activando server", server_name, datadb)
    dbmanager = SessionCollector(
        log_path=log_path, 
        **datadb)
    u = dbmanager.get_server_id(server_name)
    if u:
        dbmanager.active_server(u)
    dbmanager.close()
    return u


def deactive_server(server_name, datadb, log_path='~/log'):
    dbmanager = SessionCollector(log_path=log_path, **datadb)
    u = dbmanager.get_server_id(server_name)
    if u:
        dbmanager.deactive_server(u)
    dbmanager.close()
    return u


"""
Engine basic for collector
"""


class Aux:
    async def stop(self):
        pass


Value = Union[str, int, float, datetime]

@dataclass
class Engine(TaskScheduler):
    """
    A class for data adquisition, receive meshttp://www.cursodeprogramacion.cl/sages from anageser and
    save data on db

    """
    # ARGS: ordered and obligatory
    set_queue: List[mp.Queue]
    sleep_time: int
    est_by_proc: int
    stations: Dict[str, Dict[str, Value]]
    dbtype: Dict[str, Dict[str, Value]]
    protocol: Dict[str, Dict[str, Value]]
    status_sta: Dict[str, bool] # ok
    db_instances_sta: Dict[str,str] # ok
    status_conn: Dict[str, bool] # ok 
    db_data: Dict[str,Dict[str,str]] # can be better
    dump_list: Dict[str,str]
    proc_tasks: Dict[str,str]
    assigned_tasks: Dict[str,str]
    free_ids: Dict[str,str]
    wait: Dict[str,str]
    inc_msg: Dict[str,str]
    ids:List[str]
    idd:List[str]
    ipt:List[str]
    idm:List[str]
    ico:List[str]
    changes: Dict[str,str]
    gsof_timeout:int 
    sta_init: Dict[str,str]
    db_init:Dict[str, bool]
    db_connect:Dict[str, bool]
    status_tasks:Dict[str, Dict[str,str]]
    nproc:int
    idc: Dict[str,str]

    # KWARGS
    rdb_address: str = field(default_factory= lambda: "localhost")
    uin: int= 6
    
    args:List[Any] =  field(default_factory=list)
    kwargs:Dict[str, Any] = field(default_factory=dict)
    log_path: Path = Path.home() / "collector_log"
    server: str = "atlas"
    dt_criteria: int = 4
    raise_timeout: bool = False
    timeout: int = 15
    dbdata: Dict[str, Any] = field(default_factory=dict)
    collect_objects:Dict[str, BaseProtocol] = field(default_factory=lambda: dict(
            GSOF=Gsof,
            ERYO=Eryo
        ))
    database_objects:Dict[str, Any] = field(default_factory=lambda: dict(
            RethinkDB=Rethink_DBS,
            #Mongo=AsyncMongoDB
        ))
    folder:str = 'data'
    sep:str = '|'
    rethinkdb:Dict[str,  Any] = field(default_factory=dict)
    log_manager:Dict[str, str] = field(default_factory=dict)

    def __post_init__(self):
        self.set_queue_elems()
        self.set_stats_params()

        args = []
        kwargs_extra = {
            'ipt': self.ipt,
            'ico': self.ico,
            'assigned_tasks': self.assigned_tasks,
            'nproc': self.nproc,
            'sta_init': self.sta_init
        }
        print("ARGS", self.args)        
        print("KWARGS", self.kwargs)
        self.server_name = self.kwargs.get('server', "atlas")
        self.log_path = Path(self.kwargs.get('log_path', '~/log'))
        self.timeout = self.kwargs.get("timeout", 5)
        self.datadb = self.kwargs.get("dbdata", {})
        self.raise_timeout = self.kwargs.get("raise_timeout", False)

        print("datadb",self.datadb)
        self.kwargs.update(kwargs_extra)
        super().__init__(*self.args, **self.kwargs)
        #
        coros_callback_dict = {
            'run_task': self.process_data,
        }
        self.set_new_run_task(**coros_callback_dict)
        self.instances = dict()
        self.db_instances = dict()
        # more processing
        self.nproc = mp.cpu_count()
        # list of objects
          # LOAD DATA TO STATIONS
        self.tasks = dict()
        self.first_time = dict()
        # set the main task
        # must be in every new process ATENTION!
        self.message_manager = MessageManager(self)
        self.subscribe_data = SubscribeData(
            'collector_subscribe', self.queue_t2n)
        self.LOG_STA = check_type(os.environ.get('LOG_STA', False))
        ###############################################
        self.server = active_server(self.server_name, self.datadb)
        # table status
        self.stats = "STATIONS_STATS"


    def set_queue_elems(self):
        self.rq = self.set_queue[0]
        self.wq = self.set_queue[1]
        self.queue_process = self.set_queue[2]
        self.queue_ans_process = self.set_queue[3]
        self.queue_db = self.set_queue[4]
        self.queue_log = self.set_queue[-1]
        self.queue_n2t = self.rq
        self.queue_t2n = self.wq

    async def send_log(self, coroname, level, message, exc):
        msg = MessageLog(rdbnow(), coroname, level, message, MSGException(*exc))
        self.queue_log.put({"log":[msg.rdb]})
        
    def set_stats_params(self):
        # set ring buffer control
        self.mu = 0.5
        self.factor = 0.8
        self.sigma = 0.3
        self.buffer_size = 120*60
        self.u_base = self.mu + self.factor * self.sigma 
        self.acc = 15

    def set_datafolder(self, folder):
        """
        Set another, different, folder to save data
        """
        self.folder = folder

    def set_id(self, lista):
        """
        Defines a new id for stations, check if exists
        """
        ids = my_random_string(self.uin)
        while True:
            if ids not in lista:
                lista.append(ids)
                break
            else:
                ids = my_random_string(self.uin)
        return ids

    def set_ids(self):
        """
        Defines a new id for stations, check if exists
        """
        return self.set_id(self.ids)

    def set_idd(self):
        """
        Defines a new id for stations, check if exists
        """
        return self.set_id(self.idd)

    def set_ipt(self, ipt=""):
        """
        Defines a new id for relation process-collect_task, check if exists
        """
        if ipt:
            self.ipt.append(ipt)
        else:
            ipt = self.set_id(self.ipt) 
        return ipt

    def set_ico(self, ico):
        """
        Defines a new id for task related to collect data insice a worker, check if exists
        """
        if ico:
            self.ipt.append(ico)
        else:
            ico = self.set_id(self.ipt) 

        return ico

    def set_idm(self):
        """
        Defines a new id for relation incoming messages, check if exists
        """
        return self.set_id(self.idm)

    def load_stations(self):
        u = load_stations(self.server_name, self.datadb, log_path=self.log_path/"orm")  # ok
        for m in u:
            print(m)
            keys = ['id', 'code', 'db', 'dblist', 'ECEF_X', 'ECEF_Y', 'protocol_host',
                    'ECEF_Z', 'port', 'protocol', 'host', 'dbname']
            try:
                station = dict(
                    id=m['id'],
                    code=m['st_code'],
                    name=m['st_name'],
                    ECEF_X=m['ecef_x'],
                    ECEF_Y=m['ecef_y'],
                    ECEF_Z=m['ecef_z'],
                    db=m['db_code'],
                    dblist=m['db_list'],
                    port=m['st_port'],
                    protocol=m['prt_name'],
                    protocol_host=m['protocol_host'],
                    host=m['st_host'],
                    on_db=True
                )
                (ids, sta) = self.add_station(**station)
                # print(station)
            except Exception as exc:
                raise exc

    def add_station(self, **sta):
        """
        Add station to list for data adquisition
        """
        try:
            keys = ['id',
                    'code',
                    'name',
                    'ECEF_X',
                    'ECEF_Y',
                    'ECEF_Z',
                    'host',
                    'port',
                    'interface_port',
                    'db',
                    'dblist',
                    'protocol',
                    'protocol_host',
                    'on_db',
                    'ipt']
            ids = self.set_ids()

            # if ids in self.enqueued:
            #     self.enqueued.remove(ids)
            # self.enqueued.add(ids)

            station = dict(ids=ids)

            for k in keys:
                if k in sta.keys():
                    if k == 'protocol':
                        station[k] = sta.get(k, 'None').upper()
                    else:
                        station[k] = sta.get(k, None)
                else:
                    if k == 'host':
                        station[k] = 'localhost'
                    elif k == 'port' or k == 'interface_port':
                        station[k] = 0
                    elif k in  [f'ECEF_{v}' for v in ("X", "Y", "Z")]:
                        station[k] = 0
                    else:
                        station[k] = None
            self.stations.update({ids: station})
            self.status_sta.update({ids: False})
            self.first_time.update({ids: True})
            return (ids, sta)
        except Exception as ex:
            raise ex

    def update_station(self, ids, **sta):
        """
        Add station to list for data adquisition
        """
        try:
            keys = ['id',
                    'code',
                    'name',
                    'ECEF_X',
                    'ECEF_Y',
                    'ECEF_Z',
                    'host',
                    'port',
                    'interface_port',
                    'db',
                    'dblist',
                    'protocol',
                    'protocol_host',
                    'on_db',
                    'ipt']
            station = dict(ids=ids)

            for k in keys:
                if k in sta.keys():
                    if k == 'protocol':
                        station[k] = sta.get(k, 'None').upper()
                    else:
                        station[k] = sta.get(k, None)
                else:
                    if k == 'host':
                        station[k] = 'localhost'
                    elif k == 'port' or k == 'interface_port':
                        station[k] = 0
                    elif k in  [f'ECEF_{v}' for v in ("X", "Y", "Z")]:
                        station[k] = 0
                    else:
                        station[k] = None
            self.stations.update({ids: station})
            self.status_sta.update({ids: False})
            self.first_time.update({ids: True})
            return (ids, sta)
        except Exception as ex:
            raise ex


    def get_stations_keys(self):
        return list(self.stations.keys())

    def load_databases(self):
        u = load_databases(self.datadb, log_path=self.log_path/"orm")
        # ok
        groups = {}
        for m in u:
            dbtype = m['type_name']
            kwargs = dict(
                id=m['id'],
                code=m['code'],
                path=m['path'],
                host=m['host'],
                port=m['port'],
                user=m['user'],
                passw=m['passw'],
                info=m['info'],
                type_name=m['type_name'],
                type_db=m['type_db'],
                url=m['url'],
                data_list=m['data_list'],
                dbname=m["dbname"].rstrip(),
                address=(m['host'], m['port']),
                log_path=self.log_path/"rdb",
                on_db=True)
            groups[(m["host"], m["port"])] = kwargs
        #print("Different db destinies", len(groups), groups.keys())
        for opts in groups.values():
            self.new_datadb(dbtype, **opts)

    def new_datadb(self, dbtype, **kwargs):
        """
        Here you give the argument for every type engine for store data colected
        and instantiate the db for enable query on that
        """
        # generate a idd= database instance identifier
        try:
            keys = [
                'id',
                'user',
                'passw',
                'code',
                'host',
                'port',
                'name',
                'path',
                'data_list',
                'type_name',
                'dbname',
                'type_db,'
                'url',
                'info',
                'address',
                'on_db',
                'log_path']
            uin = 4
            idd = self.set_idd()
            # create namedtuple/dataclass 
            db_data = dict(idb=idd, name=dbtype, args={})
            for k in keys:
                if k in keys:
                    if k in kwargs.keys():
                        db_data['args'][k] = kwargs[k]
                    else:
                        if k == 'localhost':
                            db_data['args'][k] = 'localhost'
                        elif k == 'port':
                            db_data['args'][k] = 0
                        else:
                            db_data['args'][k] = ''
            self.db_data[idd] = db_data
            return idd, db_data
        except Exception as ex:
            raise ex

    def mod_station(self, ids, key, value):
        """
        Modify some value in station info

        """
        if key in self.stations.get(ids).keys():
            self.stations[ids][key] = value

    def del_station(self, ids):
        """
        Delete a station from list
        """
        del self.stations[ids]
        del self.status_sta[ids]
        del self.status_conn[ids]
        del self.instances[ids]
        k = self.ids.index(ids)
        del self.ids[k]

    def save_db(self, dbmanager, tname, args):
        """
        Save data to tname with args
        """
        # TODO: actualizar la lista de campos port table
        # TODO: añadir serverinstance
        input_args = dict(
            station=[
                'code',
                'name',
                'position_x',
                'position_y',
                'position_z',
                'host',
                'port',
                'interface_port',
                'db',
                'protocol'],
            dbdata=[
                'code',
                'path',
                'host',
                'port',
                'user',
                'passw',
                'info',
                'dbtype'],
            dbtype=['typedb', 'name', 'url', 'data_list'],
            protocol=['name', 'red_url', 'class_name', 'git_url']
        )
        name_args = input_args[tname]
        my_args = []
        id_instance = None
        if dbmanager == None:
            dbmanager = SessionCollector()
            instance = object
            if tname == 'station':
                instance = dbmanager.station(**args)
            elif tname == 'dbdata':
                instance = dbmanager.dbdata(**args)
            elif tname == 'dbtype':
                instance = dbmanager.dbtype(**args)
            elif tname == 'protocol':
                instance = dbmanager.protocol(**args)
            id_instance = instance.id
            return id_instance

    def save_station(self, ids):
        """
        Save station to database
        """
        # check if exists
        # if exist get data and compare
        # then update
        # if not, save
        pass

    def drop_station(self, ids):
        """
        Delete station from database
        """
        # get id from station ids
        # delete on database
        pass

    def del_db(self, varlist):
        """
        Delete element from database identified by idx in varlist
        """
        pass
###############

    def add_sta_instance(self, ids, loop):
        """
        Crear la instancia que accederá a los datos
        a través del socket
        """
        station =  self.stations.get(ids)
        if station:
            protocol = self.stations[ids]['protocol']
            kwargs = self.stations[ids]
            self.stations[ids].update({'on_collector': True})
            kwargs['code'] = self.stations[ids]['code']
            kwargs['host'] = self.stations[ids]['protocol_host']
            kwargs['port'] = self.stations[ids]['port']
            kwargs['sock'] = None
            kwargs['timeout'] = self.gsof_timeout
            kwargs["raise_timeout"] = False
            kwargs['loop'] = loop
            kwargs['log_path'] = self.log_path/"protocols"
            instance = self.collect_objects[protocol](**kwargs)
            code = kwargs["code"]
            table_name = f"{code}_{protocol}"
            return instance, table_name
        else:
            print("No station")
            return (None, None)

    def set_status_sta(self, ids:str, value:bool)->None:
        if isinstance(value, bool):
            self.status_sta[ids] = value

    def set_status_conn(self, ids:str, value:bool)->None:
        if isinstance(value, bool):
            self.status_conn[ids] = value

    def del_sta(self, ids:str):
        del self.instances[ids]
        del self.status_sta[ids]
        del self.status_conn[ids]
        del self.first_time[ids]
        # del self.db_instances[ids]
        del self.ids

    def get_tname(self, varname):
        assert isinstance(varname, str)
        if varname == 'STA' or varname == 'STATION':
            return 'station'
        elif varname == 'DB' or varname == 'DBDATA':
            return 'database'
        elif varname == 'PROT' or varname == 'PROTOCOL':
            return 'protocol'
        elif varname == 'DBTYPE':
            return 'dbtype'
        else:
            return None

    def get_id_by_code(self, varname, code):
        if varname == 'STATIONS':
            this_var = self.stations
            for k in this_var.keys():
                if this_var[k]['code'] == code:
                    return k

        elif varname == 'DBDATA':
            this_var = self.db_data
            # variable in function dbtype
            for k in this_var.keys():
                # code_r=''
                try:
                    if this_var[k]['args']['code'] == code:
                        return k
                except Exception as ex:
                    raise ex

    def get_var(self, varname):
        varin = ''
        if varname == 'STA':
            varin = self.stations
        elif varname == 'DB':
            varin = self.db_data
        else:
            varin = None
        return varin

    async def connect(self, ids):
        if self.status_sta[ids]:
            await self.instances[ids].connect()
            self.set_status_conn(ids, True)
            self.set_status_sta(ids, False)
            self.first_time[ids] = False

    async def stop(self, ipt, ids):
        if self.status_sta[ids]:
            icos = [ico_dict for ipt, ico_dict in self.assigned_tasks.items()]
            ico_list = []
            for ico_dict in icos:
                ico_list += [ico for ico, _ids in ico_dict.items()
                             if _ids == ids]
            for ico in ico_list:
                self.unset_sta_assigned(ipt, ico, ids)
                instance_obj = self.instances.get(ids, Aux())
                await instance_obj.stop()
                self.set_status_conn(ids, False)
                self.set_status_sta(ids, False)

    async def reset_station_conn(self, sta_insta, ids, idc):
        self.set_status_sta(ids, False)
        self.set_status_conn(ids, False)
        self.first_time[ids] = True       
        v = 1
        message = ""
        if idc:
            try:
                await sta_insta.close(idc)
                message = f"Station {sta_insta.station} closed at {idc}"
            except Exception as e:
                print("sta insta yet closed")
            except asyncio.TimeoutError as te:
                print("sta insta yet closed")
        return message, logging.INFO

    def connect_to_sta(self, ids):
        return self.sta_init[ids] and not self.status_conn[ids] and self.first_time[ids]

    def is_connected(self, ids):
        return self.sta_init[ids] and self.status_conn[ids] and not self.first_time[ids]        

    def add_db_instance(self, ipt):
        """
        Create a new instance for ending database to save the raw data

        """
        try:
            if self.db_data:
                rdbs_destinies = [key for key in self.db_data.keys()]
                key_data = rdbs_destinies.pop()
                data = self.db_data.get(key_data)
                name_db = data['name']
                object_db = self.database_objects[name_db]
                data.update({
                    "dbname": data["args"]["dbname"],
                    'address': data["args"]["address"],            
                    'hostname': 'atlas'})
                db_insta = object_db(**data)
                self.rethinkdb[ipt] = False
                self.db_init[ipt] = True
                self.db_connect[ipt] = True
                if data['name'] == 'RethinkDB':
                    self.rethinkdb[ipt] = True

                self.db_instances[ipt]  = db_insta

                return db_insta 
            else:
                print("Ipt not in DB_DATA")
                return None
        except Exception as ex:
            print("Error creando instancia database %s" % format(self.db_data))
            raise ex


    def db_task(self):
        # enable db task
        # Queue log join
        queue_log = asyncio.Queue()        
        loop = asyncio.get_event_loop()
        queue_db = self.queue_db
        control = {f"DB_{i}":DBSteps.CREATE for i in range(24)}        
        counter = {}
        task_name = f"db_task"
        ipt = "DB_PROCESS"
        flags = {key:True for key in control}
        db_instances = {}
        assigned = {key:{} for key in control}
        backup = {key:{} for key in control}
        db_args = [ipt, control, queue_db, db_instances,
                   counter, now(), now(), flags, assigned, backup, queue_log]
        db_task = TaskLoop(self.db_work, db_args, {"last_dataset":{}},
                           name=task_name)
        db_task.set_name(f"db_task_{ipt}")
        db_task.create()

        # Queue log join

        args = ["db_task_log", queue_log,]
        task_name = f"queue_log_join:{ipt}"
        task_db = TaskLoop(
            self.queue_log_join,
            args,
            {},
            **{"name": task_name})
        task_db.set_name(task_name)
        task_db.create()
        
        # db task to receive and send to pool rdb

        
        if not loop.is_running():
            loop.run_forever()


    async def db_work(self, ipt, control, queue_db, 
                      db_instances, counter,
                      last_data, last_time, flags, assigned, backup,
                      queue_log, **kwargs):
        """
        TODO: Control exceptions
        """
        # task_name = asyncio.Task.current_task()
        level = Logger.INFO
        messages = []
        message = ""
        kwargs["dataset"] = []
        last_dataset = kwargs["last_dataset"]
        now_check = now()
        task_name = f"db_task_{ipt}"
        loop = asyncio.get_event_loop()
        coroname = "db_work"
        exc = MSGException()
        control_changes = {}
        cnow = now()
        free = set()
        for key, futures in assigned.items():
            db_insta = db_instances.get(key)
            task_group = all(map(lambda f:f.done(), futures.values()))
            falses = {t:f for t,f in futures.items() if not f.done()}
            if task_group:
                flags[key] = True
                free.add(key)
            elif falses:  
                tosend = {}
                for table_name, future in falses.items():
                    bk = backup.get(key,{}).get(table_name)
                    time = bk.get("time")
                    dataset = bk.get("dataset")
                    if (not future.done()) and (cnow >= time +timedelta(seconds=15)):
                        await db_insta.close()
                        future.cancel()
                        tosend[table_name] = dataset
                        #await queue_db.put([])
                        # TODO
                        control[key] = DBSteps.CONNECT
                        exc = MSGException()
                        try:
                            future.exception()
                        except Exception as e:
                            exc = MSGException(*sys.exc_info())
                            message  = f"Task cancelled for {key}->{table_name}"
                            level = Logger.ERROR                            
                            messages.append(MessageLog(rdbnow(),coroname, level, message, exc))                            

                if tosend:
                    queue_db.put(tosend)
        for key in free:
            assigned[key] = {}

        for key, dbcontrol in control.items():
            db_insta = db_instances.get(key)
            if dbcontrol == DBSteps.CREATE and db_insta:
                if db_insta:
                    await db_insta.close()
                    del db_insta
                db_insta = None
                message = f"Deleted weird db instance at ipt {ipt}, db {key}"
                level = Logger.WARNING
                messages.append(MessageLog(rdbnow(),coroname, level, message, exc))

        for key, dbcontrol in control.items():
            #print("KEY", key,"CONTROL", dbcontrol)
            db_insta = db_instances.get(key)
            if dbcontrol == DBSteps.CREATE or not db_insta:
                db_insta = self.add_db_instance(ipt)
                db_instances[key] =  db_insta
                kwargs["instance"] = db_insta
                if db_insta:
                    control_changes[key] = DBSteps.CONNECT
                    message = f"RDB  {db_insta} at {ipt} created and passed to connect, db {key}"
                else:
                    message = f"RDB  {db_insta} at {ipt} can't created and try to recreate, db {key}"
                    level = Logger.WARNING
                    rprint("cannot create db object")
                #messages.append((level, message, {}))
                messages.append(MessageLog(rdbnow(), coroname, level, message, exc))

        control.update(control_changes)
        #print({k:db.active for k, db in db_instances.items()})

        for key, dbcontrol in control.items():
            db_insta = db_instances.get(key)
            if db_insta and dbcontrol == DBSteps.CONNECT:
                if not db_insta.active:
                    exc = MSGException()
                    try:
                        address = db_insta.client_address
                        if db_insta.active:
                            await db_insta.close()
                            db_insta.clean_client()

                        future = asyncio.create_task(db_insta.async_connect())
                        stage = "connect"

                        # await queue_control.put((
                        #     task_name, 
                        #     now(), 
                        #     stage, 
                        #     future))

                        coro = await wait_for(
                            shield(future), 
                            timeout=20)

                        await asyncio.shield(db_insta.list_dbs())
                        await asyncio.shield(db_insta.create_db(db_insta.default_db))
                        await asyncio.shield(db_insta.list_tables())

                        
                        message = f"RDB {db_insta} at {ipt} was connected, then passed to save data, db {key}"
                        level = Logger.INFO
                        control_changes[key] = DBSteps.SAVE
                    except asyncio.CancelledError as e:
                        exc = MSGException(*sys.exc_info())                        
                        message = f"RDB {db_insta} at {ipt} has canceled task, but protected by shield"
                        level = Logger.ERROR
                        control_changes[key] = DBSteps.CONNECT
                        gprint(f"Reconnect to db  IPT -> {ipt}")
                        await db_insta.close()
                        messages.append(MessageLog(rdbnow(), coroname, level, message, exc))
                        
                    except Exception as e:
                        exc = MSGException(*sys.exc_info())
                        message = f"RDB  {db_insta} at {ipt} has an exception {e}"
                        level = Logger.CRITICAL
                        control_changes[key] = DBSteps.CONNECT
                        gprint(f"Exception connecting to db  IPT -> {ipt}, {e}")
                        await asyncio.sleep(3)
                        await db_insta.close()
                        messages.append(MessageLog(rdbnow(), coroname, level, message, exc))

                    #print(now(),f"{ipt} Rethinkdb connection", db_insta.client_address)
                    #messages.append((level, message, exc))
                    messages.append(MessageLog(rdbnow(), coroname, level, message, exc))
                    
                else:
                    exc = MSGException()
                    message = f"At {ipt} tried to connect but active {db_insta.active}"
                    level = Logger.WARNING
                    #messages.append((level, message, exc))
                    messages.append(MessageLog(rdbnow(), coroname, level, message, exc))
                    
                    control_changes[key] = DBSteps.SAVE
                    
        control.update(control_changes)
        tasks = []
        for key, dbcontrol in control.items():
            db_insta = db_instances.get(key)
            db_flag = flags.get(key, True)
            opts = {}
            #print(now(), f"Saving data db {key}, flag {db_flag}")
            if db_insta.active and dbcontrol == DBSteps.SAVE and (not queue_db.empty()) and db_flag:
                """
                Leer la cola queue que debe ser una tupla (table_name,
                data)
                chequear si existe, si no crear
                """
                dataset = {}
                for i in range(queue_db.qsize()):
                    item = queue_db.get()
                    for t, array in item.items():                        
                        if t not in dataset:
                            dataset[t] = []
                        dataset[t] += array
                queue_db.task_done()

                i = 0
                # maybe group by table_name and then save as bulk
                flags[key] = False
                opts[key] = True
                assigned[key] = {}
                
                for table_name, items in dataset.items():
                    if table_name not in db_insta.tables:
                        create = await db_insta.create_table(table_name)
                        await db_insta.create_index(
                            table_name,
                            index='DT_GEN')
                    
                    message = ""
                    dataset = items
                    if table_name:
                        exc = MSGException()
                        try:
                            # print(now(), f"Saving to {table_name}"+\
                            #       f"#{len(dataset)},
                            #       {db_insta.client_address}")
                            
                            last_dt_gen = last_dataset.get(table_name, rdbnow()-timedelta(seconds=5))
                            dt_gens = [elem.get("DT_GEN") for elem in
                                       dataset if
                                       elem.get("DT_GEN")>last_dt_gen]
                            last_dt = max(dt_gens)
                            
                            filtered_dataset = [d for d in dataset if
                                                d["DT_GEN"] in dt_gens]
                            future = asyncio.create_task(
                                db_insta.save_data(
                                    table_name, 
                                    filtered_dataset), 
                                name=f"save_data_{key}_{table_name}_{len(dataset)}")
                            for d in filtered_dataset:
                                print("SAVING",
                                      my_random_string(),
                                      f"{table_name} {d.get('TRACE', -1)} {d['DT_GEN']}")
                            last_dataset[table_name] = last_dt 
                            tasks.append(future)
                            assigned[key][table_name] = future
                           
                            backup[key][table_name] = {
                                "time":now(),
                                "dataset": dataset}
                            if table_name in counter:
                                counter[table_name] += len(dataset)
                            else:
                                counter[table_name] = 0

                            if counter[table_name] == 60:
                                message = f"At ipt {ipt} saved successfully last {counter[table_name]}"+\
                                    f" messages for {table_name}, last " +\
                                    f"result"
                                level = Logger.INFO
                                counter[table_name] = 0

                            last_data = now()
                        except asyncio.CancelledError as e:
                            message = f"RDB {db_insta} at {ipt} has canceled task, but protected by shield"
                            level = Logger.ERROR
                            exc = MSGException(*sys.exc_info())
                            messages.append(MessageLog(rdbnow(), coroname, level, message, exc))                            
                            control_changes[key] = DBSteps.CONNECT
                            gprint(f"Reconnect to db  IPT -> {ipt}")
                            await db_insta.close()
                            
                            break

                        
                        except Exception as e:
                            message = f"RDB {db_insta} at {ipt} has an exception {e}"
                            level = Logger.CRITICAL
                            exc = MSGException(*sys.exc_info())
                            messages.append(MessageLog(rdbnow(), coroname, level, message, exc))                            
                            control_changes[key] = DBSteps.CONNECT
                            gprint(f"Exception connecting to db {db_insta.client_address} IPT -> {ipt}, {e}")
                            await db_insta.close()
                            break

                        if message:
                            messages.append(MessageLog(rdbnow(), coroname, level, message, exc))
                            
        control.update(control_changes)
        asyncio.gather(*tasks, return_exceptions=True)
            # stage = "free"

            # await queue_control.put((
            #     task_name, 
            #     now(), 
            #     stage, 
            #     {}))

        #gprint(f"No data on queue, db_insta {db_insta}")
        if queue_db.empty():
            await asyncio.sleep(1)
        # do log
        if messages:
            queue_messages = {"log":[msg.rdb for msg in messages]}
            queue_db.put(queue_messages)
        if level not in {logging.INFO, logging.DEBUG}:
            await asyncio.sleep(5)

        return [ipt, control, queue_db,  
                db_instances, counter, 
                last_data, last_time,
                flags, assigned, backup,
                queue_log], kwargs
    
    async def process_data(self, 
                           ipt:str, 
                           ico:str, 
                           control:CSteps, 
                           sta_insta:Gsof,
                           last_data:datetime, 
                           last_time:datetime, 
                           counter:int, 
                           queue_control:asyncio.Queue, 
                           queue_log:asyncio.Queue,
                           *args, **kwargs
                           ) -> Tuple[Tuple[
                               str,
                               str,
                               CSteps,
                               datetime,
                               datetime,
                               int,
                               asyncio.Queue,
                               asyncio.Queue], Dict[str, Any]]:
        loop = asyncio.get_event_loop()
        ids = self.assigned_tasks.get(ipt, {}).get(ico, None)
        assigned_tasks = self.assigned_tasks.get(ipt, {})
        ids = assigned_tasks.get(ico)
        level = Logger.INFO
        messages = []
        message = ""
        task_name = f"process_sta_task:{ipt}:{ico}"
        coroname = "process_data"
        connected = kwargs["connected"]
        created = kwargs["created"]
        # print(now(), task_name, control, f"IDS {ids}")
        if now() >= last_time + timedelta(seconds=5):
            stage = "update"
            # await queue_control.put((task_name, now(), stage, sta_insta))
            # counter["DB_WORK"] = 0       

        if ids:
            if self.changes.get(ids, False):
                """
                Linked to db_loop, if there are a new change then
                create new instance, 
                """
                del sta_insta 
                sta_insta = None
                control = CSteps.CREATE

            code_db = self.stations.get(ids, {}).get('db')
            code = self.stations.get(ids, {}).get('code')
            idd = self.get_id_by_code('DBDATA', code_db)
            idc = self.idc.get(ids)
            if idc and sta_insta:
                if idc not in sta_insta.clients:
                    del sta_insta
                    sta_insta = None
                    control = CSteps.CREATE
            #############
            # For some actions that modify status of
            # the variables on this coroutine
            # self.free_ids[ids] = False
            # while self.wait.get(ids, False):
            #     await asyncio.sleep(.01)

            # if not self.status_sta[ids]:
            #     v = 1
            ##############
            """
            Si no se ha creado instancia de conexion a estación
            se crea

            sta_init un diccionario  {ids:bool}

            indice si la estación fue inicializada
            """
            check_0 = now()
            if control == CSteps.CREATE:
                # step 0 initialize the objects, source and end
                exc = MSGException()
                try:
                    sta_insta, table_name = self.add_sta_instance(
                        ids, loop)
                    # print(now(), f"STA INSTA {table_name} created")
                    try:
                        ring_buffer = kwargs.get("ring_buffer")                        
                        if not ring_buffer:
                            ring_buffer = RingBuffer(name=table_name,
                                                     size=self.buffer_size)
                            kwargs["ring_buffer"] = ring_buffer
                        else:
                            ring_buffer.clear()
                    except Exception as e:
                        print("Error al crear ring buffer", e)
                        message = f"RingBuffer for {table_name} can't be created because {e}"
                        level = Logger.ERROR
                        exc = MSGException(*sys.exc_info())
                        messages.append(MessageLog(rdbnow(), coroname, level, message, exc))
                            
                    
                    kwargs["table_name"] = table_name

                    message = f"Station instance {sta_insta} created "+\
                        f"for {table_name}, control {control.value}"
                    level = Logger.INFO
                    if sta_insta:
                        control = CSteps.CONNECT
                        self.changes[ids] = False
                except Exception as ex:
                    exc = MSGException(*sys.exc_info())
                    message = f"PD_00: Conexión de estación con falla-> {ids}:{code}"
                    level = Logger.ERROR
                    idc = self.idc.get(ids, None)
                    msg, close_level = await self.reset_station_conn(sta_insta, ids, idc)
                    control = CSteps.CREATE
                    kwargs["origin_exception"] = f"PD_00 + {code}"
                if message:
                    messages.append(MessageLog(rdbnow(), coroname, level, message, exc))

            """
            Si no se ha creado la instanca de database:
            se crea la db instancia
            """
            """
            En caso que instancia de collect a estacion se haya iniciado
            1° conectar
            2° extraer datos
            """
        else:
            await asyncio.sleep(.1)

        exc = MSGException()
        message = ""
        check_1 = now()
        
        if sta_insta:
            queue_db = kwargs.get("queue_db")
            table_name = kwargs.get("table_name")

            if control == CSteps.CONNECT:
                # step 1
                # si es primera vez de acceso
                # conectar al socket correspondiente
                # step 1.a connect and set flags to run data
                code = sta_insta.station            
                idc = None
                exc = MSGException()
                try:

                    future = asyncio.create_task(sta_insta.connect())
                    stage = "connect"
                    # print(now(), f"STA INSTA {table_name} connected")
                    # drop control
                    # await queue_control.put((
                    #     task_name,
                    #     now(),
                    #     stage,
                    #     future))

                    idc = await wait_for(
                        shield(future),
                        timeout=20)
                    check_1 = now()
                    kwargs["connected"] = True
                    if idc:
                        self.idc[ids] = idc
                        self.set_status_sta(ids, True)
                        self.set_status_conn(ids, True)
                        self.first_time[ids] = False
                        check_a = now()
                        
                        
                        control = CSteps.COLLECT
                        message = f"Station {sta_insta} connected at"+\
                            f" {ipt} "+\
                            f" to address {sta_insta.address}"
                        level = Logger.INFO
                    else:
                        control = CSteps.CONNECT
                        message = f"Station {sta_insta} not connected at"+\
                            f" {ipt} "+\
                            f" to address {sta_insta.address}"
                        level = Logger.WARNING

                    messages.append(MessageLog(rdbnow(), coroname, level, message, exc))
                except asyncio.TimeoutError as e:
                    exc = MSGException(*sys.exc_info())
                    message = f"Tiempo fuera para conectar instancia " +\
                        f"de estación {sta_insta} en ipt {ipt}, idc <{idc}>"
                    level = Logger.ERROR               
                    control = CSteps.CONNECT
                    msg, lvl = await self.reset_station_conn(
                        sta_insta,
                        ids,
                        idc)
                    control = CSteps.CONNECT
                    kwargs["connected"] = False
                    messages.append(MessageLog(rdbnow(), coroname, level, message, exc))


                except Exception as ex:
                    exc = MSGException(*sys.exc_info())
                    message = f"PD_02: Error al conectar estación {sta_insta}, ids {ids}, ipt  {ipt}, {ex}"
                    level = Logger.ERROR
                    control = CSteps.CONNECT
                    msg, lvl = await self.reset_station_conn(
                        sta_insta,
                        ids,
                        idc)
                    control = CSteps.CONNECT
                    kwargs["connected"] = False
                    messages.append(MessageLog(rdbnow(), coroname, level, message, exc))

                # si ya esta conectado :), obtener dato

            """
            Si ya está inicializado y conectad
            proceder a obtener datos
            """
            sta_dict = {}
            # print(now(), f"STA INSTA {table_name} pre-collect", control, table_name)
            
            if control == CSteps.COLLECT and table_name:
                check_2 = now()
                # if connected:
                #     print(f"Table {table_name}", f"Check connect {check_1}", f"{check_2}", "Connected", connected)
                #     print(f"From connect to collect first {(check_2-check_1)}")
                #     connected = False
                ring_buffer = kwargs.get("ring_buffer")
                code = sta_insta.station
                idc = self.idc.get(ids)
                exc = MSGException()
                # just for checking
                # step 1.b collect data and process to save the raw data
                try:
                    pre_get = now()                  
                    
                    async def get_records():
                        set_header = await sta_insta.get_message_header(idc)
                        done, sta_dict = await sta_insta.get_records()
                        dt0, source = gps_time(sta_dict, sta_insta.tipo)
                        dt_iso = rdb.iso8601(dt0.isoformat())
                        rnow = now()
                        recv_now = rdbnow()
                        # print(rnow)
                        delta = (rnow - dt0).total_seconds()
                        sta_dict.update({
                            "TRACE": (ipt, ids, idc),
                            'DT_GEN': dt_iso,
                            'DT_RECV': recv_now,
                            "DELTA_TIME": delta})
                        data = Data(**{v:sta_dict.get(key) for key,v in DATA_KEYS.items()})
                        ring_buffer.add(data)
                        last_data = recv_now
                        await queue_db.put((table_name, sta_dict))                
                        return delta, last_data
                    
                    # Control criteria
                    # queue_db.pu