Data schema

@globalschema 
class File(dj.Manual):
    '''Table for tracking files stored in (s3 or local) storages.

    This table stores metadata about files including their path, storage location,
    creation date, size and MD5 checksum. It provides methods for:

    - Deleting files from both the database and S3 storage (does not delete local files)
    - Downloading files from S3 to local storage (does not download local storagefiles)
    - Checking if files are archived in S3 Glacier storage (does not check local storage)

    The table is used as a base class for AnalysisFile which handles analysis outputs.
    '''
    definition = '''
    file_path                 : varchar(300)  # Path to the file
    storage = "{0}"           : varchar(12)   # storage name 
    ---
    file_datetime             : datetime      # date created
    file_size                 : double        # using double because int64 does not exist
    file_md5 = NULL           : varchar(32)   # md5 checksum
    '''.format(DEFAULT_RAW_STORAGE)
    storage = DEFAULT_RAW_STORAGE
    # Files get deleted from AWS if the user has permissions
    def delete(
            self,
            transaction = True,
            safemode  = None,
            force_parts = False):
        '''Delete files from both the database and S3 storage.

        Parameters
        ----------
        transaction : bool, optional
            Whether to perform deletion as a transaction, by default True
        safemode : bool, optional
            Whether to run in safe mode, by default None
        force_parts : bool, optional
            Whether to force deletion of parts, by default False

        Raises
        ------
        ValueError
            If files are deleted from database but not from S3
        '''

        from ..s3 import s3_delete_file
        from tqdm import tqdm
        filesdict = [f for f in self]
        super().delete(transaction = transaction,
                       safemode = safemode,
                       force_parts = force_parts)
        if len(self) == 0:
            files_not_deleted = []
            files_kept = []
            for s in tqdm(filesdict,desc = f'Deleting objects from s3 {"storage"}:'):
                fname = s["file_path"]
                storage = prefs['storage'][s['storage']]
                if storage['protocol'] == 's3':
                    try:
                        s3_delete_file(fname,
                                   storage = prefs['storage'][s['storage']],
                                   remove_versions = True)
                    except Exception as err:
                        print(f'Could not delete {fname}.')
                        files_not_deleted.append(fname)
                else:
                    print(f'Skipping {fname} because it is not in S3.')
                    files_kept.append(fname)
            if len(files_not_deleted):
                print('\n'.join(files_not_deleted))
                raise(ValueError('''

[Integrity error] Files were deleted from the database but not from AWS.

            Save this message and show it to your database ADMIN.

{0}

'''.format('\n'.join(files_not_deleted))))
            if len(files_kept):
                print('Files where not deleted from the local storage.')

    def check_if_files_local(self, local_paths = None):
        '''
        Checks if files are in a local path, searches accross all local paths

        Parameters
        ----------
        local_paths : list of str or Path, optional
            List of local paths to check for files, by default None uses paths in preferences

        Returns
        -------
        tuple
            Tuple of local file paths and missing files

        Raises
        ------
        ValueError
            If no files in the object
        '''
        if local_paths is None:
            local_paths = prefs['local_paths']
        if not len(self):
            raise(ValueError('No files to get.'))
        # this does not work with multiple storages
        files = [f['file_path'] for f in self]
        localfiles = [find_local_filepath(a, local_paths = local_paths) for a in files]
        # check if they exist and download only missing files.
        missingfiles = []
        for f in files:
            if not np.any([str(l).endswith(str(Path(f))) for l in localfiles]):
                missingfiles.append(f)
        return [l for l in localfiles if not l is None], missingfiles

    def get(self,local_paths = None, check_if_archived = True, restore=True, download = True,):
        '''Download files from S3 to local storage.

        Parameters
        ----------
        local_paths : list of str or Path, optional
            List of local paths to download files to, by default None uses paths in preferences
        check_if_archived : bool, optional
            Whether to check if files are in Glacier storage, by default True
        restore : bool, optional
            Whether to restore archived files, by default True
        download : bool, optional
            Whether to actually download the files, by default True

        Returns
        -------
        list
            List of local file paths that were downloaded

        Raises
        ------
        ValueError
            If no files are found to download
        '''
        if local_paths is None:
            local_paths = prefs['local_paths']
        if not len(self):
            raise(ValueError('No files to get.'))

        localfiles, remotefiles = self.check_if_files_local(local_paths = local_paths)
        storage = [f['storage'] for f in self][0]
        remotefiles = self & [dict(file_path = f) for f in remotefiles]
        if len(remotefiles):
            if prefs['storage'][storage]['protocol'] == 's3':
                if check_if_archived:
                    # TODO: add to the preference file to not restore by default.
                    self.check_if_files_archived(files = remotefiles, restore = restore)
                if download:
                    print(f'Downloading {len(remotefiles)} files from S3 [{storage}].')
                    remotefiles = [r['file_path'] for r in remotefiles]
                    dstfiles = [Path(local_paths[0])/f for f in remotefiles]  # place to store file.
                    from ..s3 import copy_from_s3
                    copy_from_s3(remotefiles,dstfiles,storage_name = storage)
                    localfiles, _ = self.check_if_files_local(local_paths = local_paths)
            elif prefs['storage'][storage]['protocol'] == 'local':
                # TODO, copy files from local storage to the first local path.
                print('Downloading from local storage is not implemented, use local_paths.')
        return localfiles

    def check_if_files_archived(self, files = None, restore = True, suppress_error = False):
        '''Check if files are archived in S3 Glacier storage.

        Parameters
        ----------
        restore : bool, optional
            Whether to initiate restore of archived files, by default True
        suppress_error : bool, optional
            Whether to suppress error if files are being restored, by default False

        Returns
        -------
        bool
            True if files are archived, False otherwise

        Raises
        ------
        OSError
            If storage is not in preferences or if files are being restored
        '''
        files_restoring = []
        import boto3
        if files is None:
            files = self
        for f in files:
            # check if files are archived
            # TODO: run this in parallel because it takes a while.
            if not f['storage'] in prefs['storage'].keys():
                raise(OSError(f"Store {f['storage']} is not in the preference file."))
            store = prefs['storage'][f['storage']]

            s3 = boto3.resource('s3',aws_access_key_id = store['access_key'],
                            aws_secret_access_key = store['secret_key'])

            obj = s3.Object(bucket_name = store['bucket'],
                            key = f['file_path'])

            if not obj.archive_status is None and 'ARCHIVE' in obj.archive_status:
                if obj.restore is None:
                    if restore:
                        resp = obj.restore_object(RestoreRequest = {})
                    files_restoring.append(f['file_path'])
                elif 'true' in obj.restore:
                    files_restoring.append(f['file_path'])
        if len(files_restoring):
            import warnings
            warnings.warn(f"Files are being restored [{files_restoring}]")
            if not suppress_error:
                raise(OSError(f"Files are being restored [{files_restoring}]"))
            return True # files are in arquive
        return False # files are not in archive

@globalschema # users need permission to delete from this table
class AnalysisFile(File):
    '''Table for tracking analysis output files stored in the analysis S3 bucket.

    Extends ``File`` with upload helpers.  Analysis results (e.g. spike sorting
    outputs, segmentation masks) are uploaded here so they can be retrieved by
    any lab member with access to the analysis storage.

    See Also
    --------
    File : Base class for raw data file tracking.
    '''
    definition = '''
    file_path                 : varchar(300)  # Path to the file
    storage = "{0}"           : varchar(12)   # storage name
    ---
    file_datetime             : datetime      # date created
    file_size                 : double        # using double because int64 does not exist
    file_md5 = NULL           : varchar(32)   # md5 checksum
    '''.format(DEFAULT_ANALYSIS_STORAGE)
    storage = DEFAULT_ANALYSIS_STORAGE

    def upload_files(self, src, dataset, force=True):
        '''Upload analysis result files to the analysis S3 bucket.

        Computes MD5 checksums, uploads to S3, and inserts records into
        ``AnalysisFile``.  If a file already exists in the database it is
        deleted first when ``force=True``.

        Parameters
        ----------
        src : list of str or Path
            Local file paths to upload.
        dataset : dict
            Dataset key with keys ``subject_name``, ``session_name``,
            ``dataset_name``.  Used to construct the destination path in S3.
        force : bool, optional
            Overwrite existing database entry if the file was already uploaded.
            Default is True.

        Returns
        -------
        list of dict
            Each dict has ``file_path`` and ``storage`` keys pointing to the
            newly inserted ``AnalysisFile`` entries.
        '''

        dst = self.generate_filepaths(src, dataset)
        for d in dst:
            if len(AnalysisFile() & dict(file_path = d)) > 0:
                if not force:
                    ValueError(f'File is already in database, delete it to re-upload {d}.')
                else:
                    (AnalysisFile() & dict(file_path = d)).delete(safemode = False) # just the table
        assert self.storage in prefs['storage'].keys(),ValueError(
            'Specify an {self.storage} bucket in preferences["storage"].')
        # compute checksum and sizes
        md5 = compute_md5s(src)
        dates = [datetime.utcfromtimestamp(Path(f).stat().st_mtime) for f in src]
        sizes = [Path(f).stat().st_size for f in src]
        # upload to s3
        if prefs['storage'][self.storage]['protocol'] == 's3':
            from ..s3 import copy_to_s3
            copy_to_s3(src, dst, md5_checksum=None, storage_name = self.storage)
        elif prefs['storage'][self.storage]['protocol'] == 'local':
            pass # nothing to do in local storage
        else:
            raise NotImplemented(f"AnalysisFile does not handle protocol {prefs['storage'][self.storage]['protocol']}")
        # insert in AnalysisFile if all went well
        self.insert([dict(file_path = f,
                          storage = self.storage,
                          file_datetime = d,
                          file_md5 = m,
                          file_size = s) for f,d,s,m in zip(dst,dates,sizes,md5)])
        return [dict(file_path = f,storage = self.storage) for f in dst]

    def generate_filepaths(self, src, dataset):
        '''Generate destination S3 paths for analysis files.

        Parameters
        ----------
        src : list of str or Path
            Source local file paths (only the filename part is kept).
        dataset : dict
            Dataset key with ``subject_name``, ``session_name``, ``dataset_name``.

        Returns
        -------
        list of str
            Destination paths relative to the analysis storage root.
        '''
        assert 'subject_name' in dataset.keys(), ValueError('dataset must have subject_name')
        assert 'session_name' in dataset.keys(), ValueError('dataset must have session_name')
        assert 'dataset_name' in dataset.keys(), ValueError('dataset must have dataset_name')
        destpath = '{subject_name}/{session_name}/{dataset_name}/'.format(**dataset)
        if not schema_project is None:
            destpath = f'{schema_project}/{destpath}' # add the project name so these files can have the same name across projects or be shared easily.
        return [destpath+Path(k).name for k in src]

@dataschema
class Procedure(dj.Manual):
    '''Table for tracking experimental procedures performed on subjects.

    Each procedure entry includes:
    - Subject
    - ProcedureType
    - Date and time
    - Lab member who performed it
    - Optional metadata: weight, and notes
    '''
    definition = """
    -> Subject
    -> ProcedureType
    procedure_datetime            : datetime
    ---
    -> LabMember
    procedure_metadata = NULL     : longblob   
    -> [nullable] Weighing
    -> [nullable] Note
    """

@dataschema
class ProcedureType(dj.Lookup):
    '''Table defining types of experimental procedures.

    This lookup table enumerates the different types of experimental procedures, including:
    - Surgical procedures (surgery, implants, craniotomy)
    - Behavioral procedures (handling, training) 
    - Other manipulations (injections)

    The procedure types are used by the Procedure table to categorize and track all
    procedures performed.
    '''
    definition = """
    procedure_type : varchar(52)       #  Defines procedures that are not an experimental session
    """
    contents = zip(['surgery',
                    'chronic implant',
                    'chronic explant', 
                    'injection',
                    'window implant',
                    'window replacement',
                    'handling',
                    'training',
                    'craniotomy'])

@dataschema
class Subject(dj.Manual):
    ''' Experimental subject.'''
    definition = """
    subject_name               : varchar(20)          # unique mouse id
    ---
    subject_dob                : date                 # mouse date of birth
    subject_sex                : enum('M', 'F', 'U')  # sex of mouse - Male, Female, or Unknown
    -> Strain
    -> LabMember
    """

@dataschema
class Session(dj.Manual):
    '''An experimental session for a given subject.

    A session groups one or more datasets collected on the same day or
    experimental block. The ``session_name`` is typically a date string
    (``YYYY-MM-DD``) or a descriptive identifier unique per subject.

    Examples
    --------
    Query all sessions for a subject::

        Session() & dict(subject_name='sub001')

    Fetch sessions with experimenter info::

        (Session() * LabMember().proj(experimenter='user_name')).fetch(as_dict=True)
    '''
    definition = """
    -> Subject
    session_name             : varchar(54)     # session identifier
    ---
    session_datetime         : datetime        # experiment date
    -> [nullable] LabMember.proj(experimenter = 'user_name')
    """

@dataschema
class Dataset(dj.Manual):
    '''A single experimental dataset within a session.

    A dataset corresponds to one recording block — for example, one run of
    a behavioral task, one ephys recording, or one imaging session.  Multiple
    datasets can exist per session (e.g. multiple probes, multiple tasks).

    The ``Dataset.DataFiles`` part table links each dataset to the files in
    ``File`` that were acquired during that recording.

    Examples
    --------
    Find all ephys datasets for a subject::

        Dataset() & dict(subject_name='sub001', dataset_type='ephys')

    List files belonging to a dataset::

        (File() & Dataset.DataFiles() & dict(subject_name='sub001',
                                             session_name='2024-01-15',
                                             dataset_name='ephys_g0_t0'))
    '''
    definition = """
    -> Subject
    -> Session
    dataset_name             : varchar(128)
    ---
    -> [nullable] DatasetType
    -> [nullable] Setup
    -> [nullable] Note
    """
    class DataFiles(dj.Part):
        '''Files acquired as part of this dataset.'''
        definition = '''
        -> master
        -> File
        '''

@dataschema
class DatasetEvents(dj.Imported):
    '''Synchronization events and timestamps from each acquisition stream.

    Each row represents one named stream (e.g. ``nidq``, ``bpod``, ``imec0``)
    within a dataset.  The ``Digital`` part table holds per-event timestamps
    and values; ``AnalogChannel`` holds continuous analog data.

    Use ``DatasetEvents.Digital.fetch_synced()`` to retrieve events already
    converted to a common clock via ``StreamSync``.

    Examples
    --------
    Fetch all events for a dataset::

        (DatasetEvents.Digital() & dict(subject_name='sub001',
                                        session_name='2024-01-15',
                                        dataset_name='ephys_g0_t0')).fetch(as_dict=True)
    '''
    definition = '''
    -> Dataset
    stream_name                       : varchar(54)   # which clock is used e.g. btss, nidq, bpod, imecX
    ---
    stream_time = NULL                 : longblob     # for e.g. the analog channels
    '''
    class Digital(dj.Part):
        '''Digital event timestamps from one stream within a dataset.

        ``event_timestamps`` are in the native clock of ``stream_name``.
        Call ``fetch_synced()`` to retrieve timestamps already converted to
        the master clock defined in ``StreamSync``.
        '''
        definition = '''
        -> master
        event_name                    : varchar(54)
        ---
        event_timestamps = NULL       : longblob  # timestamps of the events
        event_values = NULL           : longblob  # event value or count
        '''
        projkeys = ['subject_name','session_name','dataset_name','stream_name','event_name']

        def fetch_synced(self, force = False,method = 'cubic-spline'):
            ''' 
            Returned events already synchronized between data streams, following the StreamSync table.
            method: cubic-spline, piecewise-linear
            '''
            keys = [dict(subject_name = s["subject_name"],
                         session_name = s["session_name"],
                         dataset_name = s["dataset_name"],
                         stream_name = s["stream_name"]) for s in self]
            evnts = []
            streams = (StreamSync() & keys)
            if not len(streams):
                from warnings import warn
                warn(f'There are no StreamSync for events {self.proj()}. This will return only the clock stream.')
            else:    
                for s in streams:
                    evs = (self & dict(stream_name = s["stream_name"])).fetch(as_dict = True)
                    func = (StreamSync() & s).apply(None, force = force,method = method)
                    for evnt in evs:
                        if not evnt['event_timestamps'] is None:
                            evnt['event_timestamps'] = func(evnt['event_timestamps'])
                            evnts.append(evnt)
            # add the events from the clock stream
            if len(streams):
                evs = (self & dict(stream_name = streams.clock_stream())).fetch(as_dict = True)
            else:
                evs = self
            for evnt in evs:
                evnts.append(evnt)
            return evnts

        def plot_synced(self, stream_colors = 'krbgyb', overlay_original = False, lw = 1,force = True):
            ''' Plot DatasetEvents.Digital.'''
            evnts = self.fetch_synced(force = force)
            ustreams = [n for n in np.unique([e['stream_name'] for e in evnts])]

            import pylab as plt
            caption = []
            ticks = []
            lns = []
            for i,e in enumerate(evnts):
                ln = plt.vlines(e['event_timestamps'],i,i+0.7,
                                color = stream_colors[np.mod(ustreams.index(e['stream_name']),len(stream_colors))],
                                lw = lw)
                if overlay_original:
                    ee = (DatasetEvents.Digital() & {k:e[k] for k in self.projkeys}).fetch('event_timestamps')[0]
                    plt.vlines(ee,i+0.6,i+0.9,color='gray',lw = lw)
                lns.append(ln)
                ticks.append(i+0.35)
                caption.append(f'{e["stream_name"]}_{e["event_name"]}')
            plt.yticks(ticks,caption);
            return lns

    class AnalogChannel(dj.Part):
        '''Continuous analog channel values from one stream within a dataset.'''
        definition = '''
        -> master
        channel_name                 : varchar(54)
        ---
        channel_values = NULL        : longblob  # analog values for channel
        '''

@analysisschema
class StreamSync(dj.Manual):
    '''Inter-stream synchronization mapping for a dataset.

    Maps a synchronization pulse shared between two acquisition streams so
    that timestamps from one stream can be converted to the clock of another.
    One stream is designated the *clock* (master); the other carries a copy
    of the same pulse that is used to align time axes.

    Use ``StreamSync.apply()`` to convert timestamps or
    ``DatasetEvents.Digital.fetch_synced()`` to retrieve all events already
    aligned to the master clock.

    Examples
    --------
    Insert a sync entry (clock = nidq, sync pulse = bpod)::

        StreamSync().insert1(dict(subject_name='sub001',
                                  session_name='2024-01-15',
                                  dataset_name='ephys_g0_t0',
                                  stream_name='bpod', event_name='clock',
                                  clock_dataset='ephys_g0_t0',
                                  clock_stream='nidq', clock_stream_event='clock'))
    '''
    definition = '''
    -> Dataset
    -> DatasetEvents.Digital
    -> DatasetEvents.Digital.proj(clock_stream='stream_name',clock_stream_event='event_name',clock_dataset = 'dataset_name')
    '''
    def get_interp_data(self,force = False, warn = True, allowed_offset = 2):
        ''' Force will attempt to remove events from the longest stream so the streams are matched. '''

        assert len(self)==1, ValueError(f"This function only takes one element at a time not {len(self)}.")
        s = self.fetch1()
        clock = (DatasetEvents.Digital() & dict(subject_name = s['subject_name'],
                                                session_name = s['session_name'],
                                                dataset_name = s['clock_dataset'],
                                                stream_name = s['clock_stream'],
                                                event_name = s['clock_stream_event'])).fetch1()
        clock_onsets = np.array(clock['event_timestamps'])
        sync = (DatasetEvents.Digital() & dict(subject_name = s['subject_name'],
                                               session_name = s['session_name'],
                                               dataset_name = s['dataset_name'],
                                               stream_name = s['stream_name'],
                                               event_name = s['event_name'])).fetch1()
        sync_onsets = np.array(sync['event_timestamps'])
        if ((len(sync_onsets)>=(len(clock_onsets)//2)-allowed_offset) and 
            (len(sync_onsets)<=(len(clock_onsets)//2)+allowed_offset)): # in case clock has both onsets and offsets
            if clock['event_values'] is None:
                clock_onsets = clock_onsets[::2]
            else:
                clock_onsets = clock_onsets[np.array(clock['event_values']) == 1]
        if ((len(clock_onsets)>=(len(sync_onsets)//2)-allowed_offset) and 
            (len(clock_onsets)<=(len(sync_onsets)//2)+allowed_offset)): # in case sync has both onsets and offsets
            if sync['event_values'] is None:
                sync_onsets = sync_onsets[::2]
            else:
                sync_onsets = sync_onsets[np.array(sync['event_values'])==1]
        if warn: 
            if not len(sync_onsets) == len(clock_onsets):
                import warnings
                warnings.warn(f"There is a potential issue with the syncronization of sessions: {s['subject_name']} {s['session_name']}", UserWarning)
                print(f"    - stream {s['clock_stream']} channel {s['clock_stream_event']} {len(clock_onsets)}")
                print(f"    - stream {s['stream_name']} channel {s['event_name']} {len(sync_onsets)}")
        if not force: # by default this is set to false.
            assert len(clock_onsets) == len(sync_onsets), ValueError(f'\n\n Length of the clock and sync not the same? \n\n {self}')
        N = np.min([len(sync_onsets),len(clock_onsets)])
        return sync_onsets[:N],clock_onsets[:N]

    def apply(self, values, sync_onsets = None, warn = True, clock_onsets = None, force = False, method = 'cubic-spline'):
        '''
        Returns synchronized signals according a sync pulse shared from a clock.
        "clock" is main, "sync" is the same acquisition system as "values"
        '''
        if sync_onsets is None or clock_onsets is None:
            sync_onsets, clock_onsets = self.get_interp_data(force = force, warn = warn)
        if method == 'cubic-spline':
            # cubic spline interpolation handles the extrapolated points better
            from scipy.interpolate import CubicSpline
            func = CubicSpline(sync_onsets,clock_onsets)
        elif method == 'piecewise-linear':
            # linear interpolation to get the time of events syncronized across streams
            func = lambda x: np.interp(x,sync_onsets,clock_onsets)
        else:
            raise(ValueError(f'Unknown interpolation method {method}'))
        if values is None: # return a function if no values passed
            return func
        return func(values)

    def clock_stream(self):
        '''
        Returns the name of the clock stream(s)
        '''
        clkstreams = np.unique(self.fetch('clock_stream'))
        if len(clkstreams) == 1:
            return clkstreams[0]
        return clkstreams

@dataschema
class Watering(dj.Manual):
    '''Table for tracking water administration to subjects.

    This table records water consumed, including:
    - Subject receiving water
    - Date and time
    - Volume of water, in microliters

    '''
    definition = """
    -> Subject
    watering_datetime : datetime
    ---
    water_volume : float  # (uL)
    """

@dataschema
class Weighing(dj.Manual):
    '''Table for tracking subject weights.

    This table stores weight measurements for experimental subjects. Each entry includes:
    - Subject
    - Date and time of weighing
    - Weight in grams

    '''
    definition = """
    -> Subject
    weighing_datetime : datetime
    ---
    weight : float  # (g)
    """

@analysisschema
class DecisionTask(dj.Imported): # imported because if comes from data but there is no 'make'
    '''Table for behavioral decision task data.

    This table serves as a general schema for decision-making behavioral tasks,
    abstracting common features that can be inherited by specific task tables
    defined in plugins. Each entry includes:
    - Total trial counts (assisted, self-performed, initiated, with choice)
    - Performance metrics (rewarded, punished trials)
    - Optional reference to water intake during session

    The table includes a Part table (TrialSet) that stores detailed sets of trials within a session, 
    dependent on the modality or condition and includes including:
    - Trial conditions and modalities
    - Performance metrics per condition
    - Timing data (initiation times, reaction times) 
    - Response and subject feedback values
    - Stimulus parameters (intensity, block)

    This data can be used to compute:
    - Psychometric curves
    - Learning curves
    - Reaction time distributions
    - Choice biases and strategies
    - ...

    The schema is designed to be flexible and is meant to be populated by specific task tables (defined as plugins) 
    while maintaining a consistent interface for analysis and visualization.
    '''
    definition = '''
    -> Dataset
    ---
    n_total_trials              : int            # number of trials in the session
    n_total_assisted = NULL     : int            # number of assisted trials in the session
    n_total_performed = NULL    : int            # number of self-performed trials
    n_total_initiated = NULL    : int            # number of initiated trials
    n_total_with_choice = NULL  : int            # number of self-initiated with choice
    n_total_rewarded = NULL     : int            # number of rewarded trials
    n_total_punished = NULL     : int            # number of punished trials
    -> [nullable] Watering                       # water intake during the session (ml) 
    '''

    class TrialSet(dj.Part):
        '''Per-condition trial subset within a behavioral session.

        Stores counts, performance metrics, timing, and per-trial arrays for
        one modality or condition (e.g. easy vs hard, left vs right).
        Multiple ``TrialSet`` rows can coexist for a single session.
        '''
        definition = '''
        -> master
        trialset_description     : varchar(54) # e.g. trial modality, unique condition
        ---
        n_trials                 : int         # total number of trials
        n_performed              : int         # number of self-performed trials
        n_with_choice            : int         # number of self-initiated trials with choice 
        n_correct                : int         # number of correct trials
        performance_easy = NULL  : float       # performance on easy trials
        performance = NULL       : float       # performance on all trials
        trial_num                : longblob    # trial number because TrialSets can be intertwined
        initiation_times = NULL  : longblob    # time between trial start and stim onset
        assisted = NULL          : longblob    # wether the trial was assisted
        response_values = NULL   : longblob    # left=1;no response=0; right=-1        
        correct_values = NULL    : longblob    # correct = 1; no_response  = NaN; wrong = 0        
        intensity_values = NULL  : longblob    # value of the stim (left-right)
        reaction_times = NULL    : longblob    # between onset of the response period and reporting  
        block_values = NULL      : longblob    # block number for each trial
        '''