Clean up and harmonize group names

code/convert_to_nix.py

@@ -1,8 +1,16 @@
 """
-This script loads a complete session in the blackrock format and converts it to a single nix file
+This script loads a complete session in the blackrock format and converts it to
+a single nix file.
+
+For sessions without online recorded LFP (i.e., 1000Hz neural
+signal in the ns2 files), this LFP is generated offline by filtering and
+downsampling.
+
+Two versions of the nix files are saved: one containing the complete data, and
+a smaller one containing everything except the AnalogSignal object with the
+raw data records.
 """
 import os
-import copy
 
 import numpy as np
 import quantities as pq
@@ -15,10 +23,11 @@ from elephant.signal_processing import butter
 
 from reachgraspio import reachgraspio
 
+##### INITIAL SETUP ############
 
 # Choose which session you want to convert into a nix file
-#session = "i140703-001"
-session = "l101210-001"
+session = "i140703-001"
+#session = "l101210-001"
 
 # Input data. i.e., original Blackrock files and odML
 dataset_dir = '../datasets_blackrock'
@@ -38,22 +47,20 @@ session = reachgraspio.ReachGraspIO(
 
 block = session.read_block(lazy=False, load_waveforms=True)
 
-# =============================================================================
-# Create offline filtered LFP
-#
+
+##### CREATE LFP IF MISSING ############
 # Here, we construct one offline filtered LFP from each ns5 (monkey L) or ns6
 # (monkey N) raw recording trace. For monkey N, this filtered LFP can be
 # compared to the LFPs in the ns2 file (note that monkey L contains only
 # behavioral signals in the ns2 file). Also, we assign telling names to each
 # Neo AnalogSignal, which is used for plotting later on in this script.
-# =============================================================================
 
 # this factor was heuristically determined as an approximate shift introduced
 # by the online filtering. Here, the offline filtering does not introduce a
-# noticable shift, so we set it to zero.
+# noticeable shift, so we set it to zero.
 time_shift_factor = 0*pq.ms
 
-# identify neuronal signals and provide labels for plotting
+# Identify neuronal signals and provide labels for plotting
 filtered_anasig = None
 raw_anasig = None
 for anasig in block.segments[0].analogsignals:
@@ -61,7 +68,7 @@ for anasig in block.segments[0].analogsignals:
     if not anasig.annotations['neural_signal']:
         continue
 
-    # identify nsx source of signals in this AnalogSignal object
+    # Identify nsx source of signals in this AnalogSignal object
     if 'nsx' in anasig.annotations:
         nsx = anasig.annotations['nsx']
     else:
@@ -76,11 +83,12 @@ for anasig in block.segments[0].analogsignals:
         # AnalogSignal is raw signal from ns5 or ns6
         raw_anasig = anasig
 
+# Does the file contain a filtered signal, i.e., an LFP?
 if filtered_anasig is None:
     print("Filtering raw time series to obtain LFP")
+
+    # Filtering must be done channel by channel for memory reasons (requires approx. 32 GB RAM)
     for f in range(raw_anasig.shape[1]):
-        # filtering must be done channel by channel for memory reasons (requires approx. 32 GB RAM)
-        print(f"Processing channel {f}")
 
         filtered_signal = butter(
             raw_anasig[:, f],
@@ -89,13 +97,13 @@ if filtered_anasig is None:
             filter_function='sosfiltfilt',
             order=4)
 
-        # For other filters that may introduce a time shift, here would be the
-        # place to correct for this shift using:
-        # ... .time_shift(time_shift_factor)
         # Downsampling 30-fold here to get to 1000Hz from 30kHz
+        # Note: For filters that may introduce a time shift, here would be the
+        # place to correct for this shift using:
+        # [...] .time_shift(time_shift_factor)
         downsampled_signal=filtered_signal.downsample(30)
 
-        # first run? Create a new Analogsignal
+        # First run? Create a new Analogsignal
         if f == 0:
             offline_filtered_anasig = neo.AnalogSignal(
                 np.zeros((downsampled_signal.shape[0], raw_anasig.shape[1]), dtype=np.float32) *\
@@ -114,8 +122,6 @@ if filtered_anasig is None:
 
             # all array annotations of the raw signal also apply to the filtered
             # signal
-            # offline_filtered_anasig.array_annotations = copy.copy(
-            #     raw_anasig.array_annotations)
             offline_filtered_anasig.array_annotate(
                 **raw_anasig.array_annotations)
 
@@ -127,21 +133,34 @@ if filtered_anasig is None:
         nsx = anasig.array_annotations["nsx"][0]
 
     offline_filtered_anasig.name = f"NeuralTimeSeriesDownsampled"
-    offline_filtered_anasig.description = "Downsampled continuous neuronal recordings, where the downsampling was " \
-                           "performed off-line during post-processing"
+    offline_filtered_anasig.description = "Downsampled continuous neuronal " \
+            "recordings, where the downsampling was " \
+            "performed off-line during post-processing"
+
+    offline_filtered_group = neo.Group(
+        name="offline",
+        stream_id=''
+    )
+    offline_filtered_group.analogsignals.append(offline_filtered_anasig)
 
-    # Attach all offline filtered LFPs to the segment of data
+    # Attach all offline filtered LFPs to the segment of data and the groups
     block.segments[0].analogsignals.insert(0, offline_filtered_anasig)
+    block.groups.insert(0, offline_filtered_group)
 
 
 ##### MISC FIXES ###################
 
-# for lilou, behavior channels were not set with nice names in the setup
+# For lilou, behavior channels were not set with nice names in the setup.
 # Here, we hard code these handwritten annotations to make both recordings
 # more similar
 block.segments[0].analogsignals[1].array_annotate(
     channel_names=['GFpr1', 'GFside1', 'GFpr2', 'GFside2', 'LF', 'Displ'])
 
+# Unify group names of ns2 for Nikos session
+if session == "i140703-001":
+    block.groups[0].name = 'nsx2'
+    block.groups[1].name = 'nsx2'
+
 
 ##### SAVE NIX FILE ###################
 nix_filename = nix_session_path + '.nix'
@@ -152,6 +171,7 @@ else:
         print(f'Saving nix file at {nix_filename}')
         io.write_block(block)
 
+
 ##### VALIDATION OF FILE CONTENT ######
 with neo.NixIO(nix_filename, mode='ro') as io:
     blocks = io.read_all_blocks()
@@ -198,3 +218,17 @@ with neo.NixIO(nix_filename, mode='ro') as io:
             assert_same_annotations(ep_old, ep_new)
             assert_same_array_annotations(ep_old, ep_new)
         print(f'Epochs are equivalent.')
+
+
+##### SAVE SMALL NIX FILE ###################
+# remove raw signal
+del block.segments[0].analogsignals[2]
+del block.groups[2]
+
+nix_filename = nix_session_path + '_no_raw.nix'
+if os.path.exists(nix_filename):
+    print('Nix file already exists and will not be overwritten.')
+else:
+    with neo.NixIO(nix_filename) as io:
+        print(f'Saving nix file at {nix_filename}')
+        io.write_block(block)