Browse Source

Merge branch 'add/nix_rawio' of NeuralEnsemble/ephy_testing_data into master

Samuel Garcia 1 year ago
parent
commit
fd3fb56ab9

+ 1 - 1
config.yml

@@ -1,3 +1,3 @@
 annex:
     minsize: 0
-    exclude: ["*README*", "*LICENSE*", "config*", "*.py", "**/.git*"]
+    exclude: ["*README*", "*LICENSE*", "config*", "*.py", "*.sh", "**/.git*"]

+ 7 - 0
nix/README.md

@@ -0,0 +1,7 @@
+# Test files for the [Neuroscience Information eXchange (NIX)](https://github.com/G-Node/nix) format
+
+The `generated_testfile_neo_*.nix`files were automatically generated using `generated_nix_testfiles.sh` and `generate_nix_testfiles.py`. To regenerate these you need to have conda installed and run. Data are generated randomly using a fixed seed and should be identical across all files.
+
+```
+. .generate_nix_testfiles.sh
+```

+ 146 - 0
nix/generate_nix_testfiles.py

@@ -0,0 +1,146 @@
+from packaging.version import Version
+import numpy as np
+import neo
+import quantities as pq
+
+n_segments = 2
+n_spiketrains = 8
+n_waveform_samples = 10
+n_analogsignals = 4
+n_irregularlysampledsignals = 1
+n_events = 2
+n_epochs = 3
+
+random_generator = np.random.default_rng(seed=42)
+
+
+def get_rand(shape=None, min=0, max=1, sorted=False):
+    data = random_generator.random(shape)
+
+    #rescaling random numbers to min-max range
+    data = data*(max-min) + min
+    if sorted:
+        data = np.sort(data)
+    return data
+
+def generate_basic_block():
+    block = neo.Block(name=r'my_block')
+    for seg_idx in range(n_segments):
+        seg = neo.Segment(name=f'my_segment_{seg_idx}')
+        block.segments.append(seg)
+
+        for spiketrain_idx in range(n_spiketrains):
+            waveforms = get_rand((10, 14)) * pq.V
+            st = neo.SpikeTrain(times=get_rand((10), max=10, sorted=True)*pq.s, t_stop=10*pq.s,
+                                name=f'my_spiketrain_{spiketrain_idx}', waveforms=waveforms,
+                                left_sweep=4)
+            st.segment = seg
+            seg.spiketrains.append(st)
+
+        for anasig_idx in range(n_analogsignals):
+            anasig = neo.AnalogSignal(signal=get_rand((100, anasig_idx+2),max=1000)*pq.V,
+                                      t_start=0*pq.s,
+                                      sampling_rate=(anasig_idx+1)*pq.Hz,
+                                      name=f'my_analogsignal_{anasig_idx}')
+            anasig.segment = seg
+            seg.analogsignals.append(anasig)
+
+        for irr_idx in range(n_irregularlysampledsignals):
+            irrsig = neo.IrregularlySampledSignal(times=get_rand((100), sorted=True, max=100)*pq.s,
+                                                  signal=get_rand((100,anasig_idx+2),max=1000)*pq.V,
+                                                  t_start=0*pq.s, t_stop=100*pq.s,
+                                                  name=f'my_irregularlysampledsignal_{irr_idx}')
+            irrsig.segment = seg
+            seg.irregularlysampledsignals.append(irrsig)
+
+        for ev_idx in range(n_events):
+            event = neo.Event(times=get_rand((10),max=10,sorted=True)*pq.s,
+                               labels=np.array([f'my_event_timestamp_{i}' for i in range(10)]),
+                               name=f'my_event_{ev_idx}')
+            event.segment = seg
+            seg.events.append(event)
+
+        for ep_idx in range(n_epochs):
+            epoch = neo.Epoch(times=get_rand((3),max=6,sorted=True)*pq.s,
+                              durations=[1.1, 2.2, 3.3]*pq.s,
+                              labels=np.array([f'my_epoch_timestamp_{i}' for i in range(3)]),
+                              name=f'my_epoch_{ep_idx}')
+            epoch.segment = seg
+            seg.epochs.append(epoch)
+
+    return block
+
+
+def add_spiketrain_groups(block):
+    spiketrain_groups = []
+    for group_idx in range(n_spiketrains):
+        group = neo.Group(name=f'my_spiketrain_group_{group_idx}', allowed_types=[neo.SpikeTrain])
+        spiketrain_groups.append(group)
+
+    # assign spiketrains to groups
+    for seg_idx, seg in enumerate(block.segments):
+        for idx, st in enumerate(seg.spiketrains):
+            st.group = spiketrain_groups[idx]
+            spiketrain_groups[idx].spiketrains.append(st)
+
+    # attach groups to block
+    block.groups.extend(spiketrain_groups)
+
+
+def add_spiketrain_units_channel_indexes(block):
+    # Linking pattern:
+    # 1 Channel_unit - last channel of analog signal
+    #                \- 2 spiketrains
+
+    spiketrain_units = []
+    for unit_idx in range(n_spiketrains):
+        unit = neo.Unit(f'my_unit_{unit_idx}')
+        spiketrain_units.append(unit)
+
+    channel_indexes = []
+    for asig_idx in range(n_analogsignals):
+        signals = [seg.analogsignals[asig_idx] for seg in block.segments]
+        channel_index = neo.ChannelIndex([asig_idx], name=f'my_channel_index_{asig_idx}')
+        channel_index.analogsignals.extend(signals)
+        channel_indexes.append(channel_index)
+        channel_index.block = block
+
+
+    # assign spiketrain to units
+    for seg_idx, seg in enumerate(block.segments):
+        for idx, st in enumerate(seg.spiketrains):
+            st.unit = spiketrain_units[idx]
+            spiketrain_units[idx].spiketrains.append(st)
+
+    # link channel_index to block
+    block.channel_indexes.extend(channel_indexes)
+
+    # attach units to channel_indexes
+    for idx in range(n_spiketrains):
+        # link two spiketrains to a single channel_index
+        channel_indexes[idx//2].units.append(spiketrain_units[idx])
+        spiketrain_units[idx].channel_index = channel_indexes[idx//2]
+
+
+if __name__ == '__main__':
+    neo_version = Version(neo.__version__)
+    print(f'Generating test file for neo version {neo_version} ...', end='')
+
+    block = generate_basic_block()
+
+    # adding spiketrain units to neo block depending on container objects available
+    if hasattr(neo, 'Group'):
+        add_spiketrain_groups(block)
+    elif hasattr(neo, 'ChannelIndex'):
+        add_spiketrain_units_channel_indexes(block)
+    else:
+        raise ValueError(f'No mechanism to represent spike units found for neo version: {neo_version}')
+
+    io = neo.NixIO(f'generated_file_neo{neo_version}.nix')
+    io.write_block(block)
+    io.close()
+
+    print('done')
+
+
+

+ 20 - 0
nix/generate_nix_testfiles.sh

@@ -0,0 +1,20 @@
+conda create -y -n generate_testfiles python=3.9 pip packaging numpy=1.20.1
+conda activate generate_testfiles
+echo "$CONDA_PREFIX"
+pip install nixio
+neo_versions=('0.6.1' '0.7.2' '0.8.0' '0.9.0' '0.10.2' '0.11.1' '0.12.0')
+pip install numpy==1.20.0
+
+for neo_version in "${neo_versions[@]}"; do
+  pip install --quiet --upgrade neo==$neo_version
+  pip install --quiet nixio==1.5.2
+  if [[ ${neo_version} == '0.6.1' ]]; then
+    echo "Installing old nixio version for ancient neo versions"
+    pip install --quiet nixio==1.4.9
+  fi
+
+  python generate_nix_testfiles.py
+done
+
+conda deactivate
+conda env remove --name generate_testfiles

+ 1 - 0
nix/generated_file_neo0.10.2.nix

@@ -0,0 +1 @@
+../.git/annex/objects/JZ/qk/MD5E-s1119453--3ee3ce2650ec24c62dc470418e6643dc.2.nix/MD5E-s1119453--3ee3ce2650ec24c62dc470418e6643dc.2.nix

+ 1 - 0
nix/generated_file_neo0.11.1.nix

@@ -0,0 +1 @@
+../.git/annex/objects/1G/6z/MD5E-s1119453--a595956eac98bbc5fc4ff3f9847acd89.1.nix/MD5E-s1119453--a595956eac98bbc5fc4ff3f9847acd89.1.nix

+ 1 - 0
nix/generated_file_neo0.12.0.nix

@@ -0,0 +1 @@
+../.git/annex/objects/6x/8M/MD5E-s1119453--fc9ee563dad771cf6563d7a8fa2c2985.0.nix/MD5E-s1119453--fc9ee563dad771cf6563d7a8fa2c2985.0.nix

+ 1 - 0
nix/generated_file_neo0.6.1.nix

@@ -0,0 +1 @@
+../.git/annex/objects/m5/JW/MD5E-s3047297--2ea3c91faee07db72977f7b284216e93.1.nix/MD5E-s3047297--2ea3c91faee07db72977f7b284216e93.1.nix

+ 1 - 0
nix/generated_file_neo0.7.2.nix

@@ -0,0 +1 @@
+../.git/annex/objects/27/V5/MD5E-s1174027--3c01249cd3051af90a6cf94461c8de44.2.nix/MD5E-s1174027--3c01249cd3051af90a6cf94461c8de44.2.nix

+ 1 - 0
nix/generated_file_neo0.8.0.nix

@@ -0,0 +1 @@
+../.git/annex/objects/5m/96/MD5E-s1174027--6494ad5fecfbb330906190ffb2f5e284.0.nix/MD5E-s1174027--6494ad5fecfbb330906190ffb2f5e284.0.nix

+ 1 - 0
nix/generated_file_neo0.9.0.nix

@@ -0,0 +1 @@
+../.git/annex/objects/Gg/V0/MD5E-s1119453--5087d34e8e021828c1cdfb037531fb3d.0.nix/MD5E-s1119453--5087d34e8e021828c1cdfb037531fb3d.0.nix

+ 1 - 0
nix/nix_rawio_compatible.nix

@@ -0,0 +1 @@
+../.git/annex/objects/PX/1j/MD5E-s839646--7b37219ce9ef97a011727f4cb545e847.nix/MD5E-s839646--7b37219ce9ef97a011727f4cb545e847.nix