FIX, CNT: Use a more robust event_table_pos && n_bytes inference by massich · Pull Request #6550 · mne-tools/mne-python

massich · 2019-07-10T12:49:46Z

This PR takes after #6537

massich · 2019-07-10T12:59:46Z

Test I've done for some private files I've been gathering for debug purposes:

import os.path as op

import numpy as np
import pytest

from mne import __file__ as _mne_file
from mne.utils import run_tests_if_main
from mne.io.cnt import read_raw_cnt
from mne.io.cnt._utils import CNTEventType1, CNTEventType2, CNTEventType3

def foo(fname):
    SETUP_NCHANNELS_OFFSET = 370
    SETUP_NSAMPLES_OFFSET = 864
    SETUP_EVENTTABLEPOS_OFFSET = 886

    def _compute(xx):
        return (900 + 75 * int(n_channels) +
                xx * int(n_channels) * int(n_samples))

    with open(fname, 'rb') as fid:

        fid.seek(SETUP_NSAMPLES_OFFSET)
        (n_samples,) = np.frombuffer(fid.read(4), dtype='<i4')

        fid.seek(SETUP_NCHANNELS_OFFSET)
        (n_channels,) = np.frombuffer(fid.read(2), dtype='<u2')

        fid.seek(SETUP_EVENTTABLEPOS_OFFSET)
        (readed_event_table_pos,) = np.frombuffer(fid.read(4), dtype='<i4')

    print('readed_event_table_pos: ', readed_event_table_pos)
    print('readed     b:', np.binary_repr(readed_event_table_pos, width=40))
    print('readed_    b:', np.binary_repr(readed_event_table_pos, width=40))
    print('compute(2) b:', np.binary_repr(_compute(2), width=40))
    print('compute(4) b:', np.binary_repr(_compute(4), width=40))
    print('compute(2) :', _compute(2))
    print('compute(4) :', _compute(4))

    readed_event_table_pos_feature = np.binary_repr(
        readed_event_table_pos).lstrip('-')

    for n_bytes_candidate in [2, 4]:
        computed_event_table_pos = _compute(n_bytes_candidate)
        if (
            np.binary_repr(computed_event_table_pos)
            .endswith(readed_event_table_pos_feature)
        ):
            n_bytes = n_bytes_candidate
            event_table_pos = computed_event_table_pos
            print('match found: (', 'n_bytes: ', n_bytes,
                  'event_table_pos: ', computed_event_table_pos, ')')
            break

        else:
            n_bytes, event_table_pos = None, None

    if event_table_pos is None:
        print('No match')

    return n_channels, n_samples, event_table_pos, n_bytes


pp = pytest.param
@pytest.mark.parametrize((
    'fname,expected_n_bytes,expected_event_type,expected_event_table_pos'), [
    pp(op.join(op.dirname(_mne_file),
               '../sandbox/data/914flankers.cnt'),
       4, CNTEventType2, 156474479),

    pp(op.join(op.dirname(_mne_file),
               '../sandbox/data/confidential/cnt/cont_68chan_32bit.cnt'),
       4, CNTEventType2, 57267440),

    pp(op.join(op.dirname(_mne_file),
               '../sandbox/data/confidential/cnt/'
               'pilote_resting_01_neurospin_2019-03-04_15-18-40.cnt'),
       2, CNTEventType2, 15518747),

    pp(op.join(op.dirname(_mne_file),
               '../sandbox/data/confidential/cnt/'
               'cont_22chan_4gb_32bit_toolong.cnt'),
       4, CNTEventType3, 4971618850),

    pp(op.join(op.dirname(_mne_file),
               '../sandbox/data/confidential/cnt/SampleCNTFile_16bit.cnt'),
       2, CNTEventType2, 133700),

    pp(op.join(op.dirname(_mne_file),
               '../sandbox/data/confidential/cnt/cnt_files/'
               'SampleCNTFile_16bit.cnt'),
       2, CNTEventType2, 133700),

    pp(op.join(op.dirname(_mne_file),
               '../sandbox/data/confidential/cnt/cnt_files/'
               'BoyoAEpic1_16bit.cnt'),
       2, CNTEventType2, 78536260),

    pp(op.join(op.dirname(_mne_file),
               '../sandbox/data/confidential/cnt/cnt_files/'
               'cont_67chan_resp_32bit.cnt'),
       4, CNTEventType2, 54570725),

    pp(op.join(op.dirname(_mne_file),
               '../sandbox/data/confidential/cnt/BoyoAEpic1_16bit.cnt'),
       2, CNTEventType2, 78536260),

    pp(op.join(op.dirname(_mne_file),
               '../sandbox/data/confidential/cnt/cont_67chan_resp_32bit.cnt'),
       4, CNTEventType2, 54570725),
], ids=[str(n) for n in range(10)]
)
def test_foo(fname, expected_n_bytes, expected_event_type,
             expected_event_table_pos):
    """Test reading raw cnt files."""
    print('\n')
    print('expected_n_bytes: ', expected_n_bytes)
    print('expected_event_table_pos: ', expected_event_table_pos)
    n_channels, n_samples, event_table_pos, n_bytes = foo(fname)
    assert True

trace:

rootdir: /home/sik/code/mne-python, inifile: setup.cfg
plugins: timeout-1.3.3, sugar-0.9.2, pudb-0.7.0, mock-1.10.3, faulthandler-1.5.0, cov-2.6.1
collecting ... 

expected_n_bytes:  4
expected_event_table_pos:  156474479
readed_event_table_pos:  156474479
readed     b: 0000000000001001010100111001110001101111
readed_    b: 0000000000001001010100111001110001101111
compute(2) b: 0000000000000100101001110011110011100101
compute(4) b: 0000000000001001010011100110001010100101
compute(2) : 78068965
compute(4) : 156132005
No match

 sandbox/mwe/6535_test_cnt.py ✓                                                            10% █         

expected_n_bytes:  4
expected_event_table_pos:  57267440
readed_event_table_pos:  57267440
readed     b: 0000000000000011011010011101010011110000
readed_    b: 0000000000000011011010011101010011110000
compute(2) b: 0000000000000001101101001111011000110000
compute(4) b: 0000000000000011011010011101010011110000
compute(2) : 28636720
compute(4) : 57267440
match found: ( n_bytes:  4 event_table_pos:  57267440 )
 sandbox/mwe/6535_test_cnt.py ✓✓                                                           20% ██        

expected_n_bytes:  2
expected_event_table_pos:  15518747
readed_event_table_pos:  15518747
readed     b: 0000000000000000111011001100110000011011
readed_    b: 0000000000000000111011001100110000011011
compute(2) b: 0000000000000000000000000101000100111011
compute(4) b: 0000000000000000000000001001100000110101
compute(2) : 20795
compute(4) : 38965
No match
 sandbox/mwe/6535_test_cnt.py ✓✓✓                                                          30% ███       

expected_n_bytes:  4
expected_event_table_pos:  4971618850
readed_event_table_pos:  676651554
readed     b: 0000000000101000010101001110001000100010
readed_    b: 0000000000101000010101001110001000100010
compute(2) b: 0000000010010100001010100111011010100010
compute(4) b: 0000000100101000010101001110001000100010
compute(2) : 2485810850
compute(4) : 4971618850
match found: ( n_bytes:  4 event_table_pos:  4971618850 )
 sandbox/mwe/6535_test_cnt.py ✓✓✓✓                                                         40% ████      

expected_n_bytes:  2
expected_event_table_pos:  133700
readed_event_table_pos:  133700
readed     b: 0000000000000000000000100000101001000100
readed_    b: 0000000000000000000000100000101001000100
compute(2) b: 0000000000000000000000100000101001000100
compute(4) b: 0000000000000000000000111111111001000100
compute(2) : 133700
compute(4) : 261700
match found: ( n_bytes:  2 event_table_pos:  133700 )
 sandbox/mwe/6535_test_cnt.py ✓✓✓✓✓                                                        50% █████     

expected_n_bytes:  2
expected_event_table_pos:  133700
readed_event_table_pos:  133700
readed     b: 0000000000000000000000100000101001000100
readed_    b: 0000000000000000000000100000101001000100
compute(2) b: 0000000000000000000000100000101001000100
compute(4) b: 0000000000000000000000111111111001000100
compute(2) : 133700
compute(4) : 261700
match found: ( n_bytes:  2 event_table_pos:  133700 )
 sandbox/mwe/6535_test_cnt.py ✓✓✓✓✓✓                                                       60% ██████    

expected_n_bytes:  2
expected_event_table_pos:  78536260
readed_event_table_pos:  78536260
readed     b: 0000000000000100101011100101111001000100
readed_    b: 0000000000000100101011100101111001000100
compute(2) b: 0001111111111111111111010100110111000100
compute(4) b: 0011111111111111111110101000010101000100
compute(2) : 137438776772
compute(4) : 274877547844
No match
 sandbox/mwe/6535_test_cnt.py ✓✓✓✓✓✓✓                                                      70% ███████   

expected_n_bytes:  4
expected_event_table_pos:  54570725
readed_event_table_pos:  54570725
readed     b: 0000000000000011010000001010111011100101
readed_    b: 0000000000000011010000001010111011100101
compute(2) b: 0000000000000001101000000110001100000101
compute(4) b: 0000000000000011010000001010111011100101
compute(2) : 27288325
compute(4) : 54570725
match found: ( n_bytes:  4 event_table_pos:  54570725 )
 sandbox/mwe/6535_test_cnt.py ✓✓✓✓✓✓✓✓                                                     80% ████████  

expected_n_bytes:  2
expected_event_table_pos:  78536260
readed_event_table_pos:  78536260
readed     b: 0000000000000100101011100101111001000100
readed_    b: 0000000000000100101011100101111001000100
compute(2) b: 0001111111111111111111010100110111000100
compute(4) b: 0011111111111111111110101000010101000100
compute(2) : 137438776772
compute(4) : 274877547844
No match
 sandbox/mwe/6535_test_cnt.py ✓✓✓✓✓✓✓✓✓                                                    90% █████████ 

expected_n_bytes:  4
expected_event_table_pos:  54570725
readed_event_table_pos:  54570725
readed     b: 0000000000000011010000001010111011100101
readed_    b: 0000000000000011010000001010111011100101
compute(2) b: 0000000000000001101000000110001100000101
compute(4) b: 0000000000000011010000001010111011100101
compute(2) : 27288325
compute(4) : 54570725
match found: ( n_bytes:  4 event_table_pos:  54570725 )
 sandbox/mwe/6535_test_cnt.py ✓✓✓✓✓✓✓✓✓✓                                                  100% ██████████
-------------------- generated xml file: /home/sik/code/mne-python/junit-results.xml --------------------


======================================= slowest 20 test durations =======================================

(0.00 durations hidden.  Use -vv to show these durations.)

Results (9.85s):
      10 passed

Compilation finished at Wed Jul 10 14:55:17

massich · 2019-07-10T13:14:13Z

914flankers is one of the files that fails. This one I found it publicly somewhere so I guess there is no problem in sharing it: https://www.dropbox.com/s/sj9u2mhx36i3lng/914flankers.cnt?dl=0

we use it in: #6025, #5493

massich · 2019-07-10T13:28:56Z

@AimerLee Feel free to:

open PRs to my repo
take over this code and open another PR (and I'll close this one)
ask for permission to push in this branch (and I'll add you as a collaborator in my repo)

massich · 2019-07-10T14:48:19Z

Travis already failed https://travis-ci.org/mne-tools/mne-python/jobs/556786989#L3415
and we have azp. canceling the build.

larsoner · 2020-08-24T19:00:15Z

I'll close this for now -- let's start a clean PR when someone is motivated to continue

Joan Massich added 5 commits July 10, 2019 11:09

fix

00916e0

update whatsnew

b0e27d9

clean up

b16c2f3

[skip travis][skip ci][skip circle][skip azp] fix typo

8b9b686

do no t use file size to drive the event_table_pos inference

6a93a33

massich mentioned this pull request Jul 10, 2019

FIX: reading large CNT files (more than 2Gb files) #6537

Merged

AimerLee mentioned this pull request Jul 10, 2019

Fix the function to extract last 32bits from number massich/mne-python#24

Open

larsoner closed this Aug 24, 2020

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

FIX, CNT: Use a more robust event_table_pos && n_bytes inference#6550

FIX, CNT: Use a more robust event_table_pos && n_bytes inference#6550
massich wants to merge 5 commits intomne-tools:masterfrom
massich:fix_cnt_better_auto_data_format

massich commented Jul 10, 2019

Uh oh!

massich commented Jul 10, 2019

Uh oh!

massich commented Jul 10, 2019 •

edited

Loading

Uh oh!

massich commented Jul 10, 2019

Uh oh!

massich commented Jul 10, 2019

Uh oh!

larsoner commented Aug 24, 2020

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

2 participants

Uh oh!

Conversation

massich commented Jul 10, 2019

Uh oh!

massich commented Jul 10, 2019

Uh oh!

massich commented Jul 10, 2019 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

massich commented Jul 10, 2019

Uh oh!

massich commented Jul 10, 2019

Uh oh!

larsoner commented Aug 24, 2020

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

2 participants

massich commented Jul 10, 2019 •

edited

Loading